From f0f1805a3f1ec935fd05fac6d9d51ddcb4eba957 Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 00:14:21 +0800 Subject: [PATCH 01/33] docs: use compile-only cross-platform checks --- ...07-11-transparent-session-filesystem-implementation.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md b/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md index 5dab6d7..b3e8fd0 100644 --- a/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md +++ b/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md @@ -476,8 +476,8 @@ Run: ```bash go test ./internal/mountfs -count=1 go test -race ./internal/mountfs -count=1 -CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test ./internal/mountfs -count=1 -CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go test ./internal/mountfs -count=1 +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test -c -o /tmp/codexfold-mountfs-linux.test ./internal/mountfs +CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go test -c -o /tmp/codexfold-mountfs-windows.test.exe ./internal/mountfs go test ./... -count=1 ``` @@ -625,8 +625,8 @@ go test ./... -count=1 go test -race ./... -count=1 go vet ./... go build ./cmd/codexfold -CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test ./... -count=1 -CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go test ./... -count=1 +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ./cmd/codexfold +CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build ./cmd/codexfold ``` Expected: all PASS. From 17564e98de1c470d7af5d5817553491e76b74b01 Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 00:14:48 +0800 Subject: [PATCH 02/33] feat: add transactional packed object resolver --- internal/fold/stream.go | 57 ++++++ internal/pack/build.go | 294 +++++++++++++++++++++++++++++++ internal/pack/cache.go | 59 +++++++ internal/pack/doctor.go | 94 ++++++++++ internal/pack/format.go | 74 ++++++++ internal/pack/pack_test.go | 200 +++++++++++++++++++++ internal/pack/replace_unix.go | 9 + internal/pack/replace_windows.go | 36 ++++ internal/pack/resolver.go | 189 ++++++++++++++++++++ 9 files changed, 1012 insertions(+) create mode 100644 internal/fold/stream.go create mode 100644 internal/pack/build.go create mode 100644 internal/pack/cache.go create mode 100644 internal/pack/doctor.go create mode 100644 internal/pack/format.go create mode 100644 internal/pack/pack_test.go create mode 100644 internal/pack/replace_unix.go create mode 100644 internal/pack/replace_windows.go create mode 100644 internal/pack/resolver.go diff --git a/internal/fold/stream.go b/internal/fold/stream.go new file mode 100644 index 0000000..c93db55 --- /dev/null +++ b/internal/fold/stream.go @@ -0,0 +1,57 @@ +package fold + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "hash" + "io" + "os" + + "github.com/klauspost/compress/zstd" +) + +type objectStream struct { + ref ObjectRef + file *os.File + decoder *zstd.Decoder + hash hash.Hash + read int64 + done bool +} + +func (s *ObjectStore) OpenStream(ref ObjectRef) (io.ReadCloser, error) { + file, err := os.Open(s.ObjectPath(ref.SHA256)) + if err != nil { + return nil, fmt.Errorf("open object %s: %w", ref.SHA256, err) + } + decoder, err := zstd.NewReader(file) + if err != nil { + _ = file.Close() + return nil, fmt.Errorf("create object stream %s: %w", ref.SHA256, err) + } + return &objectStream{ref: ref, file: file, decoder: decoder, hash: sha256.New()}, nil +} + +func (s *objectStream) Read(destination []byte) (int, error) { + n, err := s.decoder.Read(destination) + if n > 0 { + _, _ = s.hash.Write(destination[:n]) + s.read += int64(n) + } + if err == io.EOF && !s.done { + s.done = true + if s.read != s.ref.RawBytes { + return n, fmt.Errorf("object %s raw size %d, want %d", s.ref.SHA256, s.read, s.ref.RawBytes) + } + if hex.EncodeToString(s.hash.Sum(nil)) != s.ref.SHA256 { + return n, fmt.Errorf("object %s SHA-256 mismatch", s.ref.SHA256) + } + } + return n, err +} + +func (s *objectStream) Close() error { + s.decoder.Close() + return s.file.Close() +} diff --git a/internal/pack/build.go b/internal/pack/build.go new file mode 100644 index 0000000..4441107 --- /dev/null +++ b/internal/pack/build.go @@ -0,0 +1,294 @@ +package pack + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/jstar0/codexfold/internal/fold" + "github.com/klauspost/compress/zstd" +) + +type BuildOptions struct { + BlockBytes int64 + PackBytes int64 + BeforePublish func() error +} + +type BuildResult struct { + Generation string `json:"generation"` + ObjectCount int `json:"object_count"` + BlockCount int `json:"block_count"` + PackCount int `json:"pack_count"` + RawBytes int64 `json:"raw_bytes"` + StoredBytes int64 `json:"stored_bytes"` +} + +type packWriter struct { + directory string + limit int64 + sequence int + file *os.File + name string + offset int64 +} + +func Build(ctx context.Context, storeDir string, options BuildOptions) (BuildResult, error) { + if storeDir == "" { + return BuildResult{}, errors.New("pack store directory is required") + } + if options.BlockBytes <= 0 { + options.BlockBytes = defaultBlockBytes + } + if options.PackBytes <= 0 { + options.PackBytes = defaultPackBytes + } + if options.BlockBytes > int64(int(^uint(0)>>1)) { + return BuildResult{}, errors.New("pack block size exceeds platform integer size") + } + refs, err := referencedObjects(storeDir) + if err != nil { + return BuildResult{}, err + } + packsDir := filepath.Join(storeDir, "packs") + if err := os.MkdirAll(packsDir, 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create packs directory: %w", err) + } + temporaryDir, err := os.MkdirTemp(packsDir, ".generation-") + if err != nil { + return BuildResult{}, fmt.Errorf("create temporary pack generation: %w", err) + } + defer func() { _ = os.RemoveAll(temporaryDir) }() + generation := "gen-" + strings.TrimPrefix(filepath.Base(temporaryDir), ".generation-") + index := Index{Version: IndexVersion, Kind: IndexKind, Generation: generation, CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), BlockBytes: options.BlockBytes, Objects: make([]Object, 0, len(refs))} + result := BuildResult{Generation: generation, ObjectCount: len(refs)} + writer := &packWriter{directory: temporaryDir, limit: options.PackBytes} + encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault)) + if err != nil { + return BuildResult{}, fmt.Errorf("create pack encoder: %w", err) + } + defer encoder.Close() + store := fold.NewObjectStore(storeDir) + buffer := make([]byte, int(options.BlockBytes)) + for _, ref := range refs { + if err := ctx.Err(); err != nil { + return BuildResult{}, err + } + stream, err := store.OpenStream(ref) + if err != nil { + return BuildResult{}, err + } + object := Object{SHA256: ref.SHA256, RawBytes: ref.RawBytes} + objectHash := sha256.New() + var rawOffset int64 + for { + n, readErr := io.ReadFull(stream, buffer) + if n > 0 { + raw := buffer[:n] + _, _ = objectHash.Write(raw) + compressed := encoder.EncodeAll(raw, nil) + packName, packOffset, writeErr := writer.write(compressed) + if writeErr != nil { + _ = stream.Close() + return BuildResult{}, writeErr + } + blockHash := sha256.Sum256(raw) + object.Blocks = append(object.Blocks, Block{Pack: packName, PackOffset: packOffset, StoredBytes: int64(len(compressed)), RawOffset: rawOffset, RawBytes: int64(n), SHA256: hex.EncodeToString(blockHash[:])}) + rawOffset += int64(n) + result.BlockCount++ + result.RawBytes += int64(n) + result.StoredBytes += int64(len(compressed)) + } + if errors.Is(readErr, io.EOF) || errors.Is(readErr, io.ErrUnexpectedEOF) { + break + } + if readErr != nil { + _ = stream.Close() + return BuildResult{}, fmt.Errorf("read loose object %s: %w", ref.SHA256, readErr) + } + } + if err := stream.Close(); err != nil { + return BuildResult{}, err + } + if rawOffset != ref.RawBytes || hex.EncodeToString(objectHash.Sum(nil)) != ref.SHA256 { + return BuildResult{}, fmt.Errorf("loose object %s failed stream verification", ref.SHA256) + } + index.Objects = append(index.Objects, object) + } + if err := writer.close(); err != nil { + return BuildResult{}, err + } + result.PackCount = writer.sequence + if err := writeIndex(temporaryDir, index); err != nil { + return BuildResult{}, err + } + if err := verifyGeneration(ctx, temporaryDir, index); err != nil { + return BuildResult{}, fmt.Errorf("verify candidate pack generation: %w", err) + } + finalDir := filepath.Join(packsDir, generation) + if err := os.Rename(temporaryDir, finalDir); err != nil { + return BuildResult{}, fmt.Errorf("publish pack generation directory: %w", err) + } + if err := syncDirectory(packsDir); err != nil { + return BuildResult{}, err + } + if options.BeforePublish != nil { + if err := options.BeforePublish(); err != nil { + return BuildResult{}, err + } + } + if err := publishCurrent(packsDir, generation); err != nil { + return BuildResult{}, err + } + return result, nil +} + +func referencedObjects(storeDir string) ([]fold.ObjectRef, error) { + entries, err := os.ReadDir(filepath.Join(storeDir, "manifests")) + if err != nil { + return nil, fmt.Errorf("read manifests for pack build: %w", err) + } + refs := make(map[string]fold.ObjectRef) + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { + continue + } + sessionID := strings.TrimSuffix(entry.Name(), ".json") + manifest, err := fold.LoadManifest(storeDir, sessionID) + if err != nil { + return nil, err + } + for _, part := range manifest.Parts { + if existing, ok := refs[part.Object.SHA256]; ok && existing.RawBytes != part.Object.RawBytes { + return nil, fmt.Errorf("object %s has conflicting raw lengths", part.Object.SHA256) + } + refs[part.Object.SHA256] = part.Object + } + } + digests := make([]string, 0, len(refs)) + for digest := range refs { + digests = append(digests, digest) + } + sort.Strings(digests) + result := make([]fold.ObjectRef, 0, len(digests)) + for _, digest := range digests { + result = append(result, refs[digest]) + } + return result, nil +} + +func (w *packWriter) write(data []byte) (string, int64, error) { + if w.file == nil || (w.offset > 0 && w.offset+int64(len(data)) > w.limit) { + if err := w.rotate(); err != nil { + return "", 0, err + } + } + offset := w.offset + if _, err := w.file.Write(data); err != nil { + return "", 0, fmt.Errorf("write pack file: %w", err) + } + w.offset += int64(len(data)) + return w.name, offset, nil +} + +func (w *packWriter) rotate() error { + if err := w.closeCurrent(); err != nil { + return err + } + w.sequence++ + w.name = fmt.Sprintf("pack-%06d.pack", w.sequence) + file, err := os.OpenFile(filepath.Join(w.directory, w.name), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("create pack file: %w", err) + } + w.file = file + w.offset = 0 + return nil +} + +func (w *packWriter) closeCurrent() error { + if w.file == nil { + return nil + } + if err := w.file.Sync(); err != nil { + _ = w.file.Close() + return fmt.Errorf("sync pack file: %w", err) + } + if err := w.file.Close(); err != nil { + return fmt.Errorf("close pack file: %w", err) + } + w.file = nil + return nil +} + +func (w *packWriter) close() error { return w.closeCurrent() } + +func writeIndex(directory string, index Index) error { + data, err := json.Marshal(index) + if err != nil { + return fmt.Errorf("encode pack index: %w", err) + } + data = append(data, '\n') + path := filepath.Join(directory, "index.json") + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("create pack index: %w", err) + } + if _, err := file.Write(data); err != nil { + _ = file.Close() + return fmt.Errorf("write pack index: %w", err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + return fmt.Errorf("sync pack index: %w", err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("close pack index: %w", err) + } + return syncDirectory(directory) +} + +func publishCurrent(packsDir string, generation string) error { + temporary, err := os.CreateTemp(packsDir, ".CURRENT-") + if err != nil { + return fmt.Errorf("create temporary CURRENT: %w", err) + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if _, err := temporary.WriteString(generation + "\n"); err != nil { + _ = temporary.Close() + return fmt.Errorf("write CURRENT: %w", err) + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return fmt.Errorf("sync CURRENT: %w", err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close CURRENT: %w", err) + } + if err := replaceFile(temporaryPath, filepath.Join(packsDir, "CURRENT")); err != nil { + return fmt.Errorf("publish CURRENT: %w", err) + } + return syncDirectory(packsDir) +} + +func syncDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return fmt.Errorf("open directory for sync %s: %w", path, err) + } + defer directory.Close() + if err := directory.Sync(); err != nil { + return fmt.Errorf("sync directory %s: %w", path, err) + } + return nil +} diff --git a/internal/pack/cache.go b/internal/pack/cache.go new file mode 100644 index 0000000..769295b --- /dev/null +++ b/internal/pack/cache.go @@ -0,0 +1,59 @@ +package pack + +import ( + "container/list" + "sync" +) + +type cacheEntry struct { + key string + data []byte +} + +type blockCache struct { + mu sync.Mutex + budget int64 + used int64 + items map[string]*list.Element + lru list.List +} + +func newBlockCache(budget int64) *blockCache { + if budget < 0 { + budget = 0 + } + return &blockCache{budget: budget, items: make(map[string]*list.Element)} +} + +func (c *blockCache) get(key string) ([]byte, bool) { + c.mu.Lock() + defer c.mu.Unlock() + element, ok := c.items[key] + if !ok { + return nil, false + } + c.lru.MoveToFront(element) + return element.Value.(cacheEntry).data, true +} + +func (c *blockCache) put(key string, data []byte) { + if int64(len(data)) > c.budget || c.budget == 0 { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if existing, ok := c.items[key]; ok { + c.lru.MoveToFront(existing) + return + } + element := c.lru.PushFront(cacheEntry{key: key, data: data}) + c.items[key] = element + c.used += int64(len(data)) + for c.used > c.budget { + oldest := c.lru.Back() + entry := oldest.Value.(cacheEntry) + delete(c.items, entry.key) + c.used -= int64(len(entry.data)) + c.lru.Remove(oldest) + } +} diff --git a/internal/pack/doctor.go b/internal/pack/doctor.go new file mode 100644 index 0000000..8fff362 --- /dev/null +++ b/internal/pack/doctor.go @@ -0,0 +1,94 @@ +package pack + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + + "github.com/jstar0/codexfold/internal/fold" +) + +type DoctorIssue struct { + ObjectSHA256 string `json:"object_sha256,omitempty"` + Message string `json:"message"` +} + +type DoctorResult struct { + Generation string `json:"generation,omitempty"` + ObjectCount int `json:"object_count"` + VerifiedCount int `json:"verified_count"` + IssueCount int `json:"issue_count"` + Issues []DoctorIssue `json:"issues,omitempty"` +} + +func Doctor(ctx context.Context, storeDir string) (DoctorResult, error) { + resolver, err := Open(storeDir, OpenOptions{CacheBytes: 0}) + if err != nil { + return DoctorResult{IssueCount: 1, Issues: []DoctorIssue{{Message: err.Error()}}}, nil + } + defer resolver.Close() + result := DoctorResult{Generation: resolver.index.Generation, ObjectCount: len(resolver.index.Objects)} + for _, object := range resolver.index.Objects { + hasher := sha256.New() + buffer := make([]byte, 128<<10) + var offset int64 + failed := false + for offset < object.RawBytes { + n, readErr := resolver.ReadAt(ctx, fold.ObjectRef{SHA256: object.SHA256, RawBytes: object.RawBytes}, buffer, offset) + if n > 0 { + _, _ = hasher.Write(buffer[:n]) + offset += int64(n) + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + result.Issues = append(result.Issues, DoctorIssue{ObjectSHA256: object.SHA256, Message: readErr.Error()}) + failed = true + break + } + if n == 0 { + break + } + } + if !failed && (offset != object.RawBytes || hex.EncodeToString(hasher.Sum(nil)) != object.SHA256) { + result.Issues = append(result.Issues, DoctorIssue{ObjectSHA256: object.SHA256, Message: fmt.Sprintf("object reconstruction mismatch at %d of %d bytes", offset, object.RawBytes)}) + failed = true + } + if !failed { + result.VerifiedCount++ + } + } + result.IssueCount = len(result.Issues) + return result, nil +} + +func verifyGeneration(ctx context.Context, directory string, index Index) error { + resolver, err := openGeneration(directory, 0) + if err != nil { + return err + } + defer resolver.Close() + for _, object := range index.Objects { + hasher := sha256.New() + buffer := make([]byte, 128<<10) + var offset int64 + for offset < object.RawBytes { + n, readErr := resolver.ReadAt(ctx, fold.ObjectRef{SHA256: object.SHA256, RawBytes: object.RawBytes}, buffer, offset) + if n > 0 { + _, _ = hasher.Write(buffer[:n]) + offset += int64(n) + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + return readErr + } + if n == 0 { + break + } + } + if offset != object.RawBytes || hex.EncodeToString(hasher.Sum(nil)) != object.SHA256 { + return fmt.Errorf("packed object %s verification failed", object.SHA256) + } + } + return nil +} diff --git a/internal/pack/format.go b/internal/pack/format.go new file mode 100644 index 0000000..28b922a --- /dev/null +++ b/internal/pack/format.go @@ -0,0 +1,74 @@ +package pack + +import ( + "fmt" + "path/filepath" +) + +const ( + IndexVersion = 1 + IndexKind = "pack-v1" + defaultBlockBytes = int64(256 << 10) + defaultPackBytes = int64(512 << 20) + defaultCacheBytes = int64(128 << 20) +) + +type Index struct { + Version int `json:"version"` + Kind string `json:"kind"` + Generation string `json:"generation"` + CreatedAt string `json:"created_at"` + BlockBytes int64 `json:"block_bytes"` + Objects []Object `json:"objects"` +} + +type Object struct { + SHA256 string `json:"sha256"` + RawBytes int64 `json:"raw_bytes"` + Blocks []Block `json:"blocks"` +} + +type Block struct { + Pack string `json:"pack"` + PackOffset int64 `json:"pack_offset"` + StoredBytes int64 `json:"stored_bytes"` + RawOffset int64 `json:"raw_offset"` + RawBytes int64 `json:"raw_bytes"` + SHA256 string `json:"sha256"` +} + +func validateIndex(index Index) error { + if index.Version != IndexVersion || index.Kind != IndexKind { + return fmt.Errorf("unsupported pack index version=%d kind=%q", index.Version, index.Kind) + } + if !safeGeneration(index.Generation) || index.BlockBytes <= 0 { + return fmt.Errorf("invalid pack index generation or block size") + } + seen := make(map[string]struct{}, len(index.Objects)) + for objectIndex, object := range index.Objects { + if len(object.SHA256) != 64 || object.RawBytes < 0 { + return fmt.Errorf("invalid pack object %d", objectIndex) + } + if _, ok := seen[object.SHA256]; ok { + return fmt.Errorf("duplicate pack object %s", object.SHA256) + } + seen[object.SHA256] = struct{}{} + var expectedOffset int64 + for blockIndex, block := range object.Blocks { + if filepath.Base(block.Pack) != block.Pack || block.Pack == "." || block.Pack == ".." || block.PackOffset < 0 || block.StoredBytes <= 0 || block.StoredBytes > maxInt64() || block.RawBytes <= 0 || block.RawOffset != expectedOffset || len(block.SHA256) != 64 { + return fmt.Errorf("invalid block %d for object %s", blockIndex, object.SHA256) + } + expectedOffset += block.RawBytes + } + if expectedOffset != object.RawBytes { + return fmt.Errorf("object %s block bytes %d, want %d", object.SHA256, expectedOffset, object.RawBytes) + } + } + return nil +} + +func maxInt64() int64 { return int64(^uint(0) >> 1) } + +func safeGeneration(generation string) bool { + return generation != "" && generation != "." && generation != ".." && filepath.Base(generation) == generation +} diff --git a/internal/pack/pack_test.go b/internal/pack/pack_test.go new file mode 100644 index 0000000..e040688 --- /dev/null +++ b/internal/pack/pack_test.go @@ -0,0 +1,200 @@ +package pack + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/jstar0/codexfold/internal/fold" +) + +func TestBuildAndResolverReadExactRandomRanges(t *testing.T) { + root := t.TempDir() + large := bytes.Repeat([]byte("large-object-block-"), 50000) + refs := putObjects(t, root, []byte("shared-small-object"), large) + writeManifest(t, root, "first", []fold.ObjectRef{refs[0], refs[1], refs[0]}) + writeManifest(t, root, "fork", []fold.ObjectRef{refs[0], refs[1]}) + + result, err := Build(context.Background(), root, BuildOptions{BlockBytes: 256 << 10, PackBytes: 1 << 20}) + if err != nil { + t.Fatalf("Build returned error: %v", err) + } + if result.ObjectCount != 2 || result.BlockCount < 3 || result.PackCount < 1 { + t.Fatalf("unexpected build result: %#v", result) + } + loose := fold.NewObjectStore(root) + for _, ref := range refs { + if err := os.Remove(loose.ObjectPath(ref.SHA256)); err != nil { + t.Fatalf("remove loose object after pack build: %v", err) + } + } + + resolver, err := Open(root, OpenOptions{CacheBytes: 512 << 10}) + if err != nil { + t.Fatalf("Open returned error: %v", err) + } + t.Cleanup(func() { _ = resolver.Close() }) + + for _, test := range []struct { + name string + ref fold.ObjectRef + data []byte + off int64 + size int + }{ + {name: "small", ref: refs[0], data: []byte("shared-small-object"), off: 2, size: 8}, + {name: "large-first", ref: refs[1], data: large, off: 17, size: 333}, + {name: "large-block-boundary", ref: refs[1], data: large, off: (256 << 10) - 31, size: 1000}, + {name: "large-tail", ref: refs[1], data: large, off: int64(len(large) - 101), size: 200}, + } { + t.Run(test.name, func(t *testing.T) { + buffer := make([]byte, test.size) + n, readErr := resolver.ReadAt(context.Background(), test.ref, buffer, test.off) + end := int(test.off) + test.size + if end > len(test.data) { + end = len(test.data) + } + want := test.data[int(test.off):end] + if !bytes.Equal(buffer[:n], want) { + t.Fatalf("ReadAt bytes differ: got=%d want=%d", n, len(want)) + } + if len(want) < test.size && !errors.Is(readErr, io.EOF) { + t.Fatalf("ReadAt error = %v, want EOF", readErr) + } + }) + } +} + +func TestBuildInterruptionKeepsPreviousGenerationCurrent(t *testing.T) { + root := t.TempDir() + refs := putObjects(t, root, []byte("first-generation")) + writeManifest(t, root, "session", refs) + first, err := Build(context.Background(), root, BuildOptions{}) + if err != nil { + t.Fatalf("first Build returned error: %v", err) + } + + stop := errors.New("stop before publish") + if _, err := Build(context.Background(), root, BuildOptions{BeforePublish: func() error { return stop }}); !errors.Is(err, stop) { + t.Fatalf("interrupted Build error = %v, want %v", err, stop) + } + current, err := os.ReadFile(filepath.Join(root, "packs", "CURRENT")) + if err != nil { + t.Fatalf("read CURRENT: %v", err) + } + if string(bytes.TrimSpace(current)) != first.Generation { + t.Fatalf("CURRENT = %q, want %q", bytes.TrimSpace(current), first.Generation) + } + + resolver, err := Open(root, OpenOptions{}) + if err != nil { + t.Fatalf("Open previous generation: %v", err) + } + defer resolver.Close() + buffer := make([]byte, refs[0].RawBytes) + if _, err := resolver.ReadAt(context.Background(), refs[0], buffer, 0); err != nil && !errors.Is(err, io.EOF) { + t.Fatalf("read previous generation: %v", err) + } +} + +func TestResolverAndDoctorDetectPackCorruption(t *testing.T) { + root := t.TempDir() + refs := putObjects(t, root, bytes.Repeat([]byte("protected"), 10000)) + writeManifest(t, root, "session", refs) + if _, err := Build(context.Background(), root, BuildOptions{}); err != nil { + t.Fatalf("Build returned error: %v", err) + } + + index := loadCurrentIndex(t, root) + block := index.Objects[0].Blocks[0] + packPath := filepath.Join(root, "packs", index.Generation, block.Pack) + file, err := os.OpenFile(packPath, os.O_RDWR, 0) + if err != nil { + t.Fatalf("open pack: %v", err) + } + if _, err := file.WriteAt([]byte{0xff}, block.PackOffset); err != nil { + _ = file.Close() + t.Fatalf("corrupt pack: %v", err) + } + _ = file.Close() + + resolver, err := Open(root, OpenOptions{}) + if err != nil { + t.Fatalf("Open returned error: %v", err) + } + defer resolver.Close() + if _, err := resolver.ReadAt(context.Background(), refs[0], make([]byte, 16), 0); err == nil { + t.Fatal("ReadAt should reject a corrupt pack block") + } + report, err := Doctor(context.Background(), root) + if err != nil { + t.Fatalf("Doctor returned error: %v", err) + } + if report.IssueCount == 0 { + t.Fatalf("Doctor did not report corruption: %#v", report) + } +} + +func putObjects(t *testing.T, root string, values ...[]byte) []fold.ObjectRef { + t.Helper() + store := fold.NewObjectStore(root) + refs := make([]fold.ObjectRef, 0, len(values)) + for _, value := range values { + ref, _, err := store.Put(value, true) + if err != nil { + t.Fatalf("Put returned error: %v", err) + } + refs = append(refs, ref) + } + if err := store.SyncPending(context.Background()); err != nil { + t.Fatalf("SyncPending returned error: %v", err) + } + return refs +} + +func writeManifest(t *testing.T, root string, sessionID string, refs []fold.ObjectRef) { + t.Helper() + manifest := fold.Manifest{ + Version: fold.ManifestVersion, + Kind: fold.ManifestKind, + Session: fold.ManifestSession{ID: sessionID, RolloutPath: filepath.Join(root, sessionID+".jsonl")}, + Parts: make([]fold.Part, 0, len(refs)), + } + for _, ref := range refs { + manifest.Source.Bytes += ref.RawBytes + manifest.Parts = append(manifest.Parts, fold.Part{Kind: fold.PartResidual, Object: ref}) + } + manifest.Source.SHA256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + data, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("marshal manifest: %v", err) + } + if err := os.MkdirAll(filepath.Join(root, "manifests"), 0o755); err != nil { + t.Fatalf("create manifests: %v", err) + } + if err := os.WriteFile(fold.ManifestPath(root, sessionID), data, 0o644); err != nil { + t.Fatalf("write manifest: %v", err) + } +} + +func loadCurrentIndex(t *testing.T, root string) Index { + t.Helper() + current, err := os.ReadFile(filepath.Join(root, "packs", "CURRENT")) + if err != nil { + t.Fatalf("read CURRENT: %v", err) + } + data, err := os.ReadFile(filepath.Join(root, "packs", string(bytes.TrimSpace(current)), "index.json")) + if err != nil { + t.Fatalf("read index: %v", err) + } + var index Index + if err := json.Unmarshal(data, &index); err != nil { + t.Fatalf("decode index: %v", err) + } + return index +} diff --git a/internal/pack/replace_unix.go b/internal/pack/replace_unix.go new file mode 100644 index 0000000..f935f6b --- /dev/null +++ b/internal/pack/replace_unix.go @@ -0,0 +1,9 @@ +//go:build !windows + +package pack + +import "os" + +func replaceFile(source string, target string) error { + return os.Rename(source, target) +} diff --git a/internal/pack/replace_windows.go b/internal/pack/replace_windows.go new file mode 100644 index 0000000..845910c --- /dev/null +++ b/internal/pack/replace_windows.go @@ -0,0 +1,36 @@ +//go:build windows + +package pack + +import ( + "fmt" + "syscall" + "unsafe" +) + +const ( + moveFileReplaceExisting = 0x1 + moveFileWriteThrough = 0x8 +) + +var moveFileExW = syscall.NewLazyDLL("kernel32.dll").NewProc("MoveFileExW") + +func replaceFile(source string, target string) error { + sourcePointer, err := syscall.UTF16PtrFromString(source) + if err != nil { + return err + } + targetPointer, err := syscall.UTF16PtrFromString(target) + if err != nil { + return err + } + result, _, callErr := moveFileExW.Call( + uintptr(unsafe.Pointer(sourcePointer)), + uintptr(unsafe.Pointer(targetPointer)), + moveFileReplaceExisting|moveFileWriteThrough, + ) + if result == 0 { + return fmt.Errorf("replace file: %w", callErr) + } + return nil +} diff --git a/internal/pack/resolver.go b/internal/pack/resolver.go new file mode 100644 index 0000000..f361b8a --- /dev/null +++ b/internal/pack/resolver.go @@ -0,0 +1,189 @@ +package pack + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/jstar0/codexfold/internal/fold" + "github.com/klauspost/compress/zstd" +) + +type OpenOptions struct { + CacheBytes int64 +} + +type Resolver struct { + directory string + index Index + objects map[string]Object + packs map[string]*os.File + cache *blockCache + closeOnce sync.Once +} + +func Open(storeDir string, options OpenOptions) (*Resolver, error) { + current, err := os.ReadFile(filepath.Join(storeDir, "packs", "CURRENT")) + if err != nil { + return nil, fmt.Errorf("read pack CURRENT: %w", err) + } + generation := strings.TrimSpace(string(current)) + if !safeGeneration(generation) { + return nil, fmt.Errorf("unsafe pack generation %q", generation) + } + if options.CacheBytes == 0 { + options.CacheBytes = defaultCacheBytes + } + return openGeneration(filepath.Join(storeDir, "packs", generation), options.CacheBytes) +} + +func openGeneration(directory string, cacheBytes int64) (*Resolver, error) { + data, err := os.ReadFile(filepath.Join(directory, "index.json")) + if err != nil { + return nil, fmt.Errorf("read pack index: %w", err) + } + var index Index + if err := json.Unmarshal(data, &index); err != nil { + return nil, fmt.Errorf("decode pack index: %w", err) + } + directoryName := filepath.Base(directory) + if directoryName != index.Generation && !strings.HasPrefix(directoryName, ".generation-") { + return nil, fmt.Errorf("pack index generation %q does not match directory %q", index.Generation, filepath.Base(directory)) + } + if err := validateIndex(index); err != nil { + return nil, err + } + resolver := &Resolver{directory: directory, index: index, objects: make(map[string]Object, len(index.Objects)), packs: make(map[string]*os.File), cache: newBlockCache(cacheBytes)} + for _, object := range index.Objects { + resolver.objects[object.SHA256] = object + for _, block := range object.Blocks { + if _, ok := resolver.packs[block.Pack]; ok { + continue + } + file, err := os.Open(filepath.Join(directory, block.Pack)) + if err != nil { + _ = resolver.Close() + return nil, fmt.Errorf("open pack %s: %w", block.Pack, err) + } + resolver.packs[block.Pack] = file + } + } + for _, object := range index.Objects { + for blockIndex, block := range object.Blocks { + info, err := resolver.packs[block.Pack].Stat() + if err != nil { + _ = resolver.Close() + return nil, fmt.Errorf("stat pack %s: %w", block.Pack, err) + } + if block.PackOffset > info.Size() || block.StoredBytes > info.Size()-block.PackOffset { + _ = resolver.Close() + return nil, fmt.Errorf("packed block %s:%d exceeds %s size", object.SHA256, blockIndex, block.Pack) + } + } + } + return resolver, nil +} + +func (r *Resolver) ReadAt(ctx context.Context, ref fold.ObjectRef, destination []byte, offset int64) (int, error) { + if offset < 0 { + return 0, errors.New("negative object read offset") + } + if len(destination) == 0 { + return 0, nil + } + object, ok := r.objects[ref.SHA256] + if !ok { + return 0, fmt.Errorf("object %s is not packed", ref.SHA256) + } + if object.RawBytes != ref.RawBytes { + return 0, fmt.Errorf("object %s raw size %d, want %d", ref.SHA256, object.RawBytes, ref.RawBytes) + } + if offset >= object.RawBytes { + return 0, io.EOF + } + written := 0 + for written < len(destination) && offset < object.RawBytes { + if err := ctx.Err(); err != nil { + return written, err + } + blockIndex := sort.Search(len(object.Blocks), func(index int) bool { + block := object.Blocks[index] + return block.RawOffset+block.RawBytes > offset + }) + if blockIndex == len(object.Blocks) { + return written, fmt.Errorf("object %s has no block for offset %d", object.SHA256, offset) + } + block := object.Blocks[blockIndex] + data, err := r.readBlock(object.SHA256, blockIndex, block) + if err != nil { + return written, err + } + inside := offset - block.RawOffset + copied := copy(destination[written:], data[inside:]) + written += copied + offset += int64(copied) + } + if written < len(destination) { + return written, io.EOF + } + return written, nil +} + +func (r *Resolver) readBlock(objectDigest string, blockIndex int, block Block) ([]byte, error) { + key := fmt.Sprintf("%s:%d", objectDigest, blockIndex) + if data, ok := r.cache.get(key); ok { + return data, nil + } + file := r.packs[block.Pack] + if file == nil { + return nil, fmt.Errorf("pack %s is not open", block.Pack) + } + compressed := make([]byte, int(block.StoredBytes)) + if _, err := file.ReadAt(compressed, block.PackOffset); err != nil { + return nil, fmt.Errorf("read packed block %s:%d: %w", objectDigest, blockIndex, err) + } + decoder, err := zstd.NewReader(nil) + if err != nil { + return nil, fmt.Errorf("create pack decoder: %w", err) + } + data, err := decoder.DecodeAll(compressed, nil) + decoder.Close() + if err != nil { + return nil, fmt.Errorf("decode packed block %s:%d: %w", objectDigest, blockIndex, err) + } + if int64(len(data)) != block.RawBytes { + return nil, fmt.Errorf("packed block %s:%d raw size %d, want %d", objectDigest, blockIndex, len(data), block.RawBytes) + } + digest := sha256.Sum256(data) + if hex.EncodeToString(digest[:]) != block.SHA256 { + return nil, fmt.Errorf("packed block %s:%d SHA-256 mismatch", objectDigest, blockIndex) + } + r.cache.put(key, data) + return data, nil +} + +func (r *Resolver) Close() error { + var closeErr error + r.closeOnce.Do(func() { + names := make([]string, 0, len(r.packs)) + for name := range r.packs { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if err := r.packs[name].Close(); err != nil && closeErr == nil { + closeErr = err + } + } + }) + return closeErr +} From 039e6b94f53898dee0ef8461129b52a2a9f07508 Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 00:35:21 +0800 Subject: [PATCH 03/33] feat: add exact virtual rollout byte view --- internal/vfs/resolver.go | 11 ++++ internal/vfs/view.go | 102 +++++++++++++++++++++++++++++ internal/vfs/view_test.go | 134 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 247 insertions(+) create mode 100644 internal/vfs/resolver.go create mode 100644 internal/vfs/view.go create mode 100644 internal/vfs/view_test.go diff --git a/internal/vfs/resolver.go b/internal/vfs/resolver.go new file mode 100644 index 0000000..9319abc --- /dev/null +++ b/internal/vfs/resolver.go @@ -0,0 +1,11 @@ +package vfs + +import ( + "context" + + "github.com/jstar0/codexfold/internal/fold" +) + +type ObjectReader interface { + ReadAt(context.Context, fold.ObjectRef, []byte, int64) (int, error) +} diff --git a/internal/vfs/view.go b/internal/vfs/view.go new file mode 100644 index 0000000..1fdb5dd --- /dev/null +++ b/internal/vfs/view.go @@ -0,0 +1,102 @@ +package vfs + +import ( + "context" + "errors" + "fmt" + "io" + "sort" + + "github.com/jstar0/codexfold/internal/fold" +) + +type View struct { + manifest fold.Manifest + ends []int64 + reader ObjectReader +} + +func NewView(manifest fold.Manifest, reader ObjectReader) (*View, error) { + if reader == nil { + return nil, errors.New("virtual view object reader is required") + } + if manifest.Version != fold.ManifestVersion || manifest.Kind != fold.ManifestKind { + return nil, fmt.Errorf("unsupported fold manifest version=%d kind=%q", manifest.Version, manifest.Kind) + } + view := &View{manifest: manifest, reader: reader, ends: make([]int64, len(manifest.Parts))} + var total int64 + for index, part := range manifest.Parts { + if part.Kind != fold.PartResidual && part.Kind != fold.PartField { + return nil, fmt.Errorf("manifest part %d has unsupported kind %q", index, part.Kind) + } + if len(part.Object.SHA256) != 64 || part.Object.RawBytes <= 0 { + return nil, fmt.Errorf("manifest part %d has invalid object reference", index) + } + if part.Object.RawBytes > int64(^uint64(0)>>1)-total { + return nil, errors.New("manifest byte length overflows int64") + } + total += part.Object.RawBytes + view.ends[index] = total + } + if total != manifest.Source.Bytes { + return nil, fmt.Errorf("manifest parts total %d bytes, source records %d", total, manifest.Source.Bytes) + } + return view, nil +} + +func (v *View) Size() int64 { return v.manifest.Source.Bytes } + +func (v *View) ReadAt(ctx context.Context, destination []byte, offset int64) (int, error) { + if offset < 0 { + return 0, errors.New("negative virtual read offset") + } + if len(destination) == 0 { + return 0, nil + } + if err := ctx.Err(); err != nil { + return 0, err + } + if offset >= v.Size() { + return 0, io.EOF + } + written := 0 + for written < len(destination) && offset < v.Size() { + if err := ctx.Err(); err != nil { + return written, err + } + partIndex := sort.Search(len(v.ends), func(index int) bool { return v.ends[index] > offset }) + if partIndex == len(v.ends) { + return written, fmt.Errorf("manifest has no part for offset %d", offset) + } + partStart := int64(0) + if partIndex > 0 { + partStart = v.ends[partIndex-1] + } + part := v.manifest.Parts[partIndex] + inside := offset - partStart + remaining := part.Object.RawBytes - inside + need := len(destination) - written + if int64(need) > remaining { + need = int(remaining) + } + n, err := v.reader.ReadAt(ctx, part.Object, destination[written:written+need], inside) + if n < 0 || n > need { + return written, fmt.Errorf("object reader returned invalid byte count %d for request %d", n, need) + } + written += n + offset += int64(n) + if n != need { + if err == nil { + err = io.ErrUnexpectedEOF + } + return written, fmt.Errorf("read manifest part %d: %w", partIndex, err) + } + if err != nil && !errors.Is(err, io.EOF) { + return written, fmt.Errorf("read manifest part %d: %w", partIndex, err) + } + } + if written < len(destination) { + return written, io.EOF + } + return written, nil +} diff --git a/internal/vfs/view_test.go b/internal/vfs/view_test.go new file mode 100644 index 0000000..6e60089 --- /dev/null +++ b/internal/vfs/view_test.go @@ -0,0 +1,134 @@ +package vfs + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "math/rand" + "testing" + + "github.com/jstar0/codexfold/internal/fold" +) + +func TestViewReadsExactBytesAcrossPartBoundaries(t *testing.T) { + view, source := fixtureView(t) + buffer := make([]byte, 19) + n, err := view.ReadAt(context.Background(), buffer, 4) + if err != nil { + t.Fatalf("ReadAt returned error: %v", err) + } + if !bytes.Equal(buffer[:n], source[4:23]) { + t.Fatalf("cross-part bytes differ: got=%q want=%q", buffer[:n], source[4:23]) + } + if view.Size() != int64(len(source)) { + t.Fatalf("Size = %d, want %d", view.Size(), len(source)) + } +} + +func TestViewMatchesNativeBytesForTenThousandRandomReads(t *testing.T) { + view, source := fixtureView(t) + random := rand.New(rand.NewSource(42)) + for iteration := 0; iteration < 10000; iteration++ { + offset := random.Intn(len(source) + 5) + length := random.Intn(80) + buffer := make([]byte, length) + n, err := view.ReadAt(context.Background(), buffer, int64(offset)) + if length == 0 { + if n != 0 || err != nil { + t.Fatalf("iteration %d zero read = (%d, %v)", iteration, n, err) + } + continue + } + if offset >= len(source) { + if n != 0 || !errors.Is(err, io.EOF) { + t.Fatalf("iteration %d past EOF = (%d, %v)", iteration, n, err) + } + continue + } + end := offset + length + if end > len(source) { + end = len(source) + } + if !bytes.Equal(buffer[:n], source[offset:end]) { + t.Fatalf("iteration %d bytes differ offset=%d length=%d", iteration, offset, length) + } + if end < offset+length { + if !errors.Is(err, io.EOF) { + t.Fatalf("iteration %d error = %v, want EOF", iteration, err) + } + } else if err != nil { + t.Fatalf("iteration %d unexpected error: %v", iteration, err) + } + } +} + +func TestViewRejectsInconsistentManifestLength(t *testing.T) { + manifest := fold.Manifest{ + Version: fold.ManifestVersion, + Kind: fold.ManifestKind, + Source: fold.ManifestSource{Bytes: 5, SHA256: string(make([]byte, 64))}, + Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{ + SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + RawBytes: 4, + }}}, + } + if _, err := NewView(manifest, memoryReader{}); err == nil { + t.Fatal("NewView should reject inconsistent manifest bytes") + } +} + +func TestViewPropagatesCancellation(t *testing.T) { + view, _ := fixtureView(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := view.ReadAt(ctx, make([]byte, 1), 0); !errors.Is(err, context.Canceled) { + t.Fatalf("ReadAt error = %v, want context.Canceled", err) + } +} + +type memoryReader map[string][]byte + +func (r memoryReader) ReadAt(_ context.Context, ref fold.ObjectRef, destination []byte, offset int64) (int, error) { + data, ok := r[ref.SHA256] + if !ok { + return 0, errors.New("missing object") + } + if offset >= int64(len(data)) { + return 0, io.EOF + } + n := copy(destination, data[offset:]) + if n < len(destination) { + return n, io.EOF + } + return n, nil +} + +func fixtureView(t *testing.T) (*View, []byte) { + t.Helper() + parts := [][]byte{ + []byte("alpha-"), + bytes.Repeat([]byte("B"), 33), + []byte("-gamma-"), + bytes.Repeat([]byte("delta"), 17), + } + reader := memoryReader{} + manifest := fold.Manifest{Version: fold.ManifestVersion, Kind: fold.ManifestKind} + var source []byte + for _, data := range parts { + digest := sha256.Sum256(data) + hexDigest := hex.EncodeToString(digest[:]) + reader[hexDigest] = data + manifest.Parts = append(manifest.Parts, fold.Part{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: hexDigest, RawBytes: int64(len(data))}}) + source = append(source, data...) + } + sourceDigest := sha256.Sum256(source) + manifest.Source = fold.ManifestSource{Bytes: int64(len(source)), SHA256: hex.EncodeToString(sourceDigest[:])} + view, err := NewView(manifest, reader) + if err != nil { + t.Fatalf("NewView returned error: %v", err) + } + return view, source +} From 9a7f1e8bc0e9cb46e2c2c51e6d16a557063c4bc4 Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 00:42:40 +0800 Subject: [PATCH 04/33] feat: add durable append and copy-on-write sessions --- internal/vfs/handles.go | 219 +++++++++++++++ internal/vfs/session.go | 375 ++++++++++++++++++++++++++ internal/vfs/session_test.go | 231 ++++++++++++++++ internal/vfs/state.go | 114 ++++++++ internal/vfs/state_replace_unix.go | 22 ++ internal/vfs/state_replace_windows.go | 29 ++ 6 files changed, 990 insertions(+) create mode 100644 internal/vfs/handles.go create mode 100644 internal/vfs/session.go create mode 100644 internal/vfs/session_test.go create mode 100644 internal/vfs/state.go create mode 100644 internal/vfs/state_replace_unix.go create mode 100644 internal/vfs/state_replace_windows.go diff --git a/internal/vfs/handles.go b/internal/vfs/handles.go new file mode 100644 index 0000000..4e7ee7b --- /dev/null +++ b/internal/vfs/handles.go @@ -0,0 +1,219 @@ +package vfs + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "sync" +) + +type ReadHandle struct { + session *Session + generation uint64 + base *View + baseBytes int64 + file *os.File + backing bool + deltaBytes int64 + size int64 + closeOnce sync.Once + closeErr error +} + +func (h *ReadHandle) Size() int64 { return h.size } + +func (h *ReadHandle) ReadAt(ctx context.Context, destination []byte, offset int64) (int, error) { + if offset < 0 { + return 0, errors.New("negative session read offset") + } + if len(destination) == 0 { + return 0, nil + } + if err := ctx.Err(); err != nil { + return 0, err + } + if offset >= h.size { + return 0, io.EOF + } + if h.backing { + limit := len(destination) + if remaining := h.size - offset; int64(limit) > remaining { + limit = int(remaining) + } + n, err := h.file.ReadAt(destination[:limit], offset) + if err != nil && !errors.Is(err, io.EOF) { + return n, err + } + if n < len(destination) { + return n, io.EOF + } + return n, nil + } + written := 0 + if offset < h.baseBytes { + need := len(destination) + if remaining := h.baseBytes - offset; int64(need) > remaining { + need = int(remaining) + } + n, err := h.base.ReadAt(ctx, destination[:need], offset) + written += n + offset += int64(n) + if n != need { + if err == nil { + err = io.ErrUnexpectedEOF + } + return written, err + } + if err != nil && !errors.Is(err, io.EOF) { + return written, err + } + } + if written < len(destination) && offset >= h.baseBytes && offset < h.size { + deltaOffset := offset - h.baseBytes + need := len(destination) - written + if remaining := h.deltaBytes - deltaOffset; int64(need) > remaining { + need = int(remaining) + } + n, err := h.file.ReadAt(destination[written:written+need], deltaOffset) + written += n + if n != need { + if err == nil { + err = io.ErrUnexpectedEOF + } + return written, err + } + if err != nil && !errors.Is(err, io.EOF) { + return written, err + } + } + if written < len(destination) { + return written, io.EOF + } + return written, nil +} + +func (h *ReadHandle) Close() error { + h.closeOnce.Do(func() { + h.closeErr = h.file.Close() + h.session.releaseReader(h.generation) + }) + return h.closeErr +} + +type WriteHandle struct { + session *Session + leasePath string + mu sync.Mutex + closed bool +} + +func (h *WriteHandle) Append(ctx context.Context, data []byte) (int, error) { + h.mu.Lock() + defer h.mu.Unlock() + if h.closed { + return 0, errors.New("writer is closed") + } + if err := ctx.Err(); err != nil { + return 0, err + } + state := h.session.State() + path := state.DeltaPath + if state.BackingPath != "" { + path = state.BackingPath + } + file, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + return 0, fmt.Errorf("open append target: %w", err) + } + n, writeErr := file.Write(data) + closeErr := file.Close() + if writeErr != nil { + return n, writeErr + } + if closeErr != nil { + return n, closeErr + } + return n, nil +} + +func (h *WriteHandle) WriteAt(ctx context.Context, data []byte, offset int64) (int, error) { + h.mu.Lock() + defer h.mu.Unlock() + if h.closed { + return 0, errors.New("writer is closed") + } + if offset < 0 { + return 0, errors.New("negative write offset") + } + path, err := h.session.ensureBacking(ctx) + if err != nil { + return 0, err + } + file, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return 0, err + } + n, writeErr := file.WriteAt(data, offset) + closeErr := file.Close() + if writeErr != nil { + return n, writeErr + } + return n, closeErr +} + +func (h *WriteHandle) Truncate(ctx context.Context, size int64) error { + h.mu.Lock() + defer h.mu.Unlock() + if h.closed { + return errors.New("writer is closed") + } + if size < 0 { + return errors.New("negative truncate size") + } + path, err := h.session.ensureBacking(ctx) + if err != nil { + return err + } + return os.Truncate(path, size) +} + +func (h *WriteHandle) Sync() error { + h.mu.Lock() + defer h.mu.Unlock() + if h.closed { + return errors.New("writer is closed") + } + state := h.session.State() + path := state.DeltaPath + if state.BackingPath != "" { + path = state.BackingPath + } + file, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return err + } + if err := file.Sync(); err != nil { + _ = file.Close() + return err + } + return file.Close() +} + +func (h *WriteHandle) Close() error { + h.mu.Lock() + defer h.mu.Unlock() + if h.closed { + return nil + } + h.closed = true + removeErr := os.Remove(h.leasePath) + h.session.mu.Lock() + h.session.writerOpen = false + h.session.mu.Unlock() + if removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + return removeErr + } + return nil +} diff --git a/internal/vfs/session.go b/internal/vfs/session.go new file mode 100644 index 0000000..3060f6e --- /dev/null +++ b/internal/vfs/session.go @@ -0,0 +1,375 @@ +package vfs + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + + "github.com/jstar0/codexfold/internal/fold" +) + +type SessionOptions struct { + Root string + ManifestPath string + Manifest fold.Manifest + Reader ObjectReader + NativeSnapshot NativeFile + BeforeCOWPhase func(string) error +} + +type Session struct { + mu sync.Mutex + state SessionState + statePath string + directory string + view *View + readerLeases map[uint64]int + writerOpen bool + beforeCOWPhase func(string) error +} + +func OpenSession(ctx context.Context, options SessionOptions) (*Session, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if options.Root == "" || options.ManifestPath == "" || !safeSessionID(options.Manifest.Session.ID) { + return nil, errors.New("session root, manifest path, and safe session ID are required") + } + view, err := NewView(options.Manifest, options.Reader) + if err != nil { + return nil, err + } + directory := filepath.Join(options.Root, "fs", "sessions", options.Manifest.Session.ID) + if err := os.MkdirAll(directory, 0o700); err != nil { + return nil, fmt.Errorf("create virtual session directory: %w", err) + } + statePath := filepath.Join(directory, "state.json") + state, err := loadSessionState(statePath) + if errors.Is(err, os.ErrNotExist) { + if err := verifyNativeFile(options.NativeSnapshot); err != nil { + return nil, err + } + deltaPath := filepath.Join(directory, "delta.jsonl") + delta, err := os.OpenFile(deltaPath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("create session delta: %w", err) + } + if err := delta.Sync(); err != nil { + _ = delta.Close() + return nil, fmt.Errorf("sync session delta: %w", err) + } + if err := delta.Close(); err != nil { + return nil, fmt.Errorf("close session delta: %w", err) + } + state = SessionState{Version: sessionStateVersion, SessionID: options.Manifest.Session.ID, Generation: 1, ManifestPath: filepath.Clean(options.ManifestPath), BaseBytes: view.Size(), BaseSHA256: options.Manifest.Source.SHA256, DeltaPath: deltaPath, NativeSnapshot: options.NativeSnapshot} + if err := writeSessionState(statePath, state); err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } else { + if state.SessionID != options.Manifest.Session.ID || state.ManifestPath != filepath.Clean(options.ManifestPath) || state.BaseBytes != view.Size() || state.BaseSHA256 != options.Manifest.Source.SHA256 || state.NativeSnapshot != options.NativeSnapshot { + return nil, errors.New("persisted session state does not match the requested manifest") + } + if !pathWithin(directory, state.DeltaPath) || (state.BackingPath != "" && !pathWithin(directory, state.BackingPath)) { + return nil, errors.New("persisted session state contains an unsafe data path") + } + if _, err := os.Stat(state.DeltaPath); err != nil { + return nil, fmt.Errorf("stat session delta: %w", err) + } + if state.BackingPath != "" { + if _, err := os.Stat(state.BackingPath); err != nil { + return nil, fmt.Errorf("stat session backing: %w", err) + } + } + } + return &Session{state: state, statePath: statePath, directory: directory, view: view, readerLeases: make(map[uint64]int), beforeCOWPhase: options.BeforeCOWPhase}, nil +} + +func (s *Session) State() SessionState { + s.mu.Lock() + defer s.mu.Unlock() + return s.state +} + +func (s *Session) OpenReader() (*ReadHandle, error) { + s.mu.Lock() + state := s.state + s.readerLeases[state.Generation]++ + s.mu.Unlock() + + handle := &ReadHandle{session: s, generation: state.Generation, base: s.view, baseBytes: state.BaseBytes} + var path string + if state.BackingPath != "" { + path = state.BackingPath + handle.backing = true + } else { + path = state.DeltaPath + } + file, err := os.Open(path) + if err != nil { + s.releaseReader(state.Generation) + return nil, fmt.Errorf("open session reader file: %w", err) + } + info, err := file.Stat() + if err != nil { + _ = file.Close() + s.releaseReader(state.Generation) + return nil, fmt.Errorf("stat session reader file: %w", err) + } + handle.file = file + if handle.backing { + handle.size = info.Size() + } else { + handle.deltaBytes = info.Size() + handle.size = state.BaseBytes + info.Size() + } + return handle, nil +} + +func (s *Session) releaseReader(generation uint64) { + s.mu.Lock() + defer s.mu.Unlock() + if s.readerLeases[generation] <= 1 { + delete(s.readerLeases, generation) + } else { + s.readerLeases[generation]-- + } +} + +func (s *Session) OpenWriter() (*WriteHandle, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.writerOpen { + return nil, errors.New("session writer lease is already held") + } + leasePath := filepath.Join(s.directory, "writer.lease") + lease, err := os.OpenFile(leasePath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + if errors.Is(err, os.ErrExist) { + return nil, errors.New("session writer lease file already exists") + } + return nil, fmt.Errorf("create writer lease: %w", err) + } + if _, err := fmt.Fprintf(lease, "%d\n", os.Getpid()); err != nil { + _ = lease.Close() + _ = os.Remove(leasePath) + return nil, fmt.Errorf("write writer lease: %w", err) + } + if err := lease.Sync(); err != nil { + _ = lease.Close() + _ = os.Remove(leasePath) + return nil, fmt.Errorf("sync writer lease: %w", err) + } + if err := lease.Close(); err != nil { + _ = os.Remove(leasePath) + return nil, fmt.Errorf("close writer lease: %w", err) + } + s.writerOpen = true + return &WriteHandle{session: s, leasePath: leasePath}, nil +} + +func (s *Session) ensureBacking(ctx context.Context) (string, error) { + s.mu.Lock() + if s.state.BackingPath != "" { + path := s.state.BackingPath + s.mu.Unlock() + return path, nil + } + currentGeneration := s.state.Generation + s.mu.Unlock() + + reader, err := s.OpenReader() + if err != nil { + return "", err + } + defer reader.Close() + temporary, err := os.CreateTemp(s.directory, ".backing-*.tmp") + if err != nil { + return "", fmt.Errorf("create temporary backing: %w", err) + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return "", err + } + sourceHash := sha256.New() + buffer := make([]byte, 1<<20) + var offset int64 + for offset < reader.Size() { + if err := ctx.Err(); err != nil { + _ = temporary.Close() + return "", err + } + need := len(buffer) + if remaining := reader.Size() - offset; int64(need) > remaining { + need = int(remaining) + } + n, readErr := reader.ReadAt(ctx, buffer[:need], offset) + if n > 0 { + _, _ = sourceHash.Write(buffer[:n]) + if _, err := temporary.Write(buffer[:n]); err != nil { + _ = temporary.Close() + return "", fmt.Errorf("write temporary backing: %w", err) + } + offset += int64(n) + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + _ = temporary.Close() + return "", readErr + } + if n == 0 { + break + } + } + if offset != reader.Size() { + _ = temporary.Close() + return "", errors.New("temporary backing source read ended early") + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return "", fmt.Errorf("sync temporary backing: %w", err) + } + if err := temporary.Close(); err != nil { + return "", fmt.Errorf("close temporary backing: %w", err) + } + verified, err := hashNativePath(temporaryPath) + if err != nil { + return "", err + } + if verified.Bytes != reader.Size() || verified.SHA256 != hex.EncodeToString(sourceHash.Sum(nil)) { + return "", errors.New("temporary backing verification failed") + } + if s.beforeCOWPhase != nil { + if err := s.beforeCOWPhase("before-publish"); err != nil { + return "", err + } + } + backingPath := filepath.Join(s.directory, fmt.Sprintf("backing-%020d.jsonl", currentGeneration+1)) + if err := replaceStateFile(temporaryPath, backingPath); err != nil { + return "", fmt.Errorf("publish session backing: %w", err) + } + if err := syncStateDirectory(s.directory); err != nil { + return "", err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.state.Generation != currentGeneration || s.state.BackingPath != "" { + return "", errors.New("session generation changed during copy-on-write") + } + next := s.state + next.Generation++ + next.BackingPath = backingPath + if err := writeSessionState(s.statePath, next); err != nil { + return "", err + } + s.state = next + return backingPath, nil +} + +func (s *Session) MaterializeCurrent(ctx context.Context, target string, overwrite bool) (NativeFile, error) { + if target == "" { + return NativeFile{}, errors.New("materialize target is required") + } + if !overwrite { + if _, err := os.Stat(target); err == nil { + return NativeFile{}, fmt.Errorf("materialize target already exists: %s", target) + } else if !errors.Is(err, os.ErrNotExist) { + return NativeFile{}, err + } + } + reader, err := s.OpenReader() + if err != nil { + return NativeFile{}, err + } + defer reader.Close() + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return NativeFile{}, err + } + temporary, err := os.CreateTemp(filepath.Dir(target), ".materialize-*.tmp") + if err != nil { + return NativeFile{}, err + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return NativeFile{}, err + } + hasher := sha256.New() + buffer := make([]byte, 1<<20) + var offset int64 + for offset < reader.Size() { + if err := ctx.Err(); err != nil { + _ = temporary.Close() + return NativeFile{}, err + } + need := len(buffer) + if remaining := reader.Size() - offset; int64(need) > remaining { + need = int(remaining) + } + n, readErr := reader.ReadAt(ctx, buffer[:need], offset) + if n > 0 { + _, _ = hasher.Write(buffer[:n]) + if _, err := temporary.Write(buffer[:n]); err != nil { + _ = temporary.Close() + return NativeFile{}, err + } + offset += int64(n) + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + _ = temporary.Close() + return NativeFile{}, readErr + } + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return NativeFile{}, err + } + if err := temporary.Close(); err != nil { + return NativeFile{}, err + } + if overwrite { + if err := replaceStateFile(temporaryPath, target); err != nil { + return NativeFile{}, err + } + } else if err := os.Rename(temporaryPath, target); err != nil { + return NativeFile{}, err + } + if err := syncStateDirectory(filepath.Dir(target)); err != nil { + return NativeFile{}, err + } + expected := NativeFile{Path: target, Bytes: offset, SHA256: hex.EncodeToString(hasher.Sum(nil))} + verified, err := hashNativePath(target) + if err != nil { + return NativeFile{}, err + } + if verified.Bytes != expected.Bytes || verified.SHA256 != expected.SHA256 { + return NativeFile{}, errors.New("materialized current session verification failed") + } + return expected, nil +} + +func hashNativePath(path string) (NativeFile, error) { + file, err := os.Open(path) + if err != nil { + return NativeFile{}, err + } + hasher := sha256.New() + bytesRead, copyErr := io.Copy(hasher, file) + closeErr := file.Close() + if copyErr != nil { + return NativeFile{}, copyErr + } + if closeErr != nil { + return NativeFile{}, closeErr + } + return NativeFile{Path: path, Bytes: bytesRead, SHA256: hex.EncodeToString(hasher.Sum(nil))}, nil +} diff --git a/internal/vfs/session_test.go b/internal/vfs/session_test.go new file mode 100644 index 0000000..d769379 --- /dev/null +++ b/internal/vfs/session_test.go @@ -0,0 +1,231 @@ +package vfs + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/jstar0/codexfold/internal/fold" +) + +func TestSessionAppendPersistsWithoutHydratingBase(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + + oldReader, err := session.OpenReader() + if err != nil { + t.Fatalf("OpenReader before append: %v", err) + } + defer oldReader.Close() + + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + if _, err := writer.Append(context.Background(), []byte("-durable-tail")); err != nil { + t.Fatalf("Append: %v", err) + } + if err := writer.Sync(); err != nil { + t.Fatalf("Sync: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("Close writer: %v", err) + } + + if got := readHandle(t, oldReader); !bytes.Equal(got, source) { + t.Fatalf("old generation changed after append: %q", got) + } + newReader, err := session.OpenReader() + if err != nil { + t.Fatalf("OpenReader after append: %v", err) + } + want := append(append([]byte(nil), source...), []byte("-durable-tail")...) + if got := readHandle(t, newReader); !bytes.Equal(got, want) { + t.Fatalf("new generation bytes differ: got=%q want=%q", got, want) + } + _ = newReader.Close() + + state := session.State() + if state.BackingPath != "" { + t.Fatalf("append hydrated a backing file: %#v", state) + } + if info, err := os.Stat(state.DeltaPath); err != nil || info.Size() != int64(len("-durable-tail")) { + t.Fatalf("delta state differs: info=%v err=%v", info, err) + } + + reopened := openFixtureSession(t, root, manifest, reader, nil) + reopenedReader, err := reopened.OpenReader() + if err != nil { + t.Fatalf("reopened OpenReader: %v", err) + } + defer reopenedReader.Close() + if got := readHandle(t, reopenedReader); !bytes.Equal(got, want) { + t.Fatalf("reopened bytes differ: got=%q want=%q", got, want) + } +} + +func TestSessionAllowsOnlyOneWriterLease(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + first, err := session.OpenWriter() + if err != nil { + t.Fatalf("first OpenWriter: %v", err) + } + if _, err := session.OpenWriter(); err == nil { + t.Fatal("second OpenWriter should fail while the lease is held") + } + if err := first.Close(); err != nil { + t.Fatalf("close first writer: %v", err) + } + second, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter after release: %v", err) + } + _ = second.Close() +} + +func TestSessionRandomWriteTransitionsToVerifiedBacking(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + if _, err := writer.Append(context.Background(), []byte("-tail")); err != nil { + t.Fatalf("Append: %v", err) + } + if _, err := writer.WriteAt(context.Background(), []byte("PATCH"), 3); err != nil { + t.Fatalf("WriteAt: %v", err) + } + if err := writer.Sync(); err != nil { + t.Fatalf("Sync: %v", err) + } + _ = writer.Close() + + want := append(append([]byte(nil), source...), []byte("-tail")...) + copy(want[3:], []byte("PATCH")) + current, err := session.MaterializeCurrent(context.Background(), filepath.Join(root, "current.jsonl"), false) + if err != nil { + t.Fatalf("MaterializeCurrent: %v", err) + } + if current.SHA256 != digestBytes(want) || current.Bytes != int64(len(want)) { + t.Fatalf("materialized metadata differs: %#v", current) + } + got, err := os.ReadFile(current.Path) + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("materialized bytes differ: bytes=%q err=%v", got, err) + } + if session.State().BackingPath == "" { + t.Fatal("random write did not activate a backing file") + } +} + +func TestSessionTruncateTransitionsToBacking(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + newSize := int64(len(source) - 4) + if err := writer.Truncate(context.Background(), newSize); err != nil { + t.Fatalf("Truncate: %v", err) + } + _ = writer.Close() + readerHandle, err := session.OpenReader() + if err != nil { + t.Fatalf("OpenReader: %v", err) + } + defer readerHandle.Close() + if got := readHandle(t, readerHandle); !bytes.Equal(got, source[:newSize]) { + t.Fatalf("truncated bytes differ: got=%q want=%q", got, source[:newSize]) + } +} + +func TestSessionInterruptedCopyOnWriteKeepsPreviousGeneration(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + stop := errors.New("stop before COW publish") + session := openFixtureSession(t, root, manifest, reader, func(phase string) error { + if phase == "before-publish" { + return stop + } + return nil + }) + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + if _, err := writer.WriteAt(context.Background(), []byte("X"), 0); !errors.Is(err, stop) { + t.Fatalf("WriteAt error = %v, want %v", err, stop) + } + _ = writer.Close() + if session.State().BackingPath != "" { + t.Fatalf("interrupted COW published backing: %#v", session.State()) + } + handle, err := session.OpenReader() + if err != nil { + t.Fatalf("OpenReader: %v", err) + } + defer handle.Close() + if got := readHandle(t, handle); !bytes.Equal(got, source) { + t.Fatalf("previous generation changed after interrupted COW: %q", got) + } +} + +func sessionFixture(t *testing.T, root string) (fold.Manifest, memoryReader, []byte) { + t.Helper() + parts := [][]byte{[]byte("first-line\n"), bytes.Repeat([]byte("middle"), 11), []byte("\nlast-line\n")} + reader := memoryReader{} + manifest := fold.Manifest{Version: fold.ManifestVersion, Kind: fold.ManifestKind, Session: fold.ManifestSession{ID: "session", RolloutPath: filepath.Join(root, "native.jsonl")}} + var source []byte + for _, partBytes := range parts { + digest := digestBytes(partBytes) + reader[digest] = partBytes + manifest.Parts = append(manifest.Parts, fold.Part{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: digest, RawBytes: int64(len(partBytes))}}) + source = append(source, partBytes...) + } + manifest.Source = fold.ManifestSource{Bytes: int64(len(source)), SHA256: digestBytes(source)} + if err := os.WriteFile(manifest.Session.RolloutPath, source, 0o600); err != nil { + t.Fatalf("write native snapshot: %v", err) + } + return manifest, reader, source +} + +func openFixtureSession(t *testing.T, root string, manifest fold.Manifest, reader memoryReader, hook func(string) error) *Session { + t.Helper() + session, err := OpenSession(context.Background(), SessionOptions{ + Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, Reader: reader, + NativeSnapshot: NativeFile{Path: manifest.Session.RolloutPath, Bytes: manifest.Source.Bytes, SHA256: manifest.Source.SHA256}, + BeforeCOWPhase: hook, + }) + if err != nil { + t.Fatalf("OpenSession returned error: %v", err) + } + return session +} + +func readHandle(t *testing.T, handle *ReadHandle) []byte { + t.Helper() + buffer := make([]byte, handle.Size()) + n, err := handle.ReadAt(context.Background(), buffer, 0) + if err != nil && !errors.Is(err, io.EOF) { + t.Fatalf("ReadAt returned error: %v", err) + } + return buffer[:n] +} + +func digestBytes(data []byte) string { + digest := sha256.Sum256(data) + return hex.EncodeToString(digest[:]) +} diff --git a/internal/vfs/state.go b/internal/vfs/state.go new file mode 100644 index 0000000..9992550 --- /dev/null +++ b/internal/vfs/state.go @@ -0,0 +1,114 @@ +package vfs + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +const sessionStateVersion = 1 + +type NativeFile struct { + Path string `json:"path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` +} + +type SessionState struct { + Version int `json:"version"` + SessionID string `json:"session_id"` + Generation uint64 `json:"generation"` + ManifestPath string `json:"manifest_path"` + BaseBytes int64 `json:"base_bytes"` + BaseSHA256 string `json:"base_sha256"` + DeltaPath string `json:"delta_path"` + BackingPath string `json:"backing_path,omitempty"` + NativeSnapshot NativeFile `json:"native_snapshot"` +} + +func loadSessionState(path string) (SessionState, error) { + data, err := os.ReadFile(path) + if err != nil { + return SessionState{}, err + } + var state SessionState + if err := json.Unmarshal(data, &state); err != nil { + return SessionState{}, fmt.Errorf("decode session state: %w", err) + } + if state.Version != sessionStateVersion || !safeSessionID(state.SessionID) || state.Generation == 0 || state.BaseBytes < 0 || len(state.BaseSHA256) != 64 || state.DeltaPath == "" { + return SessionState{}, errors.New("invalid virtual session state") + } + return state, nil +} + +func writeSessionState(path string, state SessionState) error { + data, err := json.MarshalIndent(state, "", " ") + if err != nil { + return fmt.Errorf("encode session state: %w", err) + } + data = append(data, '\n') + directory := filepath.Dir(path) + temporary, err := os.CreateTemp(directory, ".state-*.tmp") + if err != nil { + return fmt.Errorf("create temporary session state: %w", err) + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return fmt.Errorf("chmod temporary session state: %w", err) + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return fmt.Errorf("write temporary session state: %w", err) + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return fmt.Errorf("sync temporary session state: %w", err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close temporary session state: %w", err) + } + if err := replaceStateFile(temporaryPath, path); err != nil { + return fmt.Errorf("commit session state: %w", err) + } + return syncStateDirectory(directory) +} + +func verifyNativeFile(file NativeFile) error { + if file.Path == "" || file.Bytes < 0 || len(file.SHA256) != 64 { + return errors.New("native snapshot metadata is incomplete") + } + opened, err := os.Open(file.Path) + if err != nil { + return fmt.Errorf("open native snapshot: %w", err) + } + hasher := sha256.New() + bytesRead, copyErr := io.Copy(hasher, opened) + closeErr := opened.Close() + if copyErr != nil { + return fmt.Errorf("hash native snapshot: %w", copyErr) + } + if closeErr != nil { + return fmt.Errorf("close native snapshot: %w", closeErr) + } + if bytesRead != file.Bytes || hex.EncodeToString(hasher.Sum(nil)) != file.SHA256 { + return errors.New("native snapshot bytes or SHA-256 differ from metadata") + } + return nil +} + +func safeSessionID(sessionID string) bool { + return sessionID != "" && sessionID != "." && sessionID != ".." && !strings.ContainsAny(sessionID, "/\\\x00") +} + +func pathWithin(directory string, path string) bool { + relative, err := filepath.Rel(directory, path) + return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} diff --git a/internal/vfs/state_replace_unix.go b/internal/vfs/state_replace_unix.go new file mode 100644 index 0000000..09521fe --- /dev/null +++ b/internal/vfs/state_replace_unix.go @@ -0,0 +1,22 @@ +//go:build !windows + +package vfs + +import ( + "fmt" + "os" +) + +func replaceStateFile(source string, target string) error { return os.Rename(source, target) } + +func syncStateDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + if err := directory.Sync(); err != nil { + _ = directory.Close() + return fmt.Errorf("sync directory %s: %w", path, err) + } + return directory.Close() +} diff --git a/internal/vfs/state_replace_windows.go b/internal/vfs/state_replace_windows.go new file mode 100644 index 0000000..dd9b25f --- /dev/null +++ b/internal/vfs/state_replace_windows.go @@ -0,0 +1,29 @@ +//go:build windows + +package vfs + +import ( + "fmt" + "syscall" + "unsafe" +) + +var moveFileExW = syscall.NewLazyDLL("kernel32.dll").NewProc("MoveFileExW") + +func replaceStateFile(source string, target string) error { + sourcePointer, err := syscall.UTF16PtrFromString(source) + if err != nil { + return err + } + targetPointer, err := syscall.UTF16PtrFromString(target) + if err != nil { + return err + } + result, _, callErr := moveFileExW.Call(uintptr(unsafe.Pointer(sourcePointer)), uintptr(unsafe.Pointer(targetPointer)), 0x1|0x8) + if result == 0 { + return fmt.Errorf("replace file: %w", callErr) + } + return nil +} + +func syncStateDirectory(string) error { return nil } From 076d77224fb292f747067cb5ce4100536f8185c9 Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 00:51:46 +0800 Subject: [PATCH 05/33] feat: add journaled recovery compaction and fallback --- go.mod | 2 +- internal/vfs/compact.go | 193 ++++++++++++++++++++++++++++ internal/vfs/fallback.go | 9 ++ internal/vfs/handles.go | 11 +- internal/vfs/journal.go | 73 +++++++++++ internal/vfs/recover.go | 94 ++++++++++++++ internal/vfs/recovery_test.go | 183 ++++++++++++++++++++++++++ internal/vfs/session.go | 65 ++++++++-- internal/vfs/writer_lock_unix.go | 40 ++++++ internal/vfs/writer_lock_windows.go | 43 +++++++ 10 files changed, 696 insertions(+), 17 deletions(-) create mode 100644 internal/vfs/compact.go create mode 100644 internal/vfs/fallback.go create mode 100644 internal/vfs/journal.go create mode 100644 internal/vfs/recover.go create mode 100644 internal/vfs/recovery_test.go create mode 100644 internal/vfs/writer_lock_unix.go create mode 100644 internal/vfs/writer_lock_windows.go diff --git a/go.mod b/go.mod index d2cada0..62e0f28 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.26 require ( github.com/klauspost/compress v1.19.0 github.com/spf13/cobra v1.10.2 + golang.org/x/sys v0.36.0 modernc.org/sqlite v1.40.1 ) @@ -17,7 +18,6 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/spf13/pflag v1.0.9 // indirect golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect - golang.org/x/sys v0.36.0 // indirect modernc.org/libc v1.66.10 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/internal/vfs/compact.go b/internal/vfs/compact.go new file mode 100644 index 0000000..f9e056a --- /dev/null +++ b/internal/vfs/compact.go @@ -0,0 +1,193 @@ +package vfs + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/jstar0/codexfold/internal/fold" +) + +type PreparedGeneration struct { + ManifestPath string + Manifest fold.Manifest + View *View +} + +type CompactOptions struct { + IdleFor time.Duration + Prepare func(context.Context, NativeFile, uint64) (PreparedGeneration, error) + BeforePhase func(string) error +} + +type CompactResult struct { + Generation uint64 `json:"generation"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` +} + +func (s *Session) Compact(ctx context.Context, options CompactOptions) (CompactResult, error) { + if options.Prepare == nil { + return CompactResult{}, errors.New("compact preparation function is required") + } + s.mu.Lock() + if s.writerOpen { + s.mu.Unlock() + return CompactResult{}, errors.New("cannot compact while a writer lease is held") + } + state := s.state + s.mu.Unlock() + activePath := state.DeltaPath + if state.BackingPath != "" { + activePath = state.BackingPath + } + fingerprint, err := captureFingerprint(activePath) + if err != nil { + return CompactResult{}, err + } + if options.IdleFor > 0 && time.Since(fingerprint.ModTime) < options.IdleFor { + return CompactResult{}, errors.New("session is not idle enough for compaction") + } + current, err := s.MaterializeCurrent(ctx, filepath.Join(s.directory, fmt.Sprintf(".compact-%020d.jsonl", state.Generation)), true) + if err != nil { + return CompactResult{}, err + } + defer os.Remove(current.Path) + prepared, err := options.Prepare(ctx, current, state.Generation+1) + if err != nil { + return CompactResult{}, err + } + if prepared.View == nil || prepared.ManifestPath == "" || prepared.Manifest.Source.SHA256 != current.SHA256 || prepared.Manifest.Source.Bytes != current.Bytes { + return CompactResult{}, errors.New("compact preparation returned incomplete generation") + } + if prepared.View.Size() != current.Bytes { + return CompactResult{}, errors.New("prepared generation byte length differs from current view") + } + preparedDigest, err := hashView(ctx, prepared.View) + if err != nil { + return CompactResult{}, err + } + if preparedDigest != current.SHA256 { + return CompactResult{}, errors.New("prepared generation SHA-256 differs from current view") + } + if err := ensureFingerprintUnchanged(activePath, fingerprint); err != nil { + return CompactResult{}, err + } + newDelta := filepath.Join(s.directory, fmt.Sprintf("delta-%020d.jsonl", state.Generation+1)) + delta, err := os.OpenFile(newDelta, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600) + if err != nil { + return CompactResult{}, err + } + if err := delta.Sync(); err != nil { + _ = delta.Close() + return CompactResult{}, err + } + if err := delta.Close(); err != nil { + return CompactResult{}, err + } + next := state + next.Generation++ + next.ManifestPath = prepared.ManifestPath + next.BaseBytes = prepared.View.Size() + next.BaseSHA256 = current.SHA256 + next.DeltaPath = newDelta + next.BackingPath = "" + operationID := fmt.Sprintf("compact-%020d", state.Generation) + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "prepared", Candidate: next, FinalPath: newDelta, Native: current}); err != nil { + return CompactResult{}, err + } + if options.BeforePhase != nil { + if err := options.BeforePhase("after-prepare"); err != nil { + return CompactResult{}, err + } + } + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "state-publishing", Candidate: next, FinalPath: newDelta, Native: current}); err != nil { + return CompactResult{}, err + } + if options.BeforePhase != nil { + if err := options.BeforePhase("before-state-publish"); err != nil { + return CompactResult{}, err + } + } + if err := writeSessionState(s.statePath, next); err != nil { + return CompactResult{}, err + } + s.mu.Lock() + s.state = next + s.view = prepared.View + s.mu.Unlock() + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "state-published", Candidate: next, FinalPath: newDelta, Native: current}); err != nil { + return CompactResult{}, err + } + if options.BeforePhase != nil { + if err := options.BeforePhase("after-state-publish"); err != nil { + return CompactResult{}, err + } + } + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "complete", Candidate: next, FinalPath: newDelta, Native: current}); err != nil { + return CompactResult{}, err + } + return CompactResult{Generation: next.Generation, Bytes: current.Bytes, SHA256: current.SHA256}, nil +} + +func hashView(ctx context.Context, view *View) (string, error) { + hasher := sha256.New() + buffer := make([]byte, 1<<20) + var offset int64 + for offset < view.Size() { + need := len(buffer) + if remaining := view.Size() - offset; int64(need) > remaining { + need = int(remaining) + } + n, err := view.ReadAt(ctx, buffer[:need], offset) + if n > 0 { + _, _ = hasher.Write(buffer[:n]) + offset += int64(n) + } + if err != nil && !errors.Is(err, io.EOF) { + return "", err + } + if n == 0 { + break + } + } + if offset != view.Size() { + return "", errors.New("prepared generation ended before its declared size") + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +type fileFingerprint struct { + Bytes int64 + ModTime time.Time + SHA256 string +} + +func captureFingerprint(path string) (fileFingerprint, error) { + info, err := os.Stat(path) + if err != nil { + return fileFingerprint{}, err + } + current, err := hashNativePath(path) + if err != nil { + return fileFingerprint{}, err + } + return fileFingerprint{Bytes: info.Size(), ModTime: info.ModTime(), SHA256: current.SHA256}, nil +} + +func ensureFingerprintUnchanged(path string, initial fileFingerprint) error { + current, err := captureFingerprint(path) + if err != nil { + return err + } + if current.Bytes != initial.Bytes || !current.ModTime.Equal(initial.ModTime) || current.SHA256 != initial.SHA256 { + return errors.New("active session data changed during compaction") + } + return nil +} diff --git a/internal/vfs/fallback.go b/internal/vfs/fallback.go new file mode 100644 index 0000000..ac416c1 --- /dev/null +++ b/internal/vfs/fallback.go @@ -0,0 +1,9 @@ +package vfs + +import ( + "context" +) + +func (s *Session) CreateCurrentNativeBacking(ctx context.Context, target string) (NativeFile, error) { + return s.MaterializeCurrent(ctx, target, false) +} diff --git a/internal/vfs/handles.go b/internal/vfs/handles.go index 4e7ee7b..2cad82d 100644 --- a/internal/vfs/handles.go +++ b/internal/vfs/handles.go @@ -105,6 +105,7 @@ func (h *ReadHandle) Close() error { type WriteHandle struct { session *Session leasePath string + lease *os.File mu sync.Mutex closed bool } @@ -208,12 +209,16 @@ func (h *WriteHandle) Close() error { return nil } h.closed = true - removeErr := os.Remove(h.leasePath) + unlockErr := unlockWriterFile(h.lease) + closeErr := h.lease.Close() h.session.mu.Lock() h.session.writerOpen = false h.session.mu.Unlock() - if removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { - return removeErr + if unlockErr != nil { + return unlockErr + } + if closeErr != nil { + return closeErr } return nil } diff --git a/internal/vfs/journal.go b/internal/vfs/journal.go new file mode 100644 index 0000000..6d3c3fc --- /dev/null +++ b/internal/vfs/journal.go @@ -0,0 +1,73 @@ +package vfs + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" +) + +type JournalRecord struct { + OperationID string `json:"operation_id"` + SessionID string `json:"session_id"` + Kind string `json:"kind"` + Phase string `json:"phase"` + At string `json:"at"` + TempPath string `json:"temp_path,omitempty"` + FinalPath string `json:"final_path,omitempty"` + Candidate SessionState `json:"candidate"` + Native NativeFile `json:"native,omitempty"` +} + +func journalPath(directory string) string { return filepath.Join(directory, "journal.jsonl") } + +func appendJournal(directory string, record JournalRecord) error { + record.At = time.Now().UTC().Format(time.RFC3339Nano) + data, err := json.Marshal(record) + if err != nil { + return fmt.Errorf("encode journal record: %w", err) + } + file, err := os.OpenFile(journalPath(directory), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("open session journal: %w", err) + } + if _, err := file.Write(append(data, '\n')); err != nil { + _ = file.Close() + return fmt.Errorf("write session journal: %w", err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + return fmt.Errorf("sync session journal: %w", err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("close session journal: %w", err) + } + return nil +} + +func readJournal(directory string) ([]JournalRecord, error) { + file, err := os.Open(journalPath(directory)) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + defer file.Close() + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 4096), 1<<20) + var records []JournalRecord + for scanner.Scan() { + var record JournalRecord + if err := json.Unmarshal(scanner.Bytes(), &record); err != nil { + return nil, fmt.Errorf("decode session journal: %w", err) + } + records = append(records, record) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read session journal: %w", err) + } + return records, nil +} diff --git a/internal/vfs/recover.go b/internal/vfs/recover.go new file mode 100644 index 0000000..b79f47c --- /dev/null +++ b/internal/vfs/recover.go @@ -0,0 +1,94 @@ +package vfs + +import ( + "context" + "errors" + "fmt" + "os" + "sort" +) + +func (s *Session) recover(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + records, err := readJournal(s.directory) + if err != nil { + return err + } + latest := make(map[string]JournalRecord) + lastPosition := make(map[string]int) + for index, record := range records { + if record.OperationID == "" { + return errors.New("session journal record has no operation ID") + } + latest[record.OperationID] = record + lastPosition[record.OperationID] = index + } + ordered := make([]JournalRecord, 0, len(latest)) + for _, record := range latest { + ordered = append(ordered, record) + } + sort.Slice(ordered, func(i, j int) bool { + return lastPosition[ordered[i].OperationID] < lastPosition[ordered[j].OperationID] + }) + for _, record := range ordered { + switch record.Phase { + case "complete", "rolled-back": + continue + case "after-file-publish", "state-publishing", "state-published": + if record.Candidate.SessionID != s.state.SessionID || record.Candidate.Generation == 0 || !pathWithin(s.directory, record.Candidate.DeltaPath) || (record.Candidate.BackingPath != "" && !pathWithin(s.directory, record.Candidate.BackingPath)) { + return fmt.Errorf("journal operation %s has unsafe candidate state", record.OperationID) + } + state, err := loadSessionState(s.statePath) + if err != nil { + return err + } + if record.Kind == "compact" { + if state.Generation < record.Candidate.Generation { + _ = os.Remove(record.Candidate.DeltaPath) + if err := appendJournal(s.directory, JournalRecord{OperationID: record.OperationID, SessionID: record.SessionID, Kind: record.Kind, Phase: "rolled-back", Candidate: record.Candidate, FinalPath: record.FinalPath}); err != nil { + return err + } + continue + } + if state.Generation != record.Candidate.Generation || state.ManifestPath != record.Candidate.ManifestPath { + return fmt.Errorf("journal operation %s conflicts with current compacted state", record.OperationID) + } + s.state = state + } else { + if record.FinalPath == "" || record.Candidate.BackingPath == "" { + return fmt.Errorf("journal operation %s has incomplete published state", record.OperationID) + } + verified, err := hashNativePath(record.FinalPath) + if err != nil || verified.Bytes != record.Native.Bytes || verified.SHA256 != record.Native.SHA256 { + return fmt.Errorf("journal operation %s published backing cannot be verified: %w", record.OperationID, err) + } + if state.Generation < record.Candidate.Generation { + if err := writeSessionState(s.statePath, record.Candidate); err != nil { + return err + } + s.state = record.Candidate + } + } + if err := appendJournal(s.directory, JournalRecord{OperationID: record.OperationID, SessionID: record.SessionID, Kind: record.Kind, Phase: "complete", Candidate: record.Candidate, FinalPath: record.FinalPath, Native: record.Native}); err != nil { + return err + } + case "prepared", "data-synced": + if record.TempPath != "" { + _ = os.Remove(record.TempPath) + } + if record.Kind == "compact" && record.FinalPath != "" { + _ = os.Remove(record.FinalPath) + } + if err := appendJournal(s.directory, JournalRecord{OperationID: record.OperationID, SessionID: record.SessionID, Kind: record.Kind, Phase: "rolled-back", Candidate: record.Candidate, TempPath: record.TempPath}); err != nil { + return err + } + default: + return fmt.Errorf("journal operation %s has unknown phase %q", record.OperationID, record.Phase) + } + } + return nil +} + +func (s *Session) Recover(ctx context.Context) error { return s.recover(ctx) } diff --git a/internal/vfs/recovery_test.go b/internal/vfs/recovery_test.go new file mode 100644 index 0000000..ba6dcd8 --- /dev/null +++ b/internal/vfs/recovery_test.go @@ -0,0 +1,183 @@ +package vfs + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/jstar0/codexfold/internal/fold" +) + +func TestRecoverFinishesPublishedCopyOnWriteGeneration(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + stop := errors.New("stop after backing publish") + session := openFixtureSession(t, root, manifest, reader, func(phase string) error { + if phase == "after-file-publish" { + return stop + } + return nil + }) + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + if _, err := writer.WriteAt(context.Background(), []byte("X"), 0); !errors.Is(err, stop) { + t.Fatalf("WriteAt error = %v, want %v", err, stop) + } + _ = writer.Close() + if session.State().BackingPath != "" { + t.Fatal("interrupted state should not publish backing before recovery") + } + + reopened := openFixtureSession(t, root, manifest, reader, nil) + if reopened.State().BackingPath == "" || reopened.State().Generation != 2 { + t.Fatalf("recovery did not finish COW state: %#v", reopened.State()) + } + handle, err := reopened.OpenReader() + if err != nil { + t.Fatalf("OpenReader: %v", err) + } + defer handle.Close() + if got := readHandle(t, handle); !bytes.Equal(got, source) { + t.Fatalf("recovered backing differs: got=%q want=%q", got, source) + } +} + +func TestCreateCurrentNativeBackingIncludesVirtualTail(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + if _, err := writer.Append(context.Background(), []byte("-new-tail")); err != nil { + t.Fatalf("Append: %v", err) + } + _ = writer.Close() + + target := filepath.Join(root, "fallback", "session.jsonl") + backing, err := session.CreateCurrentNativeBacking(context.Background(), target) + if err != nil { + t.Fatalf("CreateCurrentNativeBacking: %v", err) + } + want := append(append([]byte(nil), source...), []byte("-new-tail")...) + got, err := os.ReadFile(target) + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("fallback differs: got=%q err=%v", got, err) + } + if backing.SHA256 != digestBytes(want) || backing.SHA256 == manifest.Source.SHA256 { + t.Fatalf("fallback digest does not represent current bytes: %#v", backing) + } +} + +func TestCompactSwitchesGenerationAndPreservesPinnedReader(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + writer, _ := session.OpenWriter() + _, _ = writer.Append(context.Background(), []byte("-tail")) + _ = writer.Close() + oldReader, err := session.OpenReader() + if err != nil { + t.Fatalf("OpenReader before compact: %v", err) + } + defer oldReader.Close() + oldTime := time.Now().Add(-time.Hour) + if err := os.Chtimes(session.State().DeltaPath, oldTime, oldTime); err != nil { + t.Fatalf("age delta: %v", err) + } + + want := append(append([]byte(nil), source...), []byte("-tail")...) + result, err := session.Compact(context.Background(), CompactOptions{ + IdleFor: 10 * time.Minute, + Prepare: func(_ context.Context, current NativeFile, next uint64) (PreparedGeneration, error) { + data, err := os.ReadFile(current.Path) + if err != nil { + return PreparedGeneration{}, err + } + digest := digestBytes(data) + preparedManifest := fold.Manifest{ + Version: fold.ManifestVersion, Kind: fold.ManifestKind, + Session: fold.ManifestSession{ID: "session", RolloutPath: current.Path}, + Source: fold.ManifestSource{Bytes: int64(len(data)), SHA256: digest}, + Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: digest, RawBytes: int64(len(data))}}}, + } + preparedReader := memoryReader{digest: data} + view, err := NewView(preparedManifest, preparedReader) + return PreparedGeneration{ManifestPath: filepath.Join(root, "manifest-generation-2.json"), Manifest: preparedManifest, View: view}, err + }, + }) + if err != nil { + t.Fatalf("Compact: %v", err) + } + if result.Generation != 2 || session.State().BackingPath != "" { + t.Fatalf("unexpected compact result/state: result=%#v state=%#v", result, session.State()) + } + newReader, err := session.OpenReader() + if err != nil { + t.Fatalf("OpenReader after compact: %v", err) + } + defer newReader.Close() + if got := readHandle(t, newReader); !bytes.Equal(got, want) { + t.Fatalf("compacted generation differs: got=%q want=%q", got, want) + } + if got := readHandle(t, oldReader); !bytes.Equal(got, want) { + t.Fatalf("pinned old reader changed: got=%q want=%q", got, want) + } +} + +func TestCompactRejectsDeltaChangedDuringPreparation(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + writer, _ := session.OpenWriter() + _, _ = writer.Append(context.Background(), []byte("initial")) + _ = writer.Close() + oldTime := time.Now().Add(-time.Hour) + _ = os.Chtimes(session.State().DeltaPath, oldTime, oldTime) + + _, err := session.Compact(context.Background(), CompactOptions{ + IdleFor: 10 * time.Minute, + Prepare: func(_ context.Context, current NativeFile, _ uint64) (PreparedGeneration, error) { + file, openErr := os.OpenFile(session.State().DeltaPath, os.O_APPEND|os.O_WRONLY, 0) + if openErr != nil { + return PreparedGeneration{}, openErr + } + _, _ = file.WriteString("changed") + _ = file.Close() + data, _ := os.ReadFile(current.Path) + digest := digestBytes(data) + preparedManifest := fold.Manifest{Version: fold.ManifestVersion, Kind: fold.ManifestKind, Session: fold.ManifestSession{ID: "session"}, Source: fold.ManifestSource{Bytes: int64(len(data)), SHA256: digest}, Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: digest, RawBytes: int64(len(data))}}}} + view, _ := NewView(preparedManifest, memoryReader{digest: data}) + return PreparedGeneration{ManifestPath: filepath.Join(root, "next.json"), Manifest: preparedManifest, View: view}, nil + }, + }) + if err == nil { + t.Fatal("Compact should reject a delta changed during preparation") + } + if session.State().Generation != 1 { + t.Fatalf("failed compact changed generation: %#v", session.State()) + } +} + +func TestOpenSessionCleansUnlockedStaleWriterLease(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + leasePath := filepath.Join(session.directory, "writer.lease") + if err := os.WriteFile(leasePath, []byte("stale\n"), 0o600); err != nil { + t.Fatalf("write stale lease: %v", err) + } + reopened := openFixtureSession(t, root, manifest, reader, nil) + writer, err := reopened.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter after stale lease cleanup: %v", err) + } + _ = writer.Close() +} diff --git a/internal/vfs/session.go b/internal/vfs/session.go index 3060f6e..3af5d7b 100644 --- a/internal/vfs/session.go +++ b/internal/vfs/session.go @@ -50,6 +50,9 @@ func OpenSession(ctx context.Context, options SessionOptions) (*Session, error) return nil, fmt.Errorf("create virtual session directory: %w", err) } statePath := filepath.Join(directory, "state.json") + if err := cleanupStaleWriterLease(filepath.Join(directory, "writer.lease")); err != nil { + return nil, err + } state, err := loadSessionState(statePath) if errors.Is(err, os.ErrNotExist) { if err := verifyNativeFile(options.NativeSnapshot); err != nil { @@ -89,7 +92,14 @@ func OpenSession(ctx context.Context, options SessionOptions) (*Session, error) } } } - return &Session{state: state, statePath: statePath, directory: directory, view: view, readerLeases: make(map[uint64]int), beforeCOWPhase: options.BeforeCOWPhase}, nil + session := &Session{state: state, statePath: statePath, directory: directory, view: view, readerLeases: make(map[uint64]int), beforeCOWPhase: options.BeforeCOWPhase} + if err := session.recover(ctx); err != nil { + return nil, err + } + if recovered, err := loadSessionState(statePath); err == nil { + session.state = recovered + } + return session, nil } func (s *Session) State() SessionState { @@ -101,10 +111,11 @@ func (s *Session) State() SessionState { func (s *Session) OpenReader() (*ReadHandle, error) { s.mu.Lock() state := s.state + view := s.view s.readerLeases[state.Generation]++ s.mu.Unlock() - handle := &ReadHandle{session: s, generation: state.Generation, base: s.view, baseBytes: state.BaseBytes} + handle := &ReadHandle{session: s, generation: state.Generation, base: view, baseBytes: state.BaseBytes} var path string if state.BackingPath != "" { path = state.BackingPath @@ -150,29 +161,36 @@ func (s *Session) OpenWriter() (*WriteHandle, error) { return nil, errors.New("session writer lease is already held") } leasePath := filepath.Join(s.directory, "writer.lease") - lease, err := os.OpenFile(leasePath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + lease, err := os.OpenFile(leasePath, os.O_CREATE|os.O_RDWR, 0o600) if err != nil { - if errors.Is(err, os.ErrExist) { - return nil, errors.New("session writer lease file already exists") - } return nil, fmt.Errorf("create writer lease: %w", err) } + locked, err := tryLockWriterFile(lease) + if err != nil { + _ = lease.Close() + return nil, err + } + if !locked { + _ = lease.Close() + return nil, errors.New("session writer lease is held by another process") + } + if err := lease.Truncate(0); err != nil { + _ = unlockWriterFile(lease) + _ = lease.Close() + return nil, err + } if _, err := fmt.Fprintf(lease, "%d\n", os.Getpid()); err != nil { + _ = unlockWriterFile(lease) _ = lease.Close() - _ = os.Remove(leasePath) return nil, fmt.Errorf("write writer lease: %w", err) } if err := lease.Sync(); err != nil { + _ = unlockWriterFile(lease) _ = lease.Close() - _ = os.Remove(leasePath) return nil, fmt.Errorf("sync writer lease: %w", err) } - if err := lease.Close(); err != nil { - _ = os.Remove(leasePath) - return nil, fmt.Errorf("close writer lease: %w", err) - } s.writerOpen = true - return &WriteHandle{session: s, leasePath: leasePath}, nil + return &WriteHandle{session: s, leasePath: leasePath, lease: lease}, nil } func (s *Session) ensureBacking(ctx context.Context) (string, error) { @@ -247,6 +265,10 @@ func (s *Session) ensureBacking(ctx context.Context) (string, error) { if verified.Bytes != reader.Size() || verified.SHA256 != hex.EncodeToString(sourceHash.Sum(nil)) { return "", errors.New("temporary backing verification failed") } + operationID := fmt.Sprintf("cow-%020d", currentGeneration) + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: s.state.SessionID, Kind: "copy-on-write", Phase: "data-synced", TempPath: temporaryPath, Native: verified}); err != nil { + return "", err + } if s.beforeCOWPhase != nil { if err := s.beforeCOWPhase("before-publish"); err != nil { return "", err @@ -259,6 +281,17 @@ func (s *Session) ensureBacking(ctx context.Context) (string, error) { if err := syncStateDirectory(s.directory); err != nil { return "", err } + candidate := s.state + candidate.Generation = currentGeneration + 1 + candidate.BackingPath = backingPath + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: s.state.SessionID, Kind: "copy-on-write", Phase: "after-file-publish", Candidate: candidate, FinalPath: backingPath, Native: verified}); err != nil { + return "", err + } + if s.beforeCOWPhase != nil { + if err := s.beforeCOWPhase("after-file-publish"); err != nil { + return "", err + } + } s.mu.Lock() defer s.mu.Unlock() if s.state.Generation != currentGeneration || s.state.BackingPath != "" { @@ -271,6 +304,12 @@ func (s *Session) ensureBacking(ctx context.Context) (string, error) { return "", err } s.state = next + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: next.SessionID, Kind: "copy-on-write", Phase: "state-published", Candidate: next, FinalPath: backingPath, Native: verified}); err != nil { + return "", err + } + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: next.SessionID, Kind: "copy-on-write", Phase: "complete", Candidate: next, FinalPath: backingPath, Native: verified}); err != nil { + return "", err + } return backingPath, nil } diff --git a/internal/vfs/writer_lock_unix.go b/internal/vfs/writer_lock_unix.go new file mode 100644 index 0000000..a65c726 --- /dev/null +++ b/internal/vfs/writer_lock_unix.go @@ -0,0 +1,40 @@ +//go:build !windows + +package vfs + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +func tryLockWriterFile(file *os.File) (bool, error) { + err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if errors.Is(err, unix.EWOULDBLOCK) { + return false, nil + } + return err == nil, err +} + +func unlockWriterFile(file *os.File) error { return unix.Flock(int(file.Fd()), unix.LOCK_UN) } + +func cleanupStaleWriterLease(path string) error { + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return err + } + locked, err := tryLockWriterFile(file) + if err != nil { + _ = file.Close() + return err + } + if !locked { + return file.Close() + } + if err := unlockWriterFile(file); err != nil { + _ = file.Close() + return err + } + return file.Close() +} diff --git a/internal/vfs/writer_lock_windows.go b/internal/vfs/writer_lock_windows.go new file mode 100644 index 0000000..ce721e7 --- /dev/null +++ b/internal/vfs/writer_lock_windows.go @@ -0,0 +1,43 @@ +//go:build windows + +package vfs + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +func tryLockWriterFile(file *os.File) (bool, error) { + overlapped := new(windows.Overlapped) + err := windows.LockFileEx(windows.Handle(file.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, overlapped) + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return false, nil + } + return err == nil, err +} + +func unlockWriterFile(file *os.File) error { + return windows.UnlockFileEx(windows.Handle(file.Fd()), 0, 1, 0, new(windows.Overlapped)) +} + +func cleanupStaleWriterLease(path string) error { + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return err + } + locked, err := tryLockWriterFile(file) + if err != nil { + _ = file.Close() + return err + } + if !locked { + return file.Close() + } + if err := unlockWriterFile(file); err != nil { + _ = file.Close() + return err + } + return file.Close() +} From 35a53fc8a329526fac3124a714bed7c440921d0d Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 00:56:10 +0800 Subject: [PATCH 06/33] feat: add shadow doctor benchmark and fs status --- internal/fsctl/benchmark.go | 167 +++++++++++++++++++++++++++++++++++ internal/fsctl/doctor.go | 78 ++++++++++++++++ internal/fsctl/fsctl_test.go | 105 ++++++++++++++++++++++ internal/fsctl/shadow.go | 123 ++++++++++++++++++++++++++ internal/fsctl/status.go | 34 +++++++ 5 files changed, 507 insertions(+) create mode 100644 internal/fsctl/benchmark.go create mode 100644 internal/fsctl/doctor.go create mode 100644 internal/fsctl/fsctl_test.go create mode 100644 internal/fsctl/shadow.go create mode 100644 internal/fsctl/status.go diff --git a/internal/fsctl/benchmark.go b/internal/fsctl/benchmark.go new file mode 100644 index 0000000..041a9a9 --- /dev/null +++ b/internal/fsctl/benchmark.go @@ -0,0 +1,167 @@ +package fsctl + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "math/rand" + "os" + "runtime" + "sort" + "time" +) + +type BenchmarkOptions struct { + SequentialBlockBytes int + RandomBlockBytes int + RandomReads int + Seed int64 +} + +type SequentialMetric struct { + Bytes int64 `json:"bytes"` + Duration time.Duration `json:"duration"` + BytesPerSecond float64 `json:"bytes_per_second"` +} + +type RandomMetric struct { + Reads int `json:"reads"` + P50 time.Duration `json:"p50"` + P95 time.Duration `json:"p95"` + P99 time.Duration `json:"p99"` +} + +type BenchmarkReport struct { + Native SequentialMetric `json:"native"` + Virtual SequentialMetric `json:"virtual"` + Random RandomMetric `json:"random"` + GoSysBytes uint64 `json:"go_sys_bytes"` +} + +func Benchmark(ctx context.Context, nativePath string, virtual Readable, options BenchmarkOptions) (BenchmarkReport, error) { + if options.SequentialBlockBytes <= 0 { + options.SequentialBlockBytes = 1 << 20 + } + if options.RandomBlockBytes <= 0 { + options.RandomBlockBytes = 4 << 10 + } + if options.RandomReads <= 0 { + options.RandomReads = 1000 + } + native, err := os.Open(nativePath) + if err != nil { + return BenchmarkReport{}, err + } + defer native.Close() + info, err := native.Stat() + if err != nil { + return BenchmarkReport{}, err + } + if info.Size() != virtual.Size() { + return BenchmarkReport{}, errors.New("benchmark native and virtual sizes differ") + } + nativeMetric, err := benchmarkNativeSequential(ctx, native, info.Size(), options.SequentialBlockBytes) + if err != nil { + return BenchmarkReport{}, err + } + virtualMetric, err := benchmarkVirtualSequential(ctx, virtual, options.SequentialBlockBytes) + if err != nil { + return BenchmarkReport{}, err + } + randomMetric, err := benchmarkRandom(ctx, native, virtual, info.Size(), options) + if err != nil { + return BenchmarkReport{}, err + } + var memory runtime.MemStats + runtime.ReadMemStats(&memory) + return BenchmarkReport{Native: nativeMetric, Virtual: virtualMetric, Random: randomMetric, GoSysBytes: memory.Sys}, nil +} + +func benchmarkNativeSequential(ctx context.Context, file *os.File, size int64, blockBytes int) (SequentialMetric, error) { + buffer := make([]byte, blockBytes) + start := time.Now() + var offset int64 + for offset < size { + if err := ctx.Err(); err != nil { + return SequentialMetric{}, err + } + need := blockBytes + if remaining := size - offset; int64(need) > remaining { + need = int(remaining) + } + n, err := file.ReadAt(buffer[:need], offset) + if n != need || (err != nil && !errors.Is(err, io.EOF)) { + return SequentialMetric{}, fmt.Errorf("native sequential read at %d: n=%d err=%v", offset, n, err) + } + offset += int64(n) + } + return sequentialMetric(offset, time.Since(start)), nil +} + +func benchmarkVirtualSequential(ctx context.Context, virtual Readable, blockBytes int) (SequentialMetric, error) { + buffer := make([]byte, blockBytes) + start := time.Now() + var offset int64 + for offset < virtual.Size() { + need := blockBytes + if remaining := virtual.Size() - offset; int64(need) > remaining { + need = int(remaining) + } + n, err := virtual.ReadAt(ctx, buffer[:need], offset) + if n != need || (err != nil && !errors.Is(err, io.EOF)) { + return SequentialMetric{}, fmt.Errorf("virtual sequential read at %d: n=%d err=%v", offset, n, err) + } + offset += int64(n) + } + return sequentialMetric(offset, time.Since(start)), nil +} + +func benchmarkRandom(ctx context.Context, native *os.File, virtual Readable, size int64, options BenchmarkOptions) (RandomMetric, error) { + if size == 0 { + return RandomMetric{}, nil + } + random := rand.New(rand.NewSource(options.Seed)) + nativeBuffer := make([]byte, options.RandomBlockBytes) + virtualBuffer := make([]byte, options.RandomBlockBytes) + durations := make([]time.Duration, 0, options.RandomReads) + for index := 0; index < options.RandomReads; index++ { + offset := random.Int63n(size) + length := options.RandomBlockBytes + if remaining := size - offset; int64(length) > remaining { + length = int(remaining) + } + nativeN, nativeErr := native.ReadAt(nativeBuffer[:length], offset) + start := time.Now() + virtualN, virtualErr := virtual.ReadAt(ctx, virtualBuffer[:length], offset) + duration := time.Since(start) + if duration <= 0 { + duration = time.Nanosecond + } + if nativeN != length || virtualN != length || !bytes.Equal(nativeBuffer[:length], virtualBuffer[:length]) || (nativeErr != nil && !errors.Is(nativeErr, io.EOF)) || (virtualErr != nil && !errors.Is(virtualErr, io.EOF)) { + return RandomMetric{}, fmt.Errorf("random benchmark read %d differs at offset %d", index, offset) + } + durations = append(durations, duration) + } + sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) + return RandomMetric{Reads: len(durations), P50: percentile(durations, 50), P95: percentile(durations, 95), P99: percentile(durations, 99)}, nil +} + +func sequentialMetric(bytesRead int64, duration time.Duration) SequentialMetric { + if duration <= 0 { + duration = time.Nanosecond + } + return SequentialMetric{Bytes: bytesRead, Duration: duration, BytesPerSecond: float64(bytesRead) / duration.Seconds()} +} + +func percentile(values []time.Duration, percent int) time.Duration { + if len(values) == 0 { + return 0 + } + index := (len(values)*percent + 99) / 100 + if index < 1 { + index = 1 + } + return values[index-1] +} diff --git a/internal/fsctl/doctor.go b/internal/fsctl/doctor.go new file mode 100644 index 0000000..1b0454f --- /dev/null +++ b/internal/fsctl/doctor.go @@ -0,0 +1,78 @@ +package fsctl + +import ( + "context" + "fmt" +) + +const ( + ComponentDaemon = "daemon" + ComponentMount = "mount" + ComponentPack = "pack" + ComponentManifest = "manifest" + ComponentDelta = "delta" + ComponentBacking = "backing" + ComponentRoute = "route" + ComponentFallback = "fallback" + ComponentJournal = "journal" + ComponentClient = "client" +) + +var RequiredComponents = []string{ComponentDaemon, ComponentMount, ComponentPack, ComponentManifest, ComponentDelta, ComponentBacking, ComponentRoute, ComponentFallback, ComponentJournal, ComponentClient} + +type Check struct { + Component string + Run func(context.Context) error +} + +type Issue struct { + Component string `json:"component"` + Severity string `json:"severity"` + SessionID string `json:"session_id,omitempty"` + Generation uint64 `json:"generation,omitempty"` + Message string `json:"message"` + Remediation string `json:"remediation,omitempty"` +} + +type DoctorReport struct { + Healthy bool `json:"healthy"` + IssueCount int `json:"issue_count"` + Issues []Issue `json:"issues,omitempty"` + ComponentHealth map[string]bool `json:"component_health"` +} + +func Doctor(ctx context.Context, checks []Check) DoctorReport { + report := DoctorReport{Healthy: true, ComponentHealth: make(map[string]bool)} + seen := make(map[string]bool) + for _, check := range checks { + if check.Component == "" || check.Run == nil { + report.Issues = append(report.Issues, Issue{Component: check.Component, Severity: "error", Message: "invalid doctor check"}) + continue + } + if seen[check.Component] { + report.Issues = append(report.Issues, Issue{Component: check.Component, Severity: "error", Message: "duplicate doctor check"}) + continue + } + seen[check.Component] = true + if err := ctx.Err(); err != nil { + report.ComponentHealth[check.Component] = false + report.Issues = append(report.Issues, Issue{Component: check.Component, Severity: "error", Message: err.Error()}) + continue + } + if err := check.Run(ctx); err != nil { + report.ComponentHealth[check.Component] = false + report.Issues = append(report.Issues, Issue{Component: check.Component, Severity: "error", Message: err.Error()}) + } else { + report.ComponentHealth[check.Component] = true + } + } + for _, component := range RequiredComponents { + if !seen[component] { + report.ComponentHealth[component] = false + report.Issues = append(report.Issues, Issue{Component: component, Severity: "error", Message: fmt.Sprintf("required %s check is missing", component), Remediation: "register and run the required component check"}) + } + } + report.IssueCount = len(report.Issues) + report.Healthy = report.IssueCount == 0 + return report +} diff --git a/internal/fsctl/fsctl_test.go b/internal/fsctl/fsctl_test.go new file mode 100644 index 0000000..0115f97 --- /dev/null +++ b/internal/fsctl/fsctl_test.go @@ -0,0 +1,105 @@ +package fsctl + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "path/filepath" + "testing" +) + +func TestStatusAcceptsOnlyCanonicalCapabilities(t *testing.T) { + for _, capability := range []Capability{StorageEngine, FSEnginePreview, PlatformCanary, Capability("production-ready:macos"), CrossPlatformReady} { + if _, err := NewStatus(capability, "darwin"); err != nil { + t.Fatalf("NewStatus(%q) returned error: %v", capability, err) + } + } + for _, capability := range []Capability{"transparent", "stable", "production-ready"} { + if _, err := NewStatus(capability, "darwin"); err == nil { + t.Fatalf("NewStatus(%q) should reject non-canonical capability", capability) + } + } +} + +func TestShadowComparesCompleteAndRandomBytes(t *testing.T) { + root := t.TempDir() + data := bytes.Repeat([]byte("shadow-source-"), 1000) + nativePath := filepath.Join(root, "native.jsonl") + if err := os.WriteFile(nativePath, data, 0o600); err != nil { + t.Fatalf("write native: %v", err) + } + result, err := Shadow(context.Background(), nativePath, byteReader(data), ShadowOptions{BlockBytes: 257, RandomReads: 10000, Seed: 42}) + if err != nil { + t.Fatalf("Shadow returned error: %v", err) + } + if !result.Verified || result.RandomReads != 10000 || result.ComparedBytes != int64(len(data)) { + t.Fatalf("unexpected shadow result: %#v", result) + } + + corrupt := append([]byte(nil), data...) + corrupt[len(corrupt)/2] ^= 1 + if result, err := Shadow(context.Background(), nativePath, byteReader(corrupt), ShadowOptions{BlockBytes: 257, RandomReads: 100, Seed: 42}); err == nil || result.Verified { + t.Fatalf("Shadow should reject one-byte mismatch: result=%#v err=%v", result, err) + } +} + +func TestDoctorRequiresEveryComponentAndSeparatesDaemonFromMount(t *testing.T) { + checks := make([]Check, 0, len(RequiredComponents)) + for _, component := range RequiredComponents { + component := component + checks = append(checks, Check{Component: component, Run: func(context.Context) error { + if component == ComponentMount { + return errors.New("mount unavailable") + } + return nil + }}) + } + report := Doctor(context.Background(), checks) + if report.Healthy || report.ComponentHealth[ComponentDaemon] != true || report.ComponentHealth[ComponentMount] != false { + t.Fatalf("doctor did not separate daemon and mount: %#v", report) + } + if len(report.Issues) != 1 || report.Issues[0].Component != ComponentMount { + t.Fatalf("unexpected doctor issues: %#v", report.Issues) + } + + report = Doctor(context.Background(), checks[:len(checks)-1]) + if report.Healthy || report.IssueCount < 2 { + t.Fatalf("doctor should report a missing required component: %#v", report) + } +} + +func TestBenchmarkMeasuresNativeAndVirtualReads(t *testing.T) { + root := t.TempDir() + data := bytes.Repeat([]byte("benchmark-data-"), 10000) + nativePath := filepath.Join(root, "native.jsonl") + if err := os.WriteFile(nativePath, data, 0o600); err != nil { + t.Fatalf("write native: %v", err) + } + report, err := Benchmark(context.Background(), nativePath, byteReader(data), BenchmarkOptions{SequentialBlockBytes: 4096, RandomBlockBytes: 4096, RandomReads: 200, Seed: 9}) + if err != nil { + t.Fatalf("Benchmark returned error: %v", err) + } + if report.Native.Bytes != int64(len(data)) || report.Virtual.Bytes != int64(len(data)) || report.Random.Reads != 200 { + t.Fatalf("unexpected benchmark report: %#v", report) + } + if report.Native.Duration <= 0 || report.Virtual.Duration <= 0 || report.Random.P95 <= 0 { + t.Fatalf("benchmark durations were not recorded: %#v", report) + } +} + +type byteReader []byte + +func (r byteReader) Size() int64 { return int64(len(r)) } + +func (r byteReader) ReadAt(_ context.Context, destination []byte, offset int64) (int, error) { + if offset >= int64(len(r)) { + return 0, io.EOF + } + n := copy(destination, r[offset:]) + if n < len(destination) { + return n, io.EOF + } + return n, nil +} diff --git a/internal/fsctl/shadow.go b/internal/fsctl/shadow.go new file mode 100644 index 0000000..d84737c --- /dev/null +++ b/internal/fsctl/shadow.go @@ -0,0 +1,123 @@ +package fsctl + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "math/rand" + "os" +) + +type Readable interface { + Size() int64 + ReadAt(context.Context, []byte, int64) (int, error) +} + +type ShadowOptions struct { + BlockBytes int + RandomReads int + Seed int64 +} + +type ShadowResult struct { + NativePath string `json:"native_path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` + ComparedBytes int64 `json:"compared_bytes"` + RandomReads int `json:"random_reads"` + Verified bool `json:"verified"` +} + +func Shadow(ctx context.Context, nativePath string, virtual Readable, options ShadowOptions) (ShadowResult, error) { + if options.BlockBytes <= 0 { + options.BlockBytes = 1 << 20 + } + if options.RandomReads < 0 { + return ShadowResult{}, errors.New("shadow random read count cannot be negative") + } + native, err := os.Open(nativePath) + if err != nil { + return ShadowResult{}, err + } + defer native.Close() + info, err := native.Stat() + if err != nil { + return ShadowResult{}, err + } + if info.Size() != virtual.Size() { + return ShadowResult{}, fmt.Errorf("shadow size mismatch native=%d virtual=%d", info.Size(), virtual.Size()) + } + nativeHash := sha256.New() + virtualHash := sha256.New() + nativeBuffer := make([]byte, options.BlockBytes) + virtualBuffer := make([]byte, options.BlockBytes) + var offset int64 + for offset < info.Size() { + if err := ctx.Err(); err != nil { + return ShadowResult{}, err + } + need := options.BlockBytes + if remaining := info.Size() - offset; int64(need) > remaining { + need = int(remaining) + } + nativeN, nativeErr := native.ReadAt(nativeBuffer[:need], offset) + virtualN, virtualErr := virtual.ReadAt(ctx, virtualBuffer[:need], offset) + if nativeN != need || virtualN != need || (nativeErr != nil && !errors.Is(nativeErr, io.EOF)) || (virtualErr != nil && !errors.Is(virtualErr, io.EOF)) { + return ShadowResult{}, fmt.Errorf("shadow read failed at offset %d: native=(%d,%v) virtual=(%d,%v)", offset, nativeN, nativeErr, virtualN, virtualErr) + } + if !bytes.Equal(nativeBuffer[:need], virtualBuffer[:need]) { + return ShadowResult{}, fmt.Errorf("shadow byte mismatch at offset %d", offset) + } + _, _ = nativeHash.Write(nativeBuffer[:need]) + _, _ = virtualHash.Write(virtualBuffer[:need]) + offset += int64(need) + } + nativeDigest := hex.EncodeToString(nativeHash.Sum(nil)) + if nativeDigest != hex.EncodeToString(virtualHash.Sum(nil)) { + return ShadowResult{}, errors.New("shadow complete SHA-256 mismatch") + } + random := rand.New(rand.NewSource(options.Seed)) + for index := 0; index < options.RandomReads && info.Size() > 0; index++ { + readOffset := random.Int63n(info.Size()) + length := 1 + random.Intn(options.BlockBytes) + if remaining := info.Size() - readOffset; int64(length) > remaining { + length = int(remaining) + } + nativeN, nativeErr := native.ReadAt(nativeBuffer[:length], readOffset) + virtualN, virtualErr := virtual.ReadAt(ctx, virtualBuffer[:length], readOffset) + if nativeN != length || virtualN != length || !bytes.Equal(nativeBuffer[:length], virtualBuffer[:length]) || (nativeErr != nil && !errors.Is(nativeErr, io.EOF)) || (virtualErr != nil && !errors.Is(virtualErr, io.EOF)) { + return ShadowResult{}, fmt.Errorf("shadow random read %d differs at offset %d", index, readOffset) + } + } + after, err := hashFile(nativePath) + if err != nil { + return ShadowResult{}, err + } + if after.Bytes != info.Size() || after.SHA256 != nativeDigest { + return ShadowResult{}, errors.New("native source changed during shadow verification") + } + return ShadowResult{NativePath: nativePath, Bytes: info.Size(), SHA256: nativeDigest, ComparedBytes: offset, RandomReads: options.RandomReads, Verified: true}, nil +} + +type fileDigest struct { + Bytes int64 + SHA256 string +} + +func hashFile(path string) (fileDigest, error) { + file, err := os.Open(path) + if err != nil { + return fileDigest{}, err + } + defer file.Close() + hasher := sha256.New() + bytesRead, err := io.Copy(hasher, file) + if err != nil { + return fileDigest{}, err + } + return fileDigest{Bytes: bytesRead, SHA256: hex.EncodeToString(hasher.Sum(nil))}, nil +} diff --git a/internal/fsctl/status.go b/internal/fsctl/status.go new file mode 100644 index 0000000..d1a7cb2 --- /dev/null +++ b/internal/fsctl/status.go @@ -0,0 +1,34 @@ +package fsctl + +import ( + "fmt" + "strings" +) + +type Capability string + +const ( + StorageEngine Capability = "storage-engine" + FSEnginePreview Capability = "fs-engine-preview" + PlatformCanary Capability = "platform-canary" + CrossPlatformReady Capability = "cross-platform-ready" +) + +type Status struct { + Capability Capability `json:"capability"` + Platform string `json:"platform"` +} + +func NewStatus(capability Capability, platform string) (Status, error) { + valid := capability == StorageEngine || capability == FSEnginePreview || capability == PlatformCanary || capability == CrossPlatformReady + if strings.HasPrefix(string(capability), "production-ready:") && strings.TrimPrefix(string(capability), "production-ready:") != "" { + valid = true + } + if !valid { + return Status{}, fmt.Errorf("non-canonical filesystem capability %q", capability) + } + if platform == "" { + return Status{}, fmt.Errorf("status platform is required") + } + return Status{Capability: capability, Platform: platform}, nil +} From 5be10d2ee294ad72742eb5f73c2f6b2c12dae747 Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 01:02:36 +0800 Subject: [PATCH 07/33] feat: add Codex compatibility and route transactions --- internal/codex/routes.go | 118 +++++++++++++++++++ internal/codex/routes_test.go | 118 +++++++++++++++++++ internal/compat/compat_test.go | 80 +++++++++++++ internal/compat/contract.go | 183 ++++++++++++++++++++++++++++++ internal/compat/fsusage.go | 73 ++++++++++++ internal/compat/version.go | 24 ++++ internal/compat/version_darwin.go | 26 +++++ internal/compat/version_other.go | 12 ++ 8 files changed, 634 insertions(+) create mode 100644 internal/codex/routes.go create mode 100644 internal/codex/routes_test.go create mode 100644 internal/compat/compat_test.go create mode 100644 internal/compat/contract.go create mode 100644 internal/compat/fsusage.go create mode 100644 internal/compat/version.go create mode 100644 internal/compat/version_darwin.go create mode 100644 internal/compat/version_other.go diff --git a/internal/codex/routes.go b/internal/codex/routes.go new file mode 100644 index 0000000..d6abc32 --- /dev/null +++ b/internal/codex/routes.go @@ -0,0 +1,118 @@ +package codex + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + _ "modernc.org/sqlite" +) + +type RouteTarget struct { + Path string `json:"path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` +} + +type RouteOptions struct { + CodexHome string + SessionID string + ExpectedPath string + Target RouteTarget +} + +type RouteResult struct { + SessionID string `json:"session_id"` + PreviousPath string `json:"previous_path"` + CurrentPath string `json:"current_path"` +} + +func RouteSession(ctx context.Context, options RouteOptions) (RouteResult, error) { + if options.CodexHome == "" || options.SessionID == "" || options.ExpectedPath == "" || options.Target.Path == "" || options.Target.Bytes < 0 || len(options.Target.SHA256) != 64 { + return RouteResult{}, errors.New("complete route options and verified target metadata are required") + } + if err := verifyRouteTarget(options.Target); err != nil { + return RouteResult{}, err + } + db, err := sql.Open("sqlite", filepath.Join(options.CodexHome, "state_5.sqlite")) + if err != nil { + return RouteResult{}, fmt.Errorf("open Codex route database: %w", err) + } + defer db.Close() + conn, err := db.Conn(ctx) + if err != nil { + return RouteResult{}, err + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, `pragma busy_timeout = 10000`); err != nil { + return RouteResult{}, err + } + if _, err := conn.ExecContext(ctx, `begin immediate`); err != nil { + return RouteResult{}, fmt.Errorf("begin immediate Codex route transaction: %w", err) + } + committed := false + defer func() { + if !committed { + _, _ = conn.ExecContext(context.Background(), `rollback`) + } + }() + var current string + if err := conn.QueryRowContext(ctx, `select rollout_path from threads where id = ?`, options.SessionID).Scan(¤t); err != nil { + return RouteResult{}, fmt.Errorf("read current Codex route: %w", err) + } + if filepath.Clean(current) != filepath.Clean(options.ExpectedPath) { + return RouteResult{}, fmt.Errorf("Codex route changed: current=%s expected=%s", current, options.ExpectedPath) + } + if err := verifyRouteTarget(options.Target); err != nil { + return RouteResult{}, fmt.Errorf("revalidate route target inside transaction: %w", err) + } + result, err := conn.ExecContext(ctx, `update threads set rollout_path = ? where id = ? and rollout_path = ?`, options.Target.Path, options.SessionID, current) + if err != nil { + return RouteResult{}, fmt.Errorf("update Codex route: %w", err) + } + rows, err := result.RowsAffected() + if err != nil { + return RouteResult{}, err + } + if rows != 1 { + return RouteResult{}, fmt.Errorf("Codex route update affected %d rows", rows) + } + if _, err := conn.ExecContext(ctx, `commit`); err != nil { + return RouteResult{}, fmt.Errorf("commit Codex route: %w", err) + } + committed = true + var confirmed string + if err := conn.QueryRowContext(ctx, `select rollout_path from threads where id = ?`, options.SessionID).Scan(&confirmed); err != nil { + return RouteResult{}, fmt.Errorf("confirm Codex route: %w", err) + } + if filepath.Clean(confirmed) != filepath.Clean(options.Target.Path) { + return RouteResult{}, errors.New("committed Codex route did not persist") + } + return RouteResult{SessionID: options.SessionID, PreviousPath: current, CurrentPath: confirmed}, nil +} + +func verifyRouteTarget(target RouteTarget) error { + file, err := os.Open(target.Path) + if err != nil { + return fmt.Errorf("open route target: %w", err) + } + hasher := sha256.New() + bytesRead, copyErr := io.Copy(hasher, file) + closeErr := file.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + if bytesRead != target.Bytes || hex.EncodeToString(hasher.Sum(nil)) != target.SHA256 { + return errors.New("route target does not match current-byte metadata") + } + return nil +} diff --git a/internal/codex/routes_test.go b/internal/codex/routes_test.go new file mode 100644 index 0000000..f269802 --- /dev/null +++ b/internal/codex/routes_test.go @@ -0,0 +1,118 @@ +package codex + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "os" + "path/filepath" + "testing" + + _ "modernc.org/sqlite" +) + +func TestRouteSessionOptimisticallyUpdatesExpectedPath(t *testing.T) { + home, nativePath := routeFixture(t) + virtualPath := filepath.Join(home, "mounted", "session.jsonl") + if err := os.MkdirAll(filepath.Dir(virtualPath), 0o700); err != nil { + t.Fatalf("create mount fixture: %v", err) + } + data := []byte("current virtual bytes\n") + if err := os.WriteFile(virtualPath, data, 0o600); err != nil { + t.Fatalf("write target: %v", err) + } + result, err := RouteSession(context.Background(), RouteOptions{CodexHome: home, SessionID: "session", ExpectedPath: nativePath, Target: RouteTarget{Path: virtualPath, Bytes: int64(len(data)), SHA256: routeDigest(data)}}) + if err != nil { + t.Fatalf("RouteSession returned error: %v", err) + } + if result.PreviousPath != nativePath || result.CurrentPath != virtualPath { + t.Fatalf("unexpected route result: %#v", result) + } + if got := queryRoute(t, home); got != virtualPath { + t.Fatalf("database route = %q, want %q", got, virtualPath) + } +} + +func TestRouteSessionRejectsConcurrentOrStaleExpectedPath(t *testing.T) { + home, nativePath := routeFixture(t) + changedPath := filepath.Join(home, "changed.jsonl") + if err := updateRoute(t, home, changedPath); err != nil { + t.Fatalf("change route: %v", err) + } + target := filepath.Join(home, "target.jsonl") + data := []byte("target") + _ = os.WriteFile(target, data, 0o600) + if _, err := RouteSession(context.Background(), RouteOptions{CodexHome: home, SessionID: "session", ExpectedPath: nativePath, Target: RouteTarget{Path: target, Bytes: int64(len(data)), SHA256: routeDigest(data)}}); err == nil { + t.Fatal("RouteSession should reject a stale expected path") + } + if got := queryRoute(t, home); got != changedPath { + t.Fatalf("rejected transaction changed route to %q", got) + } +} + +func TestRouteSessionRejectsStaleOrCorruptFallbackBytes(t *testing.T) { + home, nativePath := routeFixture(t) + target := filepath.Join(home, "fallback.jsonl") + if err := os.WriteFile(target, []byte("old snapshot"), 0o600); err != nil { + t.Fatalf("write fallback: %v", err) + } + current := []byte("latest bytes") + if _, err := RouteSession(context.Background(), RouteOptions{CodexHome: home, SessionID: "session", ExpectedPath: nativePath, Target: RouteTarget{Path: target, Bytes: int64(len(current)), SHA256: routeDigest(current)}}); err == nil { + t.Fatal("RouteSession should reject a target not equal to current-byte metadata") + } + if got := queryRoute(t, home); got != nativePath { + t.Fatalf("corrupt fallback changed route to %q", got) + } +} + +func routeFixture(t *testing.T) (string, string) { + t.Helper() + home := t.TempDir() + nativePath := filepath.Join(home, "native.jsonl") + if err := os.WriteFile(nativePath, []byte("native\n"), 0o600); err != nil { + t.Fatalf("write native: %v", err) + } + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatalf("open state: %v", err) + } + _, err = db.Exec(`create table threads (id text primary key, rollout_path text not null); insert into threads values ('session', ?)`, nativePath) + if closeErr := db.Close(); err == nil { + err = closeErr + } + if err != nil { + t.Fatalf("create state: %v", err) + } + return home, nativePath +} + +func queryRoute(t *testing.T, home string) string { + t.Helper() + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatalf("open state: %v", err) + } + defer db.Close() + var path string + if err := db.QueryRow(`select rollout_path from threads where id = 'session'`).Scan(&path); err != nil { + t.Fatalf("query route: %v", err) + } + return path +} + +func updateRoute(t *testing.T, home, path string) error { + t.Helper() + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + return err + } + defer db.Close() + _, err = db.Exec(`update threads set rollout_path = ? where id = 'session'`, path) + return err +} + +func routeDigest(data []byte) string { + digest := sha256.Sum256(data) + return hex.EncodeToString(digest[:]) +} diff --git a/internal/compat/compat_test.go b/internal/compat/compat_test.go new file mode 100644 index 0000000..ce1af99 --- /dev/null +++ b/internal/compat/compat_test.go @@ -0,0 +1,80 @@ +package compat + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestParseFSUsageProducesSanitizedOperationContract(t *testing.T) { + trace := strings.Join([]string{ + "12:00:00.000 open F=3 (R_____) /Users/example/.codex/sessions/private.jsonl codex.123", + "12:00:00.001 read F=3 B=4096 /Users/example/.codex/sessions/private.jsonl codex.123", + "12:00:00.002 fsync F=3 /Users/example/.codex/sessions/private.jsonl codex.123", + "12:00:00.003 read F=3 B=4096 /Users/example/.codex/sessions/private.jsonl codex.123", + }, "\n") + contract, err := ParseFSUsage(strings.NewReader(trace), ContractOptions{Platform: "darwin", ClientKind: "cli", ClientVersion: "0.1.0"}) + if err != nil { + t.Fatalf("ParseFSUsage returned error: %v", err) + } + if contract.TraceSHA256 == "" || len(contract.Operations) != 3 { + t.Fatalf("unexpected contract: %#v", contract) + } + encoded, err := json.Marshal(contract) + if err != nil { + t.Fatalf("marshal contract: %v", err) + } + if bytes.Contains(encoded, []byte("/Users/example")) || bytes.Contains(encoded, []byte("private.jsonl")) { + t.Fatalf("contract leaked trace paths: %s", encoded) + } + if contract.Operations[1].Name != "read" || contract.Operations[1].Count != 2 { + t.Fatalf("operation aggregation differs: %#v", contract.Operations) + } +} + +func TestEvaluateQuarantinesUnknownClientVersion(t *testing.T) { + contracts := []Contract{{Version: ContractVersion, Platform: "darwin", ClientKind: "cli", ClientVersion: "1.0.0", TraceSHA256: strings.Repeat("a", 64)}} + approved := Evaluate([]ClientVersion{{Platform: "darwin", Kind: "cli", Version: "1.0.0"}}, contracts) + if approved.Quarantine || !approved.Approved { + t.Fatalf("known version should be approved: %#v", approved) + } + unknown := Evaluate([]ClientVersion{{Platform: "darwin", Kind: "cli", Version: "1.1.0"}}, contracts) + if !unknown.Quarantine || unknown.Approved || len(unknown.Unknown) != 1 { + t.Fatalf("unknown version should quarantine: %#v", unknown) + } +} + +func TestSaveAndLoadContractRoundTrip(t *testing.T) { + root := t.TempDir() + contract := Contract{Version: ContractVersion, Platform: "darwin", ClientKind: "desktop", ClientVersion: "26.1", TraceSHA256: strings.Repeat("b", 64), Operations: []Operation{{Name: "open", Count: 1}}} + path, err := Save(root, contract) + if err != nil { + t.Fatalf("Save returned error: %v", err) + } + loaded, err := Load(path) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if loaded.ClientVersion != contract.ClientVersion || loaded.TraceSHA256 != contract.TraceSHA256 { + t.Fatalf("loaded contract differs: %#v", loaded) + } +} + +func TestDetectCLIVersionParsesCommandOutput(t *testing.T) { + root := t.TempDir() + command := filepath.Join(root, "codex-test") + if err := os.WriteFile(command, []byte("#!/bin/sh\necho 'codex-cli 9.8.7'\n"), 0o700); err != nil { + t.Fatalf("write fake CLI: %v", err) + } + version, err := DetectCLIVersion(context.Background(), command) + if err != nil { + t.Fatalf("DetectCLIVersion returned error: %v", err) + } + if version.Platform == "" || version.Kind != "cli" || version.Version != "9.8.7" { + t.Fatalf("unexpected CLI version: %#v", version) + } +} diff --git a/internal/compat/contract.go b/internal/compat/contract.go new file mode 100644 index 0000000..39c248a --- /dev/null +++ b/internal/compat/contract.go @@ -0,0 +1,183 @@ +package compat + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +const ContractVersion = 1 + +type Operation struct { + Name string `json:"name"` + Count int `json:"count"` + Signatures []Signature `json:"signatures,omitempty"` +} + +type Signature struct { + Value string `json:"value"` + Count int `json:"count"` +} + +type Contract struct { + Version int `json:"version"` + Platform string `json:"platform"` + ClientKind string `json:"client_kind"` + ClientVersion string `json:"client_version"` + Operations []Operation `json:"operations"` + TraceSHA256 string `json:"trace_sha256"` +} + +type ClientVersion struct { + Platform string `json:"platform,omitempty"` + Kind string `json:"kind"` + Version string `json:"version"` +} + +type Evaluation struct { + Approved bool `json:"approved"` + Quarantine bool `json:"quarantine"` + Unknown []ClientVersion `json:"unknown,omitempty"` +} + +func Evaluate(installed []ClientVersion, contracts []Contract) Evaluation { + known := make(map[string]struct{}, len(contracts)) + for _, contract := range contracts { + if validateContract(contract) == nil { + known[contract.Platform+"\x00"+contract.ClientKind+"\x00"+contract.ClientVersion] = struct{}{} + } + } + result := Evaluation{Approved: true} + for _, client := range installed { + key := client.Platform + "\x00" + client.Kind + "\x00" + client.Version + if _, ok := known[key]; !ok || client.Kind == "" || client.Version == "" { + result.Unknown = append(result.Unknown, client) + } + } + if len(result.Unknown) != 0 { + result.Approved = false + result.Quarantine = true + } + return result +} + +func Save(root string, contract Contract) (string, error) { + if err := validateContract(contract); err != nil { + return "", err + } + directory := filepath.Join(root, safeName(contract.Platform), safeName(contract.ClientKind)) + if err := os.MkdirAll(directory, 0o700); err != nil { + return "", err + } + path := filepath.Join(directory, safeName(contract.ClientVersion)+".json") + data, err := json.MarshalIndent(contract, "", " ") + if err != nil { + return "", err + } + data = append(data, '\n') + temporary, err := os.CreateTemp(directory, ".contract-*.tmp") + if err != nil { + return "", err + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return "", err + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return "", err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return "", err + } + if err := temporary.Close(); err != nil { + return "", err + } + if err := os.Rename(temporaryPath, path); err != nil { + return "", err + } + return path, nil +} + +func Load(path string) (Contract, error) { + data, err := os.ReadFile(path) + if err != nil { + return Contract{}, err + } + var contract Contract + if err := json.Unmarshal(data, &contract); err != nil { + return Contract{}, err + } + if err := validateContract(contract); err != nil { + return Contract{}, err + } + return contract, nil +} + +func LoadAll(root string) ([]Contract, error) { + var contracts []Contract + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { + return nil + } + contract, err := Load(path) + if err != nil { + return err + } + contracts = append(contracts, contract) + return nil + }) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + sort.Slice(contracts, func(i, j int) bool { + if contracts[i].Platform != contracts[j].Platform { + return contracts[i].Platform < contracts[j].Platform + } + if contracts[i].ClientKind != contracts[j].ClientKind { + return contracts[i].ClientKind < contracts[j].ClientKind + } + return contracts[i].ClientVersion < contracts[j].ClientVersion + }) + return contracts, err +} + +func validateContract(contract Contract) error { + if contract.Version != ContractVersion || contract.Platform == "" || contract.ClientKind == "" || contract.ClientVersion == "" || len(contract.TraceSHA256) != 64 { + return errors.New("invalid compatibility contract metadata") + } + for _, operation := range contract.Operations { + if operation.Name == "" || operation.Count <= 0 { + return errors.New("invalid compatibility operation") + } + for _, signature := range operation.Signatures { + if signature.Value == "" || signature.Count <= 0 || strings.ContainsAny(signature.Value, "/\\") { + return errors.New("invalid compatibility operation signature") + } + } + } + return nil +} + +func safeName(value string) string { + value = strings.Map(func(character rune) rune { + if character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' || character >= '0' && character <= '9' || strings.ContainsRune("._-", character) { + return character + } + return '_' + }, value) + if value == "" || value == "." || value == ".." { + return fmt.Sprintf("value-%x", []byte(value)) + } + return value +} diff --git a/internal/compat/fsusage.go b/internal/compat/fsusage.go new file mode 100644 index 0000000..a959ca3 --- /dev/null +++ b/internal/compat/fsusage.go @@ -0,0 +1,73 @@ +package compat + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "regexp" + "sort" + "strings" +) + +type ContractOptions struct { + Platform string + ClientKind string + ClientVersion string +} + +var operationPattern = regexp.MustCompile(`(?i)\b(open|openat|close|read|pread|write|pwrite|fsync|fdatasync|stat|stat64|fstat|mmap|truncate|ftruncate|rename|unlink|flock|fcntl|clonefile)\b`) +var signaturePattern = regexp.MustCompile(`\b(?:F|B|O|FLAGS)=[A-Za-z0-9_()+-]+`) + +func ParseFSUsage(reader io.Reader, options ContractOptions) (Contract, error) { + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 4096), 4<<20) + hasher := sha256.New() + counts := make(map[string]int) + signatures := make(map[string]map[string]int) + var names []string + for scanner.Scan() { + line := scanner.Bytes() + _, _ = hasher.Write(line) + _, _ = hasher.Write([]byte{'\n'}) + match := operationPattern.FindSubmatch(line) + if len(match) == 0 { + continue + } + name := strings.ToLower(string(match[1])) + if counts[name] == 0 { + names = append(names, name) + } + counts[name]++ + if signatures[name] == nil { + signatures[name] = make(map[string]int) + } + for _, signature := range signaturePattern.FindAllString(string(line), -1) { + signatures[name][signature]++ + } + } + if err := scanner.Err(); err != nil { + return Contract{}, err + } + if len(counts) == 0 { + return Contract{}, errors.New("trace contains no recognized filesystem operations") + } + contract := Contract{Version: ContractVersion, Platform: options.Platform, ClientKind: options.ClientKind, ClientVersion: options.ClientVersion, TraceSHA256: hex.EncodeToString(hasher.Sum(nil))} + for _, name := range names { + operation := Operation{Name: name, Count: counts[name]} + values := make([]string, 0, len(signatures[name])) + for value := range signatures[name] { + values = append(values, value) + } + sort.Strings(values) + for _, value := range values { + operation.Signatures = append(operation.Signatures, Signature{Value: value, Count: signatures[name][value]}) + } + contract.Operations = append(contract.Operations, operation) + } + if err := validateContract(contract); err != nil { + return Contract{}, err + } + return contract, nil +} diff --git a/internal/compat/version.go b/internal/compat/version.go new file mode 100644 index 0000000..8832204 --- /dev/null +++ b/internal/compat/version.go @@ -0,0 +1,24 @@ +package compat + +import ( + "context" + "errors" + "os/exec" + "runtime" + "strings" +) + +func DetectCLIVersion(ctx context.Context, binary string) (ClientVersion, error) { + if binary == "" { + return ClientVersion{}, errors.New("Codex CLI path is required") + } + output, err := exec.CommandContext(ctx, binary, "--version").CombinedOutput() + if err != nil { + return ClientVersion{}, err + } + fields := strings.Fields(strings.TrimSpace(string(output))) + if len(fields) < 2 { + return ClientVersion{}, errors.New("Codex CLI returned an unrecognized version") + } + return ClientVersion{Platform: runtime.GOOS, Kind: "cli", Version: fields[len(fields)-1]}, nil +} diff --git a/internal/compat/version_darwin.go b/internal/compat/version_darwin.go new file mode 100644 index 0000000..b1c670c --- /dev/null +++ b/internal/compat/version_darwin.go @@ -0,0 +1,26 @@ +//go:build darwin + +package compat + +import ( + "context" + "errors" + "os/exec" + "strings" +) + +func DetectDesktopVersion(ctx context.Context, appPath string) (ClientVersion, error) { + if appPath == "" { + return ClientVersion{}, errors.New("Codex application path is required") + } + plist := appPath + "/Contents/Info.plist" + short, err := exec.CommandContext(ctx, "/usr/bin/plutil", "-extract", "CFBundleShortVersionString", "raw", plist).Output() + if err != nil { + return ClientVersion{}, err + } + build, err := exec.CommandContext(ctx, "/usr/bin/plutil", "-extract", "CFBundleVersion", "raw", plist).Output() + if err != nil { + return ClientVersion{}, err + } + return ClientVersion{Platform: "darwin", Kind: "desktop", Version: strings.TrimSpace(string(short)) + "+" + strings.TrimSpace(string(build))}, nil +} diff --git a/internal/compat/version_other.go b/internal/compat/version_other.go new file mode 100644 index 0000000..efd36db --- /dev/null +++ b/internal/compat/version_other.go @@ -0,0 +1,12 @@ +//go:build !darwin + +package compat + +import ( + "context" + "errors" +) + +func DetectDesktopVersion(context.Context, string) (ClientVersion, error) { + return ClientVersion{}, errors.New("desktop version detection is not implemented on this platform") +} From 3f51aa53edc1c2d86fec2a0a49baddbe75b152ce Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 01:12:43 +0800 Subject: [PATCH 08/33] feat: add platform filesystem and tagged fuse host --- go.mod | 1 + go.sum | 2 + internal/mountfs/filesystem.go | 308 ++++++++++++++++++++++++++++ internal/mountfs/filesystem_test.go | 147 +++++++++++++ internal/mountfs/host.go | 21 ++ internal/mountfs/host_cgofuse.go | 163 +++++++++++++++ internal/mountfs/host_stub.go | 7 + internal/vfs/session.go | 32 ++- 8 files changed, 679 insertions(+), 2 deletions(-) create mode 100644 internal/mountfs/filesystem.go create mode 100644 internal/mountfs/filesystem_test.go create mode 100644 internal/mountfs/host.go create mode 100644 internal/mountfs/host_cgofuse.go create mode 100644 internal/mountfs/host_stub.go diff --git a/go.mod b/go.mod index 62e0f28..365be0e 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.26 require ( github.com/klauspost/compress v1.19.0 github.com/spf13/cobra v1.10.2 + github.com/winfsp/cgofuse v1.6.0 golang.org/x/sys v0.36.0 modernc.org/sqlite v1.40.1 ) diff --git a/go.sum b/go.sum index 1db7c0c..f642994 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/winfsp/cgofuse v1.6.0 h1:re3W+HTd0hj4fISPBqfsrwyvPFpzqhDu8doJ9nOPDB0= +github.com/winfsp/cgofuse v1.6.0/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= diff --git a/internal/mountfs/filesystem.go b/internal/mountfs/filesystem.go new file mode 100644 index 0000000..0715b50 --- /dev/null +++ b/internal/mountfs/filesystem.go @@ -0,0 +1,308 @@ +package mountfs + +import ( + "context" + "errors" + "io" + "os" + "path" + "sort" + "strings" + "sync" + "syscall" + "time" + + "github.com/jstar0/codexfold/internal/vfs" +) + +type Attr struct { + Mode uint32 `json:"mode"` + Size int64 `json:"size"` + ModTime time.Time `json:"mod_time"` +} + +type fileHandle struct { + mu sync.Mutex + session *vfs.Session + read *vfs.ReadHandle + write *vfs.WriteHandle + append bool +} + +type Filesystem struct { + mu sync.RWMutex + sessions map[string]*vfs.Session + handles map[uint64]*fileHandle + next uint64 +} + +func New() *Filesystem { + return &Filesystem{sessions: make(map[string]*vfs.Session), handles: make(map[uint64]*fileHandle), next: 1} +} + +func (f *Filesystem) AddSession(sessionID string, session *vfs.Session) error { + if sessionID == "" || strings.ContainsAny(sessionID, "/\\\x00") || session == nil { + return errors.New("safe session ID and session are required") + } + f.mu.Lock() + defer f.mu.Unlock() + if _, exists := f.sessions[sessionID]; exists { + return errors.New("session is already mounted") + } + f.sessions[sessionID] = session + return nil +} + +func (f *Filesystem) ReadDir(name string) ([]string, syscall.Errno) { + if cleanPath(name) != "/" { + return nil, syscall.ENOTDIR + } + f.mu.RLock() + entries := make([]string, 0, len(f.sessions)) + for sessionID := range f.sessions { + entries = append(entries, sessionID+".jsonl") + } + f.mu.RUnlock() + sort.Strings(entries) + return entries, 0 +} + +func (f *Filesystem) Getattr(name string) (Attr, syscall.Errno) { + if cleanPath(name) == "/" { + return Attr{Mode: syscall.S_IFDIR | 0o700}, 0 + } + session, errno := f.sessionForPath(name) + if errno != 0 { + return Attr{}, errno + } + info, err := session.VisibleInfo() + if err != nil { + return Attr{}, errnoFor(err) + } + return Attr{Mode: syscall.S_IFREG | 0o600, Size: info.Size, ModTime: info.ModTime}, 0 +} + +func (f *Filesystem) Open(name string, flags int) (uint64, syscall.Errno) { + session, errno := f.sessionForPath(name) + if errno != 0 { + return 0, errno + } + handle := &fileHandle{session: session, append: flags&os.O_APPEND != 0} + access := flags & (os.O_WRONLY | os.O_RDWR) + if access != os.O_WRONLY { + reader, err := session.OpenReader() + if err != nil { + return 0, errnoFor(err) + } + handle.read = reader + } + if access == os.O_WRONLY || access == os.O_RDWR { + writer, err := session.OpenWriter() + if err != nil { + if handle.read != nil { + _ = handle.read.Close() + } + return 0, errnoFor(err) + } + handle.write = writer + if flags&os.O_TRUNC != 0 { + if err := writer.Truncate(context.Background(), 0); err != nil { + _ = writer.Close() + if handle.read != nil { + _ = handle.read.Close() + } + return 0, errnoFor(err) + } + } + } + f.mu.Lock() + handleID := f.next + f.next++ + f.handles[handleID] = handle + f.mu.Unlock() + return handleID, 0 +} + +func (f *Filesystem) Read(handleID uint64, destination []byte, offset int64) (int, syscall.Errno) { + handle, errno := f.handle(handleID) + if errno != 0 || handle.read == nil { + return 0, syscall.EBADF + } + handle.mu.Lock() + defer handle.mu.Unlock() + n, err := handle.read.ReadAt(context.Background(), destination, offset) + if err != nil && !errors.Is(err, io.EOF) { + return n, errnoFor(err) + } + return n, 0 +} + +func (f *Filesystem) Write(handleID uint64, data []byte, offset int64) (int, syscall.Errno) { + handle, errno := f.handle(handleID) + if errno != 0 || handle.write == nil { + return 0, syscall.EBADF + } + handle.mu.Lock() + defer handle.mu.Unlock() + var n int + var err error + if handle.append { + n, err = handle.write.Append(context.Background(), data) + } else { + n, err = handle.write.WriteAt(context.Background(), data, offset) + } + if err != nil { + return n, errnoFor(err) + } + if handle.read != nil { + if errno := refreshReader(handle); errno != 0 { + return n, errno + } + } + return n, 0 +} + +func (f *Filesystem) Truncate(handleID uint64, size int64) syscall.Errno { + handle, errno := f.handle(handleID) + if errno != 0 || handle.write == nil { + return syscall.EBADF + } + handle.mu.Lock() + defer handle.mu.Unlock() + if err := handle.write.Truncate(context.Background(), size); err != nil { + return errnoFor(err) + } + if handle.read != nil { + return refreshReader(handle) + } + return 0 +} + +func (f *Filesystem) TruncatePath(name string, size int64) syscall.Errno { + session, errno := f.sessionForPath(name) + if errno != 0 { + return errno + } + writer, err := session.OpenWriter() + if err != nil { + return errnoFor(err) + } + truncateErr := writer.Truncate(context.Background(), size) + closeErr := writer.Close() + if truncateErr != nil { + return errnoFor(truncateErr) + } + return errnoFor(closeErr) +} + +func (f *Filesystem) Fsync(handleID uint64) syscall.Errno { + handle, errno := f.handle(handleID) + if errno != 0 { + return errno + } + handle.mu.Lock() + defer handle.mu.Unlock() + if handle.write == nil { + return 0 + } + return errnoFor(handle.write.Sync()) +} + +func (f *Filesystem) Flush(handleID uint64) syscall.Errno { + _, errno := f.handle(handleID) + return errno +} + +func (f *Filesystem) Release(handleID uint64) syscall.Errno { + f.mu.Lock() + handle, ok := f.handles[handleID] + if ok { + delete(f.handles, handleID) + } + f.mu.Unlock() + if !ok { + return syscall.EBADF + } + handle.mu.Lock() + defer handle.mu.Unlock() + var result syscall.Errno + if handle.read != nil { + if err := handle.read.Close(); err != nil { + result = errnoFor(err) + } + } + if handle.write != nil { + if err := handle.write.Close(); err != nil && result == 0 { + result = errnoFor(err) + } + } + return result +} + +func refreshReader(handle *fileHandle) syscall.Errno { + reader, err := handle.session.OpenReader() + if err != nil { + return errnoFor(err) + } + old := handle.read + handle.read = reader + if old != nil { + if err := old.Close(); err != nil { + return errnoFor(err) + } + } + return 0 +} + +func (f *Filesystem) Rename(string, string) syscall.Errno { return syscall.EPERM } +func (f *Filesystem) Unlink(string) syscall.Errno { return syscall.EPERM } + +func (f *Filesystem) sessionForPath(name string) (*vfs.Session, syscall.Errno) { + cleaned := cleanPath(name) + if cleaned == "/" || strings.Count(cleaned, "/") != 1 || !strings.HasSuffix(cleaned, ".jsonl") { + return nil, syscall.ENOENT + } + sessionID := strings.TrimSuffix(strings.TrimPrefix(cleaned, "/"), ".jsonl") + f.mu.RLock() + session := f.sessions[sessionID] + f.mu.RUnlock() + if session == nil { + return nil, syscall.ENOENT + } + return session, 0 +} + +func (f *Filesystem) handle(handleID uint64) (*fileHandle, syscall.Errno) { + f.mu.RLock() + handle := f.handles[handleID] + f.mu.RUnlock() + if handle == nil { + return nil, syscall.EBADF + } + return handle, 0 +} + +func cleanPath(name string) string { + if name == "" || strings.ContainsRune(name, '\x00') || strings.Contains(name, "..") { + return "" + } + return path.Clean("/" + strings.TrimPrefix(name, "/")) +} + +func errnoFor(err error) syscall.Errno { + if err == nil { + return 0 + } + switch { + case errors.Is(err, vfs.ErrWriterBusy): + return syscall.EBUSY + case errors.Is(err, os.ErrNotExist): + return syscall.ENOENT + case errors.Is(err, os.ErrPermission): + return syscall.EACCES + case errors.Is(err, context.Canceled): + return syscall.EINTR + default: + return syscall.EIO + } +} diff --git a/internal/mountfs/filesystem_test.go b/internal/mountfs/filesystem_test.go new file mode 100644 index 0000000..3e92e55 --- /dev/null +++ b/internal/mountfs/filesystem_test.go @@ -0,0 +1,147 @@ +package mountfs + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "syscall" + "testing" + + "github.com/jstar0/codexfold/internal/fold" + "github.com/jstar0/codexfold/internal/vfs" +) + +func TestFilesystemListsStatsReadsAndAppendsSession(t *testing.T) { + filesystem, source := mountFixture(t) + entries, errno := filesystem.ReadDir("/") + if errno != 0 || len(entries) != 1 || entries[0] != "session.jsonl" { + t.Fatalf("ReadDir = %#v errno=%v", entries, errno) + } + attribute, errno := filesystem.Getattr("/session.jsonl") + if errno != 0 || attribute.Mode&syscall.S_IFREG == 0 || attribute.Size != int64(len(source)) { + t.Fatalf("Getattr = %#v errno=%v", attribute, errno) + } + + readHandle, errno := filesystem.Open("/session.jsonl", os.O_RDONLY) + if errno != 0 { + t.Fatalf("Open read errno=%v", errno) + } + buffer := make([]byte, len(source)) + n, errno := filesystem.Read(readHandle, buffer, 0) + if errno != 0 || !bytes.Equal(buffer[:n], source) { + t.Fatalf("Read = %d errno=%v bytes=%q", n, errno, buffer[:n]) + } + if errno := filesystem.Release(readHandle); errno != 0 { + t.Fatalf("Release read errno=%v", errno) + } + + writeHandle, errno := filesystem.Open("/session.jsonl", os.O_WRONLY|os.O_APPEND) + if errno != 0 { + t.Fatalf("Open append errno=%v", errno) + } + if n, errno := filesystem.Write(writeHandle, []byte("-tail"), 0); errno != 0 || n != 5 { + t.Fatalf("Write append = %d errno=%v", n, errno) + } + if errno := filesystem.Fsync(writeHandle); errno != 0 { + t.Fatalf("Fsync errno=%v", errno) + } + _ = filesystem.Release(writeHandle) + + newHandle, errno := filesystem.Open("/session.jsonl", os.O_RDONLY) + if errno != 0 { + t.Fatalf("Open current errno=%v", errno) + } + current := make([]byte, len(source)+5) + n, errno = filesystem.Read(newHandle, current, 0) + if errno != 0 || !bytes.Equal(current[:n], append(append([]byte(nil), source...), []byte("-tail")...)) { + t.Fatalf("current read differs: n=%d errno=%v bytes=%q", n, errno, current[:n]) + } + _ = filesystem.Release(newHandle) +} + +func TestFilesystemRandomWriteTruncateAndWriterExclusion(t *testing.T) { + filesystem, source := mountFixture(t) + first, errno := filesystem.Open("/session.jsonl", os.O_RDWR) + if errno != 0 { + t.Fatalf("Open first writer errno=%v", errno) + } + if _, errno := filesystem.Open("/session.jsonl", os.O_WRONLY); errno != syscall.EBUSY { + t.Fatalf("second writer errno=%v, want EBUSY", errno) + } + if n, errno := filesystem.Write(first, []byte("PATCH"), 2); errno != 0 || n != 5 { + t.Fatalf("random Write = %d errno=%v", n, errno) + } + current := make([]byte, len(source)) + if n, errno := filesystem.Read(first, current, 0); errno != 0 || n != len(source) || string(current[2:7]) != "PATCH" { + t.Fatalf("read-after-write = %d errno=%v bytes=%q", n, errno, current) + } + if errno := filesystem.Truncate(first, int64(len(source)-3)); errno != 0 { + t.Fatalf("Truncate errno=%v", errno) + } + _ = filesystem.Release(first) + attribute, errno := filesystem.Getattr("/session.jsonl") + if errno != 0 || attribute.Size != int64(len(source)-3) { + t.Fatalf("truncated attribute=%#v errno=%v", attribute, errno) + } +} + +func TestFilesystemRejectsUnsafeAndManagementMutations(t *testing.T) { + filesystem, _ := mountFixture(t) + if _, errno := filesystem.Open("/../session.jsonl", os.O_RDONLY); errno != syscall.ENOENT { + t.Fatalf("unsafe path errno=%v", errno) + } + if errno := filesystem.Rename("/session.jsonl", "/other.jsonl"); errno != syscall.EPERM { + t.Fatalf("Rename errno=%v, want EPERM", errno) + } + if errno := filesystem.Unlink("/session.jsonl"); errno != syscall.EPERM { + t.Fatalf("Unlink errno=%v, want EPERM", errno) + } +} + +func TestMountWithoutFuseBuildReturnsPrerequisiteError(t *testing.T) { + err := Mount(context.Background(), HostOptions{MountPoint: t.TempDir(), Filesystem: New()}) + if !errors.Is(err, ErrPrerequisite) { + t.Fatalf("Mount error = %v, want ErrPrerequisite", err) + } +} + +type mountReader map[string][]byte + +func (r mountReader) ReadAt(_ context.Context, ref fold.ObjectRef, destination []byte, offset int64) (int, error) { + data := r[ref.SHA256] + if offset >= int64(len(data)) { + return 0, io.EOF + } + n := copy(destination, data[offset:]) + if n < len(destination) { + return n, io.EOF + } + return n, nil +} + +func mountFixture(t *testing.T) (*Filesystem, []byte) { + t.Helper() + root := t.TempDir() + source := []byte("first\nsecond\nthird\n") + digest := sha256.Sum256(source) + hexDigest := hex.EncodeToString(digest[:]) + nativePath := filepath.Join(root, "native.jsonl") + if err := os.WriteFile(nativePath, source, 0o600); err != nil { + t.Fatalf("write native: %v", err) + } + manifest := fold.Manifest{Version: fold.ManifestVersion, Kind: fold.ManifestKind, Session: fold.ManifestSession{ID: "session", RolloutPath: nativePath}, Source: fold.ManifestSource{Bytes: int64(len(source)), SHA256: hexDigest}, Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: hexDigest, RawBytes: int64(len(source))}}}} + session, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, Reader: mountReader{hexDigest: source}, NativeSnapshot: vfs.NativeFile{Path: nativePath, Bytes: int64(len(source)), SHA256: hexDigest}}) + if err != nil { + t.Fatalf("OpenSession: %v", err) + } + filesystem := New() + if err := filesystem.AddSession("session", session); err != nil { + t.Fatalf("AddSession: %v", err) + } + return filesystem, source +} diff --git a/internal/mountfs/host.go b/internal/mountfs/host.go new file mode 100644 index 0000000..2faaf71 --- /dev/null +++ b/internal/mountfs/host.go @@ -0,0 +1,21 @@ +package mountfs + +import ( + "context" + "errors" +) + +var ErrPrerequisite = errors.New("FUSE host prerequisite is unavailable in this build") + +type HostOptions struct { + MountPoint string + Filesystem *Filesystem + Foreground bool +} + +func Mount(ctx context.Context, options HostOptions) error { + if options.MountPoint == "" || options.Filesystem == nil { + return errors.New("mount point and filesystem are required") + } + return mountHost(ctx, options) +} diff --git a/internal/mountfs/host_cgofuse.go b/internal/mountfs/host_cgofuse.go new file mode 100644 index 0000000..e596b4e --- /dev/null +++ b/internal/mountfs/host_cgofuse.go @@ -0,0 +1,163 @@ +//go:build fuse && cgo + +package mountfs + +import ( + "context" + "errors" + "fmt" + "os" + "runtime" + "syscall" + + "github.com/winfsp/cgofuse/fuse" +) + +type fuseFilesystem struct { + fuse.FileSystemBase + core *Filesystem +} + +func (f *fuseFilesystem) Getattr(name string, stat *fuse.Stat_t, _ uint64) int { + attribute, errno := f.core.Getattr(name) + if errno != 0 { + return -int(errno) + } + stat.Mode = attribute.Mode + stat.Size = attribute.Size + stat.Nlink = 1 + stat.Blksize = 4096 + stat.Blocks = (attribute.Size + 511) / 512 + stat.Mtim = fuse.NewTimespec(attribute.ModTime) + stat.Ctim = stat.Mtim + stat.Atim = stat.Mtim + stat.Uid, stat.Gid, _ = fuse.Getcontext() + return 0 +} + +func (f *fuseFilesystem) Opendir(name string) (int, uint64) { + if _, errno := f.core.ReadDir(name); errno != 0 { + return -int(errno), ^uint64(0) + } + return 0, 0 +} + +func (f *fuseFilesystem) Readdir(name string, fill func(string, *fuse.Stat_t, int64) bool, _ int64, _ uint64) int { + entries, errno := f.core.ReadDir(name) + if errno != 0 { + return -int(errno) + } + fill(".", nil, 0) + fill("..", nil, 0) + for _, entry := range entries { + if !fill(entry, nil, 0) { + break + } + } + return 0 +} + +func (f *fuseFilesystem) Open(name string, flags int) (int, uint64) { + handle, errno := f.core.Open(name, translateOpenFlags(flags)) + if errno != 0 { + return -int(errno), ^uint64(0) + } + return 0, handle +} + +func (f *fuseFilesystem) Read(_ string, destination []byte, offset int64, handle uint64) int { + n, errno := f.core.Read(handle, destination, offset) + if errno != 0 { + return -int(errno) + } + return n +} + +func (f *fuseFilesystem) Write(_ string, data []byte, offset int64, handle uint64) int { + n, errno := f.core.Write(handle, data, offset) + if errno != 0 { + return -int(errno) + } + return n +} + +func (f *fuseFilesystem) Truncate(name string, size int64, handle uint64) int { + var errno syscall.Errno + if handle == 0 || handle == ^uint64(0) { + errno = f.core.TruncatePath(name, size) + } else { + errno = f.core.Truncate(handle, size) + } + return -int(errno) +} + +func (f *fuseFilesystem) Flush(_ string, handle uint64) int { + return -int(f.core.Flush(handle)) +} + +func (f *fuseFilesystem) Fsync(_ string, _ bool, handle uint64) int { + return -int(f.core.Fsync(handle)) +} + +func (f *fuseFilesystem) Release(_ string, handle uint64) int { + return -int(f.core.Release(handle)) +} + +func (f *fuseFilesystem) Rename(oldName string, newName string) int { + return -int(f.core.Rename(oldName, newName)) +} + +func (f *fuseFilesystem) Unlink(name string) int { return -int(f.core.Unlink(name)) } + +func (f *fuseFilesystem) Access(name string, _ uint32) int { + _, errno := f.core.Getattr(name) + return -int(errno) +} + +func translateOpenFlags(flags int) int { + translated := os.O_RDONLY + switch flags & fuse.O_ACCMODE { + case fuse.O_WRONLY: + translated = os.O_WRONLY + case fuse.O_RDWR: + translated = os.O_RDWR + } + if flags&fuse.O_APPEND != 0 { + translated |= os.O_APPEND + } + if flags&fuse.O_TRUNC != 0 { + translated |= os.O_TRUNC + } + return translated +} + +func mountHost(ctx context.Context, options HostOptions) (result error) { + defer func() { + if recovered := recover(); recovered != nil { + result = fmt.Errorf("%w: %v", ErrPrerequisite, recovered) + } + }() + filesystem := &fuseFilesystem{core: options.Filesystem} + host := fuse.NewFileSystemHost(filesystem) + arguments := []string{"-o", "fsname=codexfold", "-o", "default_permissions", "-o", "attr_timeout=0", "-o", "entry_timeout=0", "-o", "negative_timeout=0"} + if options.Foreground { + arguments = append(arguments, "-f") + } + if runtime.GOOS == "darwin" { + arguments = append(arguments, "-o", "volname=CodexFold") + } + done := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + _ = host.Unmount() + case <-done: + } + }() + mounted := host.Mount(options.MountPoint, arguments) + close(done) + if !mounted { + return errors.New("FUSE host exited without mounting") + } + return ctx.Err() +} diff --git a/internal/mountfs/host_stub.go b/internal/mountfs/host_stub.go new file mode 100644 index 0000000..6006738 --- /dev/null +++ b/internal/mountfs/host_stub.go @@ -0,0 +1,7 @@ +//go:build !fuse || !cgo + +package mountfs + +import "context" + +func mountHost(context.Context, HostOptions) error { return ErrPrerequisite } diff --git a/internal/vfs/session.go b/internal/vfs/session.go index 3af5d7b..a3a5e93 100644 --- a/internal/vfs/session.go +++ b/internal/vfs/session.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "sync" + "time" "github.com/jstar0/codexfold/internal/fold" ) @@ -34,6 +35,14 @@ type Session struct { beforeCOWPhase func(string) error } +type VisibleInfo struct { + Size int64 + ModTime time.Time + Generation uint64 +} + +var ErrWriterBusy = errors.New("session writer lease is already held") + func OpenSession(ctx context.Context, options SessionOptions) (*Session, error) { if err := ctx.Err(); err != nil { return nil, err @@ -108,6 +117,25 @@ func (s *Session) State() SessionState { return s.state } +func (s *Session) VisibleInfo() (VisibleInfo, error) { + s.mu.Lock() + state := s.state + s.mu.Unlock() + path := state.DeltaPath + if state.BackingPath != "" { + path = state.BackingPath + } + info, err := os.Stat(path) + if err != nil { + return VisibleInfo{}, err + } + size := info.Size() + if state.BackingPath == "" { + size += state.BaseBytes + } + return VisibleInfo{Size: size, ModTime: info.ModTime(), Generation: state.Generation}, nil +} + func (s *Session) OpenReader() (*ReadHandle, error) { s.mu.Lock() state := s.state @@ -158,7 +186,7 @@ func (s *Session) OpenWriter() (*WriteHandle, error) { s.mu.Lock() defer s.mu.Unlock() if s.writerOpen { - return nil, errors.New("session writer lease is already held") + return nil, ErrWriterBusy } leasePath := filepath.Join(s.directory, "writer.lease") lease, err := os.OpenFile(leasePath, os.O_CREATE|os.O_RDWR, 0o600) @@ -172,7 +200,7 @@ func (s *Session) OpenWriter() (*WriteHandle, error) { } if !locked { _ = lease.Close() - return nil, errors.New("session writer lease is held by another process") + return nil, ErrWriterBusy } if err := lease.Truncate(0); err != nil { _ = unlockWriterFile(lease) From 3352b87e7cc5cc33663b4d5181c39ad0fca0138d Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 01:36:21 +0800 Subject: [PATCH 09/33] feat: expose transparent filesystem command surface --- internal/cli/fs.go | 894 +++++++++++++++++++++++++++ internal/cli/fs_test.go | 352 +++++++++++ internal/cli/pack.go | 79 +++ internal/cli/root.go | 2 + internal/fold/doctor.go | 46 +- internal/fold/doctor_gc_test.go | 21 + internal/fold/fold.go | 31 +- internal/fold/fold_test.go | 32 + internal/fold/gc.go | 4 +- internal/fold/manifest.go | 13 +- internal/mountfs/filesystem.go | 10 + internal/mountfs/filesystem_test.go | 47 ++ internal/pack/build.go | 22 +- internal/pack/pack_test.go | 37 ++ internal/vfs/state.go | 40 ++ internal/vfs/state_discovery_test.go | 55 ++ 16 files changed, 1642 insertions(+), 43 deletions(-) create mode 100644 internal/cli/fs.go create mode 100644 internal/cli/fs_test.go create mode 100644 internal/cli/pack.go create mode 100644 internal/vfs/state_discovery_test.go diff --git a/internal/cli/fs.go b/internal/cli/fs.go new file mode 100644 index 0000000..7ae1a9f --- /dev/null +++ b/internal/cli/fs.go @@ -0,0 +1,894 @@ +package cli + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/jstar0/codexfold/internal/cdc" + "github.com/jstar0/codexfold/internal/codex" + "github.com/jstar0/codexfold/internal/compat" + "github.com/jstar0/codexfold/internal/fold" + "github.com/jstar0/codexfold/internal/fsctl" + "github.com/jstar0/codexfold/internal/mountfs" + "github.com/jstar0/codexfold/internal/pack" + "github.com/jstar0/codexfold/internal/vfs" + "github.com/spf13/cobra" +) + +type FSMigrateResult struct { + SessionID string `json:"session_id"` + Native vfs.NativeFile `json:"native"` + Target string `json:"target"` + Shadow fsctl.ShadowResult `json:"shadow"` + DryRun bool `json:"dry_run"` + Routed bool `json:"routed"` +} + +type FSCompatibilityResult struct { + Installed []compat.ClientVersion `json:"installed,omitempty"` + Contracts int `json:"contracts"` + DetectionErrors []string `json:"detection_errors,omitempty"` + Evaluation compat.Evaluation `json:"evaluation"` +} + +type FSServeResult struct { + MountPoint string `json:"mount_point"` + ManagedSessions int `json:"managed_sessions"` + DryRun bool `json:"dry_run"` +} + +type FSRollbackResult struct { + SessionID string `json:"session_id"` + From string `json:"from"` + Target vfs.NativeFile `json:"target"` + DryRun bool `json:"dry_run"` + Routed bool `json:"routed"` +} + +type FSCompactResult struct { + SessionID string `json:"session_id"` + CurrentGeneration uint64 `json:"current_generation"` + NextGeneration uint64 `json:"next_generation"` + Bytes int64 `json:"bytes,omitempty"` + SHA256 string `json:"sha256,omitempty"` + DryRun bool `json:"dry_run"` +} + +type FSRecoverResult struct { + SessionIDs []string `json:"session_ids"` + Recovered int `json:"recovered"` + DryRun bool `json:"dry_run"` +} + +type compatibilityFlags struct { + contractsPath string + cliPath string + desktopPath string +} + +func newFSCommand() *cobra.Command { + command := &cobra.Command{Use: "fs", Short: "Operate the transparent session filesystem"} + command.AddCommand(newFSStatusCommand()) + command.AddCommand(newFSDoctorCommand()) + command.AddCommand(newFSCompatibilityCommand()) + command.AddCommand(newFSBenchmarkCommand()) + command.AddCommand(newFSServeCommand()) + command.AddCommand(newFSMigrateCommand()) + command.AddCommand(newFSRollbackCommand()) + command.AddCommand(newFSCompactCommand()) + command.AddCommand(newFSRecoverCommand()) + return command +} + +func newFSStatusCommand() *cobra.Command { + var jsonOutput bool + command := &cobra.Command{ + Use: "status", + Short: "Report the highest verified filesystem capability", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + status, err := fsctl.NewStatus(fsctl.StorageEngine, runtime.GOOS) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, status) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "capability=%s platform=%s\n", status.Capability, status.Platform) + return err + }, + } + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSDoctorCommand() *cobra.Command { + var codexHome string + var storeDir string + var mountPoint string + var jsonOutput bool + command := &cobra.Command{ + Use: "doctor", + Short: "Verify filesystem storage, state, route, client, daemon, and mount components", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) + mount := defaultMountPoint(home, mountPoint) + report := fsDoctor(command.Context(), home, store, mount) + if jsonOutput { + return writeJSON(command, report) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "healthy=%t issues=%d daemon=%t mount=%t pack=%t manifest=%t\n", report.Healthy, report.IssueCount, report.ComponentHealth[fsctl.ComponentDaemon], report.ComponentHealth[fsctl.ComponentMount], report.ComponentHealth[fsctl.ComponentPack], report.ComponentHealth[fsctl.ComponentManifest]) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSCompatibilityCommand() *cobra.Command { + var codexHome string + var storeDir string + var flags compatibilityFlags + var jsonOutput bool + command := &cobra.Command{ + Use: "compatibility", + Short: "Evaluate installed Codex clients against exact-version contracts", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + result, err := evaluateCompatibility(command.Context(), resolveFoldStore(home, storeDir), flags) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "approved=%t quarantine=%t installed=%d contracts=%d detection_errors=%d\n", result.Evaluation.Approved, result.Evaluation.Quarantine, len(result.Installed), result.Contracts, len(result.DetectionErrors)) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + addCompatibilityFlags(command, &flags) + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSBenchmarkCommand() *cobra.Command { + var codexHome string + var storeDir string + var jsonOutput bool + var options fsctl.BenchmarkOptions + command := &cobra.Command{ + Use: "benchmark ", + Short: "Compare native and packed virtual reads without changing routes", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) + session, manifest, resolver, view, err := openFoldView(home, store, args[0]) + if err != nil { + return err + } + defer resolver.Close() + if manifest.Source.SHA256 == "" { + return errors.New("manifest source digest is missing") + } + report, err := fsctl.Benchmark(command.Context(), session.RolloutPath, view, options) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, report) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "native=%.0fB/s virtual=%.0fB/s random_p95=%s go_sys=%s\n", report.Native.BytesPerSecond, report.Virtual.BytesPerSecond, report.Random.P95, formatBytes(int64(report.GoSysBytes))) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().IntVar(&options.SequentialBlockBytes, "sequential-block-bytes", 0, "Sequential read block size") + command.Flags().IntVar(&options.RandomBlockBytes, "random-block-bytes", 0, "Random read block size") + command.Flags().IntVar(&options.RandomReads, "random-reads", 0, "Random read count") + command.Flags().Int64Var(&options.Seed, "seed", 1, "Deterministic random seed") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSServeCommand() *cobra.Command { + var codexHome string + var storeDir string + var mountPoint string + var apply bool + var foreground bool + var jsonOutput bool + command := &cobra.Command{ + Use: "serve", + Short: "Mount managed sessions and hot-load newly enrolled state", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) + mount := defaultMountPoint(home, mountPoint) + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return err + } + result := FSServeResult{MountPoint: mount, ManagedSessions: len(states), DryRun: !apply} + if !apply { + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "dry_run=true mount=%s sessions=%d\n", mount, len(states)) + return err + } + if err := os.MkdirAll(mount, 0o700); err != nil { + return err + } + filesystem := mountfs.New() + ctx, cancel := context.WithCancel(command.Context()) + defer cancel() + closers := make([]io.Closer, 0) + known := make(map[string]uint64) + load := func() error { + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return err + } + for _, state := range states { + if known[state.SessionID] == state.Generation { + continue + } + managed, resolver, err := openManagedSession(ctx, store, state) + if err != nil { + return err + } + closers = append(closers, resolver) + if err := filesystem.UpsertSession(state.SessionID, managed); err != nil { + return err + } + known[state.SessionID] = state.Generation + } + return nil + } + if err := load(); err != nil { + return err + } + watcherDone := make(chan struct{}) + watcherErrors := make(chan error, 1) + go func() { + defer close(watcherDone) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := load(); err != nil { + watcherErrors <- err + cancel() + return + } + } + } + }() + mountErr := mountfs.Mount(ctx, mountfs.HostOptions{MountPoint: mount, Filesystem: filesystem, Foreground: foreground}) + cancel() + <-watcherDone + for _, closer := range closers { + _ = closer.Close() + } + select { + case watcherErr := <-watcherErrors: + return watcherErr + default: + return mountErr + } + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + command.Flags().BoolVar(&apply, "apply", false, "Start the filesystem host") + command.Flags().BoolVar(&foreground, "foreground", true, "Keep the FUSE host in the foreground") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output for dry-run") + return command +} + +func newFSMigrateCommand() *cobra.Command { + var codexHome string + var storeDir string + var mountPoint string + var mountWait time.Duration + var apply bool + var jsonOutput bool + var compatibility compatibilityFlags + command := &cobra.Command{ + Use: "migrate ", + Short: "Shadow and optionally route an eligible session to the mounted filesystem", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) + session, manifest, resolver, view, err := openFoldView(home, store, args[0]) + if err != nil { + return err + } + defer resolver.Close() + if !session.Archived { + return errors.New("only archived sessions are eligible for filesystem migration") + } + shadow, err := fsctl.Shadow(command.Context(), session.RolloutPath, view, fsctl.ShadowOptions{RandomReads: 10000, Seed: 1}) + if err != nil { + return err + } + mount := defaultMountPoint(home, mountPoint) + target := filepath.Join(mount, session.ID+".jsonl") + native := vfs.NativeFile{Path: session.RolloutPath, Bytes: shadow.Bytes, SHA256: shadow.SHA256} + result := FSMigrateResult{SessionID: session.ID, Native: native, Target: target, Shadow: shadow, DryRun: !apply} + if apply { + if err := requireStorageHealth(command.Context(), store); err != nil { + return err + } + compatibilityResult, err := evaluateCompatibility(command.Context(), store, compatibility) + if err != nil { + return err + } + if len(compatibilityResult.DetectionErrors) != 0 || !compatibilityResult.Evaluation.Approved { + return errors.New("installed Codex client versions are not covered by compatibility contracts") + } + if info, err := os.Stat(mount); err != nil || !info.IsDir() { + return errors.New("filesystem mount point is not available") + } + if _, err := vfs.OpenSession(command.Context(), vfs.SessionOptions{Root: store, ManifestPath: fold.ManifestPath(store, session.ID), Manifest: manifest, Reader: resolver, NativeSnapshot: native}); err != nil { + return err + } + targetFile, err := waitForTarget(command.Context(), target, mountWait) + if err != nil { + return fmt.Errorf("verify mounted target: %w", err) + } + if targetFile.Bytes != shadow.Bytes || targetFile.SHA256 != shadow.SHA256 { + return errors.New("mounted target differs from the shadow-verified native session") + } + if _, err := codex.RouteSession(command.Context(), codex.RouteOptions{CodexHome: home, SessionID: session.ID, ExpectedPath: session.RolloutPath, Target: codex.RouteTarget{Path: target, Bytes: targetFile.Bytes, SHA256: targetFile.SHA256}}); err != nil { + return err + } + result.Routed = true + result.DryRun = false + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "session=%s shadow=%t dry_run=%t routed=%t target=%s\n", result.SessionID, result.Shadow.Verified, result.DryRun, result.Routed, result.Target) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + command.Flags().DurationVar(&mountWait, "mount-wait", 5*time.Second, "Maximum wait for the mounted session target") + command.Flags().BoolVar(&apply, "apply", false, "Enroll and route the session after all gates pass") + addCompatibilityFlags(command, &compatibility) + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSRollbackCommand() *cobra.Command { + var codexHome string + var storeDir string + var targetPath string + var apply bool + var jsonOutput bool + command := &cobra.Command{ + Use: "rollback ", + Short: "Route a managed session to a verified native file containing its latest visible bytes", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) + state, err := managedState(store, args[0]) + if err != nil { + return err + } + sessions, err := codex.LoadSessions(home) + if err != nil { + return err + } + current, err := findSession(sessions, args[0]) + if err != nil { + return err + } + if targetPath == "" { + targetPath = filepath.Join(store, "fs", "sessions", state.SessionID, "fallback-current.jsonl") + } + result := FSRollbackResult{SessionID: state.SessionID, From: current.RolloutPath, Target: vfs.NativeFile{Path: filepath.Clean(targetPath)}, DryRun: !apply} + if apply { + managed, resolver, err := openManagedSession(command.Context(), store, state) + if err != nil { + return err + } + defer resolver.Close() + target, err := managed.MaterializeCurrent(command.Context(), filepath.Clean(targetPath), true) + if err != nil { + return err + } + if _, err := codex.RouteSession(command.Context(), codex.RouteOptions{CodexHome: home, SessionID: state.SessionID, ExpectedPath: current.RolloutPath, Target: codex.RouteTarget{Path: target.Path, Bytes: target.Bytes, SHA256: target.SHA256}}); err != nil { + return err + } + result.Target = target + result.Routed = true + result.DryRun = false + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "session=%s dry_run=%t routed=%t from=%s target=%s\n", result.SessionID, result.DryRun, result.Routed, result.From, result.Target.Path) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().StringVar(&targetPath, "to", "", "Native rollback target; defaults to the managed session directory") + command.Flags().BoolVar(&apply, "apply", false, "Materialize current bytes and update the Codex route") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSCompactCommand() *cobra.Command { + var codexHome string + var storeDir string + var idleFor time.Duration + var apply bool + var jsonOutput bool + command := &cobra.Command{ + Use: "compact ", + Short: "Fold the latest visible bytes into a new verified immutable generation", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) + state, err := managedState(store, args[0]) + if err != nil { + return err + } + result := FSCompactResult{SessionID: state.SessionID, CurrentGeneration: state.Generation, NextGeneration: state.Generation + 1, DryRun: !apply} + if apply { + managed, resolver, err := openManagedSession(command.Context(), store, state) + if err != nil { + return err + } + defer resolver.Close() + var preparedResolver *pack.Resolver + defer func() { + if preparedResolver != nil { + _ = preparedResolver.Close() + } + }() + compact, err := managed.Compact(command.Context(), vfs.CompactOptions{IdleFor: idleFor, Prepare: func(ctx context.Context, current vfs.NativeFile, generation uint64) (vfs.PreparedGeneration, error) { + currentManifest, err := fold.LoadManifestPath(state.ManifestPath) + if err != nil { + return vfs.PreparedGeneration{}, err + } + manifestPath := filepath.Join(store, "manifests", "generations", state.SessionID, fmt.Sprintf("%020d.json", generation)) + options := fold.FoldOptions{ + StoreDir: store, ManifestPathOverride: manifestPath, Apply: true, Overwrite: true, + FieldThreshold: currentManifest.Settings.FieldThreshold, MaxJSONLineBytes: currentManifest.Settings.MaxJSONLineBytes, + CDC: cdc.Options{MinBytes: currentManifest.Settings.CDCMinBytes, AverageBytes: currentManifest.Settings.CDCAverageBytes, MaxBytes: currentManifest.Settings.CDCMaxBytes}, + } + if _, err := fold.Fold(ctx, codex.Session{ID: state.SessionID, Title: currentManifest.Session.Title, CWD: currentManifest.Session.CWD, RolloutPath: current.Path, Archived: true}, options); err != nil { + return vfs.PreparedGeneration{}, err + } + if _, err := pack.Build(ctx, store, pack.BuildOptions{}); err != nil { + return vfs.PreparedGeneration{}, err + } + manifest, err := fold.LoadManifestPath(manifestPath) + if err != nil { + return vfs.PreparedGeneration{}, err + } + preparedResolver, err = pack.Open(store, pack.OpenOptions{}) + if err != nil { + return vfs.PreparedGeneration{}, err + } + view, err := vfs.NewView(manifest, preparedResolver) + if err != nil { + return vfs.PreparedGeneration{}, err + } + return vfs.PreparedGeneration{ManifestPath: manifestPath, Manifest: manifest, View: view}, nil + }}) + if err != nil { + return err + } + result.NextGeneration = compact.Generation + result.Bytes = compact.Bytes + result.SHA256 = compact.SHA256 + result.DryRun = false + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "session=%s dry_run=%t generation=%d->%d bytes=%s sha256=%s\n", result.SessionID, result.DryRun, result.CurrentGeneration, result.NextGeneration, formatBytes(result.Bytes), result.SHA256) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().DurationVar(&idleFor, "idle-for", 0, "Minimum stable time before compaction") + command.Flags().BoolVar(&apply, "apply", false, "Commit the new compacted generation") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSRecoverCommand() *cobra.Command { + var codexHome string + var storeDir string + var all bool + var apply bool + var jsonOutput bool + command := &cobra.Command{ + Use: "recover [session-id]", + Short: "Inspect or recover interrupted managed session operations", + Args: cobra.MaximumNArgs(1), + RunE: func(command *cobra.Command, args []string) error { + if len(args) == 0 && !all { + return errors.New("provide a session ID or --all") + } + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return err + } + selected := make([]vfs.SessionState, 0) + for _, state := range states { + if all || state.SessionID == args[0] { + selected = append(selected, state) + } + } + if !all && len(selected) == 0 { + return fmt.Errorf("managed session not found: %s", args[0]) + } + result := FSRecoverResult{DryRun: !apply} + for _, state := range selected { + result.SessionIDs = append(result.SessionIDs, state.SessionID) + if !apply { + continue + } + managed, resolver, err := openManagedSession(command.Context(), store, state) + if err != nil { + return err + } + if err := managed.Recover(command.Context()); err != nil { + _ = resolver.Close() + return err + } + _ = resolver.Close() + result.Recovered++ + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "dry_run=%t selected=%d recovered=%d\n", result.DryRun, len(result.SessionIDs), result.Recovered) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().BoolVar(&all, "all", false, "Recover every managed session") + command.Flags().BoolVar(&apply, "apply", false, "Apply deterministic journal recovery") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func addCompatibilityFlags(command *cobra.Command, flags *compatibilityFlags) { + command.Flags().StringVar(&flags.contractsPath, "contracts", "", "Compatibility contract directory; defaults to /compatibility") + command.Flags().StringVar(&flags.cliPath, "cli", "codex", "Codex CLI path, or 'none' to skip CLI evaluation") + defaultDesktop := "none" + if runtime.GOOS == "darwin" { + defaultDesktop = "/Applications/ChatGPT.app" + } + command.Flags().StringVar(&flags.desktopPath, "desktop-app", defaultDesktop, "Codex desktop application path, or 'none' to skip desktop evaluation") +} + +func evaluateCompatibility(ctx context.Context, store string, flags compatibilityFlags) (FSCompatibilityResult, error) { + contractsPath := flags.contractsPath + if contractsPath == "" { + contractsPath = filepath.Join(store, "compatibility") + } + contracts, err := compat.LoadAll(contractsPath) + if err != nil { + return FSCompatibilityResult{}, err + } + result := FSCompatibilityResult{Contracts: len(contracts)} + if flags.cliPath != "none" { + binary := flags.cliPath + if !strings.ContainsRune(binary, filepath.Separator) { + resolved, err := exec.LookPath(binary) + if err != nil { + result.DetectionErrors = append(result.DetectionErrors, "cli: "+err.Error()) + } else { + binary = resolved + } + } + if len(result.DetectionErrors) == 0 { + client, err := compat.DetectCLIVersion(ctx, binary) + if err != nil { + result.DetectionErrors = append(result.DetectionErrors, "cli: "+err.Error()) + } else { + result.Installed = append(result.Installed, client) + } + } + } + if flags.desktopPath != "none" { + if _, err := os.Stat(flags.desktopPath); err != nil { + result.DetectionErrors = append(result.DetectionErrors, "desktop: "+err.Error()) + } else { + client, err := compat.DetectDesktopVersion(ctx, flags.desktopPath) + if err != nil { + result.DetectionErrors = append(result.DetectionErrors, "desktop: "+err.Error()) + } else { + result.Installed = append(result.Installed, client) + } + } + } + result.Evaluation = compat.Evaluate(result.Installed, contracts) + if len(result.Installed) == 0 { + result.Evaluation = compat.Evaluation{Approved: false, Quarantine: true} + } + return result, nil +} + +func openFoldView(home string, store string, sessionID string) (codex.Session, fold.Manifest, *pack.Resolver, *vfs.View, error) { + sessions, err := codex.LoadSessions(home) + if err != nil { + return codex.Session{}, fold.Manifest{}, nil, nil, err + } + session, err := findSession(sessions, sessionID) + if err != nil { + return codex.Session{}, fold.Manifest{}, nil, nil, err + } + manifest, err := fold.LoadManifest(store, session.ID) + if err != nil { + return codex.Session{}, fold.Manifest{}, nil, nil, err + } + resolver, err := pack.Open(store, pack.OpenOptions{}) + if err != nil { + return codex.Session{}, fold.Manifest{}, nil, nil, err + } + view, err := vfs.NewView(manifest, resolver) + if err != nil { + _ = resolver.Close() + return codex.Session{}, fold.Manifest{}, nil, nil, err + } + return session, manifest, resolver, view, nil +} + +func openManagedSession(ctx context.Context, store string, state vfs.SessionState) (*vfs.Session, *pack.Resolver, error) { + manifest, err := fold.LoadManifestPath(state.ManifestPath) + if err != nil { + return nil, nil, err + } + resolver, err := pack.Open(store, pack.OpenOptions{}) + if err != nil { + return nil, nil, err + } + managed, err := vfs.OpenSession(ctx, vfs.SessionOptions{Root: store, ManifestPath: state.ManifestPath, Manifest: manifest, Reader: resolver, NativeSnapshot: state.NativeSnapshot}) + if err != nil { + _ = resolver.Close() + return nil, nil, err + } + return managed, resolver, nil +} + +func managedState(store string, sessionID string) (vfs.SessionState, error) { + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return vfs.SessionState{}, err + } + for _, state := range states { + if state.SessionID == sessionID { + return state, nil + } + } + return vfs.SessionState{}, fmt.Errorf("managed session not found: %s", sessionID) +} + +func requireStorageHealth(ctx context.Context, store string) error { + packReport, err := pack.Doctor(ctx, store) + if err != nil { + return err + } + if packReport.IssueCount != 0 { + return fmt.Errorf("pack doctor reported %d issues", packReport.IssueCount) + } + foldReport, err := fold.Doctor(ctx, store) + if err != nil { + return err + } + if foldReport.IssueCount != 0 { + return fmt.Errorf("fold doctor reported %d issues", foldReport.IssueCount) + } + return nil +} + +func fsDoctor(ctx context.Context, home string, store string, mount string) fsctl.DoctorReport { + checks := []fsctl.Check{ + {Component: fsctl.ComponentDaemon, Run: func(context.Context) error { return errors.New("managed service lifecycle is not installed") }}, + {Component: fsctl.ComponentMount, Run: func(context.Context) error { + info, err := os.Stat(mount) + if err != nil || !info.IsDir() { + return errors.New("filesystem mount point is unavailable") + } + return nil + }}, + {Component: fsctl.ComponentPack, Run: func(ctx context.Context) error { + report, err := pack.Doctor(ctx, store) + if err != nil { + return err + } + if report.IssueCount != 0 { + return fmt.Errorf("pack doctor reported %d issues", report.IssueCount) + } + return nil + }}, + {Component: fsctl.ComponentManifest, Run: func(ctx context.Context) error { + report, err := fold.Doctor(ctx, store) + if err != nil { + return err + } + if report.IssueCount != 0 { + return fmt.Errorf("fold doctor reported %d issues", report.IssueCount) + } + return nil + }}, + } + states, stateErr := vfs.DiscoverSessionStates(store) + stateCheck := func(kind string) fsctl.Check { + return fsctl.Check{Component: kind, Run: func(context.Context) error { + if stateErr != nil { + return stateErr + } + for _, state := range states { + paths := []string{state.DeltaPath} + if kind == fsctl.ComponentBacking && state.BackingPath != "" { + paths = []string{state.BackingPath} + } + for _, path := range paths { + if _, err := os.Stat(path); err != nil { + return err + } + } + } + return nil + }} + } + checks = append(checks, stateCheck(fsctl.ComponentDelta), stateCheck(fsctl.ComponentBacking)) + checks = append(checks, + fsctl.Check{Component: fsctl.ComponentRoute, Run: func(context.Context) error { + sessions, err := codex.LoadSessions(home) + if err != nil { + return err + } + for _, session := range sessions { + if _, err := os.Stat(session.RolloutPath); err != nil { + return fmt.Errorf("session %s route: %w", session.ID, err) + } + } + return nil + }}, + fsctl.Check{Component: fsctl.ComponentFallback, Run: func(context.Context) error { + if stateErr != nil { + return stateErr + } + for _, state := range states { + if _, err := os.Stat(state.NativeSnapshot.Path); err != nil { + return err + } + } + return nil + }}, + fsctl.Check{Component: fsctl.ComponentJournal, Run: func(context.Context) error { + if stateErr != nil { + return stateErr + } + for _, state := range states { + path := filepath.Join(store, "fs", "sessions", state.SessionID, "journal.jsonl") + if file, err := os.Open(path); err == nil { + _ = file.Close() + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + } + return nil + }}, + fsctl.Check{Component: fsctl.ComponentClient, Run: func(context.Context) error { return errors.New("run fs compatibility with explicit client contracts") }}, + ) + return fsctl.Doctor(ctx, checks) +} + +func defaultMountPoint(home string, explicit string) string { + if explicit != "" { + return filepath.Clean(explicit) + } + return filepath.Join(home, "fold-fs") +} + +func waitForTarget(ctx context.Context, target string, timeout time.Duration) (vfs.NativeFile, error) { + if timeout <= 0 { + timeout = 5 * time.Second + } + deadline := time.Now().Add(timeout) + for { + file, err := hashPath(target) + if err == nil { + return file, nil + } + if !errors.Is(err, os.ErrNotExist) { + return vfs.NativeFile{}, err + } + if time.Now().After(deadline) { + return vfs.NativeFile{}, os.ErrNotExist + } + select { + case <-ctx.Done(): + return vfs.NativeFile{}, ctx.Err() + case <-time.After(50 * time.Millisecond): + } + } +} + +func hashPath(path string) (vfs.NativeFile, error) { + file, err := os.Open(path) + if err != nil { + return vfs.NativeFile{}, err + } + hasher := sha256.New() + bytesRead, copyErr := io.Copy(hasher, file) + closeErr := file.Close() + if copyErr != nil { + return vfs.NativeFile{}, copyErr + } + if closeErr != nil { + return vfs.NativeFile{}, closeErr + } + return vfs.NativeFile{Path: path, Bytes: bytesRead, SHA256: hex.EncodeToString(hasher.Sum(nil))}, nil +} diff --git a/internal/cli/fs_test.go b/internal/cli/fs_test.go new file mode 100644 index 0000000..dd19977 --- /dev/null +++ b/internal/cli/fs_test.go @@ -0,0 +1,352 @@ +package cli + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/jstar0/codexfold/internal/codex" + "github.com/jstar0/codexfold/internal/compat" + "github.com/jstar0/codexfold/internal/fold" + "github.com/jstar0/codexfold/internal/fsctl" + "github.com/jstar0/codexfold/internal/pack" + "github.com/jstar0/codexfold/internal/vfs" +) + +func TestRootExposesPackAndFilesystemCommands(t *testing.T) { + root := NewRootCommand() + for _, commandPath := range [][]string{ + {"pack", "build"}, {"pack", "doctor"}, + {"fs", "status"}, {"fs", "doctor"}, {"fs", "compatibility"}, {"fs", "benchmark"}, + {"fs", "serve"}, {"fs", "migrate"}, {"fs", "rollback"}, {"fs", "compact"}, {"fs", "recover"}, + } { + if _, _, err := root.Find(commandPath); err != nil { + t.Fatalf("command %v should be exposed: %v", commandPath, err) + } + } +} + +func TestPackBuildAndDoctorCommandsUseFoldStore(t *testing.T) { + home, storeDir, _ := fsFixture(t, true) + var output bytes.Buffer + root := NewRootCommand() + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"pack", "build", "--codex-home", home, "--store", storeDir, "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("pack build: %v", err) + } + var build pack.BuildResult + if err := json.Unmarshal(output.Bytes(), &build); err != nil || build.ObjectCount == 0 { + t.Fatalf("unexpected pack build: %#v err=%v output=%s", build, err, output.String()) + } + + output.Reset() + root = NewRootCommand() + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"pack", "doctor", "--store", storeDir, "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("pack doctor: %v", err) + } + var doctor pack.DoctorResult + if err := json.Unmarshal(output.Bytes(), &doctor); err != nil || doctor.IssueCount != 0 { + t.Fatalf("unexpected pack doctor: %#v err=%v output=%s", doctor, err, output.String()) + } +} + +func TestFSMigrateIsDryRunByDefaultAndDoesNotChangeRoute(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "migrate", "session", "--codex-home", home, "--store", storeDir, "--mount", filepath.Join(home, "mount"), "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("fs migrate dry-run: %v", err) + } + var result FSMigrateResult + if err := json.Unmarshal(output.Bytes(), &result); err != nil || !result.DryRun || result.Routed { + t.Fatalf("unexpected migrate result: %#v err=%v output=%s", result, err, output.String()) + } + sessions, err := codex.LoadSessions(home) + if err != nil || sessions[0].RolloutPath != nativePath { + t.Fatalf("dry-run changed route: sessions=%#v err=%v", sessions, err) + } +} + +func TestFSMigrateApplyFailsClosedWithoutMountedTarget(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "migrate", "session", "--codex-home", home, "--store", storeDir, "--mount", filepath.Join(home, "missing-mount"), "--apply"}) + if err := root.Execute(); err == nil { + t.Fatal("fs migrate --apply should fail without a live mounted target") + } + sessions, _ := codex.LoadSessions(home) + if sessions[0].RolloutPath != nativePath { + t.Fatalf("failed apply changed route to %q", sessions[0].RolloutPath) + } +} + +func TestFSStatusDoesNotClaimTransparentReadiness(t *testing.T) { + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "status", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("fs status: %v", err) + } + if bytes.Contains(output.Bytes(), []byte("production-ready")) || bytes.Contains(output.Bytes(), []byte("platform-canary")) { + t.Fatalf("status overclaimed readiness: %s", output.String()) + } +} + +func TestFSCompatibilityApprovesOnlyExactInstalledClientContract(t *testing.T) { + _, storeDir, _ := fsFixture(t, true) + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "compatibility", "--store", storeDir, "--cli", cliPath, "--desktop-app", "none", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("fs compatibility: %v", err) + } + var result FSCompatibilityResult + if err := json.Unmarshal(output.Bytes(), &result); err != nil || !result.Evaluation.Approved || result.Evaluation.Quarantine { + t.Fatalf("unexpected compatibility result: %#v err=%v output=%s", result, err, output.String()) + } +} + +func TestFSMigrateApplyInitializesManagedStateAndRoutesVerifiedTarget(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + mount := filepath.Join(home, "mount") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(mount, "session.jsonl") + data, err := os.ReadFile(nativePath) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target, data, 0o600); err != nil { + t.Fatal(err) + } + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "migrate", "session", "--codex-home", home, "--store", storeDir, "--mount", mount, "--cli", cliPath, "--desktop-app", "none", "--apply"}) + if err := root.Execute(); err != nil { + t.Fatalf("fs migrate --apply: %v", err) + } + sessions, err := codex.LoadSessions(home) + if err != nil || sessions[0].RolloutPath != target { + t.Fatalf("route not updated: sessions=%#v err=%v", sessions, err) + } + states, err := vfs.DiscoverSessionStates(storeDir) + if err != nil || len(states) != 1 || states[0].NativeSnapshot.Path != nativePath { + t.Fatalf("managed state missing: %#v err=%v", states, err) + } +} + +func TestFSRollbackUsesLatestVisibleBytesAfterVirtualAppend(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + mount := filepath.Join(home, "mount") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(mount, "session.jsonl") + original, _ := os.ReadFile(nativePath) + if err := os.WriteFile(target, original, 0o600); err != nil { + t.Fatal(err) + } + executeFS(t, []string{"fs", "migrate", "session", "--codex-home", home, "--store", storeDir, "--mount", mount, "--cli", cliPath, "--desktop-app", "none", "--apply"}) + state, err := managedState(storeDir, "session") + if err != nil { + t.Fatal(err) + } + managed, resolver, err := openManagedSession(context.Background(), storeDir, state) + if err != nil { + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + tail := []byte("{\"appended\":true}\n") + if _, err := writer.Append(context.Background(), tail); err != nil { + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + t.Fatal(err) + } + _ = writer.Close() + _ = resolver.Close() + executeFS(t, []string{"fs", "rollback", "session", "--codex-home", home, "--store", storeDir, "--apply"}) + sessions, err := codex.LoadSessions(home) + if err != nil { + t.Fatal(err) + } + fallback, err := os.ReadFile(sessions[0].RolloutPath) + if err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), original...), tail...) + if !bytes.Equal(fallback, want) { + t.Fatalf("rollback used stale bytes: got=%q want=%q", fallback, want) + } +} + +func TestFSCompactCommitsNewExactGeneration(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + native, err := hashPath(nativePath) + if err != nil { + t.Fatal(err) + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, Reader: resolver, NativeSnapshot: native}) + if err != nil { + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + if _, err := writer.Append(context.Background(), []byte("{\"tail\":2}\n")); err != nil { + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + t.Fatal(err) + } + _ = writer.Close() + before := filepath.Join(home, "before.jsonl") + expected, err := managed.MaterializeCurrent(context.Background(), before, false) + if err != nil { + t.Fatal(err) + } + _ = resolver.Close() + executeFS(t, []string{"fs", "compact", "session", "--codex-home", home, "--store", storeDir, "--apply"}) + state, err := managedState(storeDir, "session") + if err != nil || state.Generation != 2 || state.BaseSHA256 != expected.SHA256 { + t.Fatalf("unexpected compacted state: %#v err=%v", state, err) + } + reopened, nextResolver, err := openManagedSession(context.Background(), storeDir, state) + if err != nil { + t.Fatal(err) + } + after, err := reopened.MaterializeCurrent(context.Background(), filepath.Join(home, "after.jsonl"), false) + _ = nextResolver.Close() + if err != nil || after.Bytes != expected.Bytes || after.SHA256 != expected.SHA256 { + t.Fatalf("compacted bytes changed: before=%#v after=%#v err=%v", expected, after, err) + } +} + +func TestFSReadOnlyCommandsRunWithoutClaimingMountHealth(t *testing.T) { + home, storeDir, _ := fsFixture(t, true) + for _, args := range [][]string{ + {"fs", "doctor", "--codex-home", home, "--store", storeDir, "--json"}, + {"fs", "benchmark", "session", "--codex-home", home, "--store", storeDir, "--random-reads", "10", "--json"}, + {"fs", "serve", "--codex-home", home, "--store", storeDir, "--json"}, + } { + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs(args) + if err := root.Execute(); err != nil { + t.Fatalf("%v: %v", args, err) + } + if bytes.Contains(output.Bytes(), []byte("production-ready")) || bytes.Contains(output.Bytes(), []byte("platform-canary")) { + t.Fatalf("%v overclaimed readiness: %s", args, output.String()) + } + } + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "serve", "--codex-home", home, "--store", storeDir, "--mount", filepath.Join(home, "mount"), "--apply"}) + if err := root.Execute(); err == nil { + t.Fatal("default build should not claim the FUSE prerequisite is available") + } + status, _ := fsctl.NewStatus(fsctl.StorageEngine, runtime.GOOS) + if status.Capability != fsctl.StorageEngine { + t.Fatalf("unexpected capability: %#v", status) + } +} + +func fsFixture(t *testing.T, archived bool) (string, string, string) { + t.Helper() + home := t.TempDir() + storeDir := filepath.Join(home, "fold-store") + nativePath := filepath.Join(home, "session.jsonl") + source := []byte("{\"type\":\"session_meta\"}\n{\"value\":\"repeated-field-value\"}\n") + if err := os.WriteFile(nativePath, source, 0o600); err != nil { + t.Fatalf("write rollout: %v", err) + } + writeStateFixture(t, home, nativePath) + if archived { + dbPath := filepath.Join(home, "state_5.sqlite") + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open state: %v", err) + } + _, err = db.Exec(`update threads set archived = 1 where id = 'fixture'; update threads set id = 'session' where id = 'fixture'`) + _ = db.Close() + if err != nil { + t.Fatalf("archive fixture: %v", err) + } + } + session := codex.Session{ID: "session", RolloutPath: nativePath, Archived: archived} + if _, err := fold.Fold(context.Background(), session, fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { + t.Fatalf("fold fixture: %v", err) + } + if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { + t.Fatalf("pack fixture: %v", err) + } + return home, storeDir, nativePath +} + +func approvedCLIContract(t *testing.T, storeDir string, version string) string { + t.Helper() + cliPath := filepath.Join(t.TempDir(), "codex") + script := "#!/bin/sh\necho 'codex-cli " + version + "'\n" + if err := os.WriteFile(cliPath, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + _, err := compat.Save(filepath.Join(storeDir, "compatibility"), compat.Contract{ + Version: compat.ContractVersion, Platform: runtime.GOOS, ClientKind: "cli", ClientVersion: version, + Operations: []compat.Operation{{Name: "read", Count: 1}}, + TraceSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }) + if err != nil { + t.Fatal(err) + } + return cliPath +} + +func executeFS(t *testing.T, args []string) { + t.Helper() + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs(args) + if err := root.Execute(); err != nil { + t.Fatalf("execute %v: %v", args, err) + } +} diff --git a/internal/cli/pack.go b/internal/cli/pack.go new file mode 100644 index 0000000..72c725f --- /dev/null +++ b/internal/cli/pack.go @@ -0,0 +1,79 @@ +package cli + +import ( + "fmt" + + "github.com/jstar0/codexfold/internal/codex" + "github.com/jstar0/codexfold/internal/pack" + "github.com/spf13/cobra" +) + +func newPackCommand() *cobra.Command { + command := &cobra.Command{Use: "pack", Short: "Build and verify packed object generations"} + command.AddCommand(newPackBuildCommand()) + command.AddCommand(newPackDoctorCommand()) + return command +} + +func newPackBuildCommand() *cobra.Command { + var codexHome string + var storeDir string + var options pack.BuildOptions + var jsonOutput bool + command := &cobra.Command{ + Use: "build", + Short: "Build a verified immutable pack generation", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + result, err := pack.Build(command.Context(), resolveFoldStore(home, storeDir), options) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "generation=%s objects=%d blocks=%d packs=%d raw=%s stored=%s\n", result.Generation, result.ObjectCount, result.BlockCount, result.PackCount, formatBytes(result.RawBytes), formatBytes(result.StoredBytes)) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().Int64Var(&options.BlockBytes, "block-bytes", 0, "Uncompressed bytes per independently compressed block") + command.Flags().Int64Var(&options.PackBytes, "pack-bytes", 0, "Maximum stored bytes per pack file") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newPackDoctorCommand() *cobra.Command { + var codexHome string + var storeDir string + var jsonOutput bool + command := &cobra.Command{ + Use: "doctor", + Short: "Verify the active packed object generation", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + result, err := pack.Doctor(command.Context(), resolveFoldStore(home, storeDir)) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "generation=%s objects=%d verified=%d issues=%d\n", result.Generation, result.ObjectCount, result.VerifiedCount, result.IssueCount) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 0a86c2e..fdb819d 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -30,6 +30,8 @@ func NewRootCommand() *cobra.Command { root.AddCommand(newUnfoldCommand("materialize")) root.AddCommand(newDoctorCommand()) root.AddCommand(newGCCommand()) + root.AddCommand(newPackCommand()) + root.AddCommand(newFSCommand()) return root } diff --git a/internal/fold/doctor.go b/internal/fold/doctor.go index b04a7e4..e1188ce 100644 --- a/internal/fold/doctor.go +++ b/internal/fold/doctor.go @@ -2,11 +2,11 @@ package fold import ( "context" + "errors" "fmt" "io/fs" "os" "path/filepath" - "strings" ) type DoctorIssue struct { @@ -25,6 +25,11 @@ type DoctorResult struct { Issues []DoctorIssue `json:"issues"` } +type loadedManifest struct { + Path string + Manifest Manifest +} + func Doctor(ctx context.Context, storeDir string) (DoctorResult, error) { result := DoctorResult{StoreDir: storeDir, Issues: make([]DoctorIssue, 0)} manifests, loadIssues, err := loadAllManifests(storeDir) @@ -35,7 +40,8 @@ func Doctor(ctx context.Context, storeDir string) (DoctorResult, error) { result.Issues = append(result.Issues, loadIssues...) store := NewObjectStore(storeDir) unique := make(map[string]ObjectRef) - for _, manifest := range manifests { + for _, loaded := range manifests { + manifest := loaded.Manifest if err := ctx.Err(); err != nil { return DoctorResult{}, err } @@ -45,7 +51,7 @@ func Doctor(ctx context.Context, storeDir string) (DoctorResult, error) { } if err := verifyStoredManifest(ctx, store, manifest); err != nil { result.Issues = append(result.Issues, DoctorIssue{ - Scope: "manifest", Path: ManifestPath(storeDir, manifest.Session.ID), Error: err.Error(), + Scope: "manifest", Path: loaded.Path, Error: err.Error(), }) } else { result.VerifiedManifestCount++ @@ -66,28 +72,30 @@ func Doctor(ctx context.Context, storeDir string) (DoctorResult, error) { return result, nil } -func loadAllManifests(storeDir string) ([]Manifest, []DoctorIssue, error) { +func loadAllManifests(storeDir string) ([]loadedManifest, []DoctorIssue, error) { manifestDir := filepath.Join(storeDir, "manifests") - entries, err := os.ReadDir(manifestDir) - if err != nil { - if os.IsNotExist(err) { - return []Manifest{}, []DoctorIssue{}, nil - } - return nil, nil, fmt.Errorf("read manifest directory: %w", err) - } - manifests := make([]Manifest, 0, len(entries)) + manifests := make([]loadedManifest, 0) issues := make([]DoctorIssue, 0) - for _, entry := range entries { + err := filepath.WalkDir(manifestDir, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { - continue + return nil } - sessionID := strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name())) - manifest, err := LoadManifest(storeDir, sessionID) + manifest, err := LoadManifestPath(path) if err != nil { - issues = append(issues, DoctorIssue{Scope: "manifest", Path: filepath.Join(manifestDir, entry.Name()), Error: err.Error()}) - continue + issues = append(issues, DoctorIssue{Scope: "manifest", Path: path, Error: err.Error()}) + return nil } - manifests = append(manifests, manifest) + manifests = append(manifests, loadedManifest{Path: path, Manifest: manifest}) + return nil + }) + if errors.Is(err, os.ErrNotExist) { + return []loadedManifest{}, []DoctorIssue{}, nil + } + if err != nil { + return nil, nil, fmt.Errorf("read manifest directory: %w", err) } return manifests, issues, nil } diff --git a/internal/fold/doctor_gc_test.go b/internal/fold/doctor_gc_test.go index add712e..be40dba 100644 --- a/internal/fold/doctor_gc_test.go +++ b/internal/fold/doctor_gc_test.go @@ -90,6 +90,27 @@ func TestGCDryRunAndApplyRemoveOnlyUnreferencedObjects(t *testing.T) { } } +func TestDoctorAndGCKeepGenerationManifestObjects(t *testing.T) { + root := t.TempDir() + storeDir := filepath.Join(root, "store") + sourcePath := filepath.Join(root, "generation.jsonl") + if err := os.WriteFile(sourcePath, []byte("{\"value\":\"generation-only-field\"}\n"), 0o600); err != nil { + t.Fatal(err) + } + manifestPath := filepath.Join(storeDir, "manifests", "generations", "session", "2.json") + if _, err := Fold(context.Background(), codex.Session{ID: "session", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, ManifestPathOverride: manifestPath, Apply: true, FieldThreshold: 4}); err != nil { + t.Fatalf("Fold generation: %v", err) + } + doctor, err := Doctor(context.Background(), storeDir) + if err != nil || doctor.ManifestCount != 1 || doctor.IssueCount != 0 { + t.Fatalf("generation manifest not covered by doctor: %#v err=%v", doctor, err) + } + gc, err := GC(context.Background(), storeDir, true) + if err != nil || gc.OrphanCount != 0 || gc.Referenced == 0 { + t.Fatalf("generation object treated as orphan: %#v err=%v", gc, err) + } +} + func TestRemoveSourceRequiresGuardAndCanMaterializeAgain(t *testing.T) { root := t.TempDir() storeDir := filepath.Join(root, "store") diff --git a/internal/fold/fold.go b/internal/fold/fold.go index ac7478a..1a8982d 100644 --- a/internal/fold/fold.go +++ b/internal/fold/fold.go @@ -11,6 +11,8 @@ import ( "hash" "io" "os" + "path/filepath" + "strings" "github.com/jstar0/codexfold/internal/cdc" "github.com/jstar0/codexfold/internal/codex" @@ -18,15 +20,16 @@ import ( ) type FoldOptions struct { - StoreDir string - Apply bool - Overwrite bool - RemoveSource bool - AllowActive bool - FieldThreshold int64 - MaxJSONLineBytes int64 - CDC cdc.Options - beforeCommit func() error + StoreDir string + ManifestPathOverride string + Apply bool + Overwrite bool + RemoveSource bool + AllowActive bool + FieldThreshold int64 + MaxJSONLineBytes int64 + CDC cdc.Options + beforeCommit func() error } type FoldResult struct { @@ -68,6 +71,14 @@ func Fold(ctx context.Context, session codex.Session, options FoldOptions) (Fold options.CDC = cdc.Options{MinBytes: 4 * 1024, AverageBytes: 16 * 1024, MaxBytes: 64 * 1024} } manifestPath := ManifestPath(options.StoreDir, session.ID) + if options.ManifestPathOverride != "" { + manifestPath = filepath.Clean(options.ManifestPathOverride) + manifestRoot := filepath.Join(options.StoreDir, "manifests") + relative, err := filepath.Rel(manifestRoot, manifestPath) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return FoldResult{}, errors.New("manifest override must remain inside the fold manifest directory") + } + } if options.Apply && !options.Overwrite { if _, err := os.Stat(manifestPath); err == nil { return FoldResult{}, fmt.Errorf("fold manifest already exists: %s", manifestPath) @@ -260,7 +271,7 @@ complete: if err := store.SyncPending(ctx); err != nil { return FoldResult{}, err } - if err := writeManifest(options.StoreDir, manifest, options.Overwrite); err != nil { + if err := writeManifestPath(manifestPath, manifest, options.Overwrite); err != nil { return FoldResult{}, err } if options.RemoveSource { diff --git a/internal/fold/fold_test.go b/internal/fold/fold_test.go index b17ee24..b2da913 100644 --- a/internal/fold/fold_test.go +++ b/internal/fold/fold_test.go @@ -162,6 +162,38 @@ func TestFoldDryRunDoesNotCreateStore(t *testing.T) { } } +func TestFoldWritesToExplicitGenerationManifestWithoutReplacingPrimary(t *testing.T) { + root := t.TempDir() + storeDir := filepath.Join(root, "store") + sourcePath := filepath.Join(root, "session.jsonl") + data := []byte("{\"value\":\"generation-manifest\"}\n") + if err := os.WriteFile(sourcePath, data, 0o600); err != nil { + t.Fatal(err) + } + primary := ManifestPath(storeDir, "session") + if err := os.MkdirAll(filepath.Dir(primary), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(primary, []byte("primary-sentinel"), 0o600); err != nil { + t.Fatal(err) + } + generationPath := filepath.Join(storeDir, "manifests", "generations", "session", "2.json") + result, err := Fold(context.Background(), codex.Session{ID: "session", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8, ManifestPathOverride: generationPath}) + if err != nil { + t.Fatalf("Fold generation manifest: %v", err) + } + if result.ManifestPath != generationPath { + t.Fatalf("manifest path = %q, want %q", result.ManifestPath, generationPath) + } + if primaryData, err := os.ReadFile(primary); err != nil || string(primaryData) != "primary-sentinel" { + t.Fatalf("primary manifest changed: %q err=%v", primaryData, err) + } + manifest, err := LoadManifestPath(generationPath) + if err != nil || manifest.Source.Bytes != int64(len(data)) { + t.Fatalf("load generation manifest: %#v err=%v", manifest, err) + } +} + func TestFoldRoundTripsEmptyInvalidAndOversizedRollouts(t *testing.T) { for _, test := range []struct { name string diff --git a/internal/fold/gc.go b/internal/fold/gc.go index 9139455..e0d564b 100644 --- a/internal/fold/gc.go +++ b/internal/fold/gc.go @@ -28,8 +28,8 @@ func GC(ctx context.Context, storeDir string, apply bool) (GCResult, error) { return GCResult{}, fmt.Errorf("refusing GC with %d invalid manifest(s)", len(issues)) } referenced := make(map[string]struct{}) - for _, manifest := range manifests { - for _, part := range manifest.Parts { + for _, loaded := range manifests { + for _, part := range loaded.Manifest.Parts { referenced[part.Object.SHA256] = struct{}{} } } diff --git a/internal/fold/manifest.go b/internal/fold/manifest.go index 8044a17..e4703b0 100644 --- a/internal/fold/manifest.go +++ b/internal/fold/manifest.go @@ -62,7 +62,10 @@ func LoadManifest(storeDir string, sessionID string) (Manifest, error) { if err := validateSessionID(sessionID); err != nil { return Manifest{}, err } - path := ManifestPath(storeDir, sessionID) + return LoadManifestPath(ManifestPath(storeDir, sessionID)) +} + +func LoadManifestPath(path string) (Manifest, error) { data, err := os.ReadFile(path) if err != nil { return Manifest{}, fmt.Errorf("read fold manifest: %w", err) @@ -81,7 +84,13 @@ func writeManifest(storeDir string, manifest Manifest, overwrite bool) error { if err := validateSessionID(manifest.Session.ID); err != nil { return err } - path := ManifestPath(storeDir, manifest.Session.ID) + return writeManifestPath(ManifestPath(storeDir, manifest.Session.ID), manifest, overwrite) +} + +func writeManifestPath(path string, manifest Manifest, overwrite bool) error { + if err := validateManifest(manifest); err != nil { + return err + } if !overwrite { if _, err := os.Stat(path); err == nil { return fmt.Errorf("fold manifest already exists: %s", path) diff --git a/internal/mountfs/filesystem.go b/internal/mountfs/filesystem.go index 0715b50..97148e7 100644 --- a/internal/mountfs/filesystem.go +++ b/internal/mountfs/filesystem.go @@ -53,6 +53,16 @@ func (f *Filesystem) AddSession(sessionID string, session *vfs.Session) error { return nil } +func (f *Filesystem) UpsertSession(sessionID string, session *vfs.Session) error { + if sessionID == "" || strings.ContainsAny(sessionID, "/\\\x00") || session == nil { + return errors.New("safe session ID and session are required") + } + f.mu.Lock() + f.sessions[sessionID] = session + f.mu.Unlock() + return nil +} + func (f *Filesystem) ReadDir(name string) ([]string, syscall.Errno) { if cleanPath(name) != "/" { return nil, syscall.ENOTDIR diff --git a/internal/mountfs/filesystem_test.go b/internal/mountfs/filesystem_test.go index 3e92e55..2b0c762 100644 --- a/internal/mountfs/filesystem_test.go +++ b/internal/mountfs/filesystem_test.go @@ -103,6 +103,36 @@ func TestFilesystemRejectsUnsafeAndManagementMutations(t *testing.T) { } } +func TestFilesystemUpsertChangesNewOpensWithoutInvalidatingExistingHandles(t *testing.T) { + first := mountSessionFixture(t, "first-session", []byte("first")) + second := mountSessionFixture(t, "second-session", []byte("second")) + filesystem := New() + if err := filesystem.AddSession("session", first); err != nil { + t.Fatal(err) + } + oldHandle, errno := filesystem.Open("/session.jsonl", os.O_RDONLY) + if errno != 0 { + t.Fatalf("open old handle: %v", errno) + } + if err := filesystem.UpsertSession("session", second); err != nil { + t.Fatalf("UpsertSession: %v", err) + } + newHandle, errno := filesystem.Open("/session.jsonl", os.O_RDONLY) + if errno != 0 { + t.Fatalf("open new handle: %v", errno) + } + oldBytes := make([]byte, 5) + if n, errno := filesystem.Read(oldHandle, oldBytes, 0); errno != 0 || n != 5 || string(oldBytes) != "first" { + t.Fatalf("old handle changed: n=%d errno=%v bytes=%q", n, errno, oldBytes) + } + newBytes := make([]byte, 6) + if n, errno := filesystem.Read(newHandle, newBytes, 0); errno != 0 || n != 6 || string(newBytes) != "second" { + t.Fatalf("new handle did not use replacement: n=%d errno=%v bytes=%q", n, errno, newBytes) + } + _ = filesystem.Release(oldHandle) + _ = filesystem.Release(newHandle) +} + func TestMountWithoutFuseBuildReturnsPrerequisiteError(t *testing.T) { err := Mount(context.Background(), HostOptions{MountPoint: t.TempDir(), Filesystem: New()}) if !errors.Is(err, ErrPrerequisite) { @@ -145,3 +175,20 @@ func mountFixture(t *testing.T) (*Filesystem, []byte) { } return filesystem, source } + +func mountSessionFixture(t *testing.T, sessionID string, source []byte) *vfs.Session { + t.Helper() + root := t.TempDir() + digest := sha256.Sum256(source) + hexDigest := hex.EncodeToString(digest[:]) + nativePath := filepath.Join(root, "native.jsonl") + if err := os.WriteFile(nativePath, source, 0o600); err != nil { + t.Fatal(err) + } + manifest := fold.Manifest{Version: fold.ManifestVersion, Kind: fold.ManifestKind, Session: fold.ManifestSession{ID: sessionID, RolloutPath: nativePath}, Source: fold.ManifestSource{Bytes: int64(len(source)), SHA256: hexDigest}, Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: hexDigest, RawBytes: int64(len(source))}}}} + session, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, Reader: mountReader{hexDigest: source}, NativeSnapshot: vfs.NativeFile{Path: nativePath, Bytes: int64(len(source)), SHA256: hexDigest}}) + if err != nil { + t.Fatal(err) + } + return session +} diff --git a/internal/pack/build.go b/internal/pack/build.go index 4441107..4bd372e 100644 --- a/internal/pack/build.go +++ b/internal/pack/build.go @@ -153,26 +153,28 @@ func Build(ctx context.Context, storeDir string, options BuildOptions) (BuildRes } func referencedObjects(storeDir string) ([]fold.ObjectRef, error) { - entries, err := os.ReadDir(filepath.Join(storeDir, "manifests")) - if err != nil { - return nil, fmt.Errorf("read manifests for pack build: %w", err) - } refs := make(map[string]fold.ObjectRef) - for _, entry := range entries { + err := filepath.WalkDir(filepath.Join(storeDir, "manifests"), func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { - continue + return nil } - sessionID := strings.TrimSuffix(entry.Name(), ".json") - manifest, err := fold.LoadManifest(storeDir, sessionID) + manifest, err := fold.LoadManifestPath(path) if err != nil { - return nil, err + return err } for _, part := range manifest.Parts { if existing, ok := refs[part.Object.SHA256]; ok && existing.RawBytes != part.Object.RawBytes { - return nil, fmt.Errorf("object %s has conflicting raw lengths", part.Object.SHA256) + return fmt.Errorf("object %s has conflicting raw lengths", part.Object.SHA256) } refs[part.Object.SHA256] = part.Object } + return nil + }) + if err != nil { + return nil, fmt.Errorf("read manifests for pack build: %w", err) } digests := make([]string, 0, len(refs)) for digest := range refs { diff --git a/internal/pack/pack_test.go b/internal/pack/pack_test.go index e040688..783c8d1 100644 --- a/internal/pack/pack_test.go +++ b/internal/pack/pack_test.go @@ -140,6 +140,43 @@ func TestResolverAndDoctorDetectPackCorruption(t *testing.T) { } } +func TestBuildIncludesObjectsReferencedByGenerationManifests(t *testing.T) { + root := t.TempDir() + data := []byte("generation-only-object") + store := fold.NewObjectStore(root) + ref, _, err := store.Put(data, true) + if err != nil { + t.Fatal(err) + } + if err := store.SyncPending(context.Background()); err != nil { + t.Fatal(err) + } + manifest := fold.Manifest{ + Version: fold.ManifestVersion, Kind: fold.ManifestKind, + Session: fold.ManifestSession{ID: "session", RolloutPath: "session.jsonl", Archived: true}, + Source: fold.ManifestSource{Bytes: int64(len(data)), SHA256: ref.SHA256}, + Parts: []fold.Part{{Kind: fold.PartResidual, Object: ref}}, + } + manifestPath := filepath.Join(root, "manifests", "generations", "session", "2.json") + if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(manifestPath, encoded, 0o600); err != nil { + t.Fatal(err) + } + result, err := Build(context.Background(), root, BuildOptions{}) + if err != nil { + t.Fatalf("Build: %v", err) + } + if result.ObjectCount != 1 { + t.Fatalf("object count = %d, want 1", result.ObjectCount) + } +} + func putObjects(t *testing.T, root string, values ...[]byte) []fold.ObjectRef { t.Helper() store := fold.NewObjectStore(root) diff --git a/internal/vfs/state.go b/internal/vfs/state.go index 9992550..367bc01 100644 --- a/internal/vfs/state.go +++ b/internal/vfs/state.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "sort" "strings" ) @@ -47,6 +48,45 @@ func loadSessionState(path string) (SessionState, error) { return state, nil } +func LoadSessionState(path string) (SessionState, error) { + state, err := loadSessionState(path) + if err != nil { + return SessionState{}, err + } + directory := filepath.Dir(filepath.Clean(path)) + if filepath.Base(directory) != state.SessionID || filepath.Base(filepath.Dir(directory)) != "sessions" { + return SessionState{}, errors.New("session state path does not match its session ID") + } + if !pathWithin(directory, state.DeltaPath) || (state.BackingPath != "" && !pathWithin(directory, state.BackingPath)) { + return SessionState{}, errors.New("session state contains an unsafe data path") + } + return state, nil +} + +func DiscoverSessionStates(root string) ([]SessionState, error) { + directory := filepath.Join(root, "fs", "sessions") + entries, err := os.ReadDir(directory) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read managed session states: %w", err) + } + states := make([]SessionState, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + state, err := LoadSessionState(filepath.Join(directory, entry.Name(), "state.json")) + if err != nil { + return nil, fmt.Errorf("load managed session %s: %w", entry.Name(), err) + } + states = append(states, state) + } + sort.Slice(states, func(i, j int) bool { return states[i].SessionID < states[j].SessionID }) + return states, nil +} + func writeSessionState(path string, state SessionState) error { data, err := json.MarshalIndent(state, "", " ") if err != nil { diff --git a/internal/vfs/state_discovery_test.go b/internal/vfs/state_discovery_test.go new file mode 100644 index 0000000..f26ab7c --- /dev/null +++ b/internal/vfs/state_discovery_test.go @@ -0,0 +1,55 @@ +package vfs + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDiscoverSessionStatesReturnsValidatedStatesInSessionOrder(t *testing.T) { + root := t.TempDir() + for _, sessionID := range []string{"beta", "alpha"} { + directory := filepath.Join(root, "fs", "sessions", sessionID) + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatalf("create state directory: %v", err) + } + state := SessionState{ + Version: sessionStateVersion, SessionID: sessionID, Generation: 1, + ManifestPath: filepath.Join(root, "manifests", sessionID+".json"), + BaseBytes: 1, BaseSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + DeltaPath: filepath.Join(directory, "delta.jsonl"), + NativeSnapshot: NativeFile{Path: filepath.Join(root, sessionID+".jsonl"), Bytes: 1, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + } + if err := writeSessionState(filepath.Join(directory, "state.json"), state); err != nil { + t.Fatalf("write state: %v", err) + } + } + states, err := DiscoverSessionStates(root) + if err != nil { + t.Fatalf("DiscoverSessionStates: %v", err) + } + if len(states) != 2 || states[0].SessionID != "alpha" || states[1].SessionID != "beta" { + t.Fatalf("unexpected states: %#v", states) + } +} + +func TestLoadSessionStateRejectsStateOutsideManagedSessionDirectory(t *testing.T) { + root := t.TempDir() + directory := filepath.Join(root, "fs", "sessions", "session") + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + state := SessionState{ + Version: sessionStateVersion, SessionID: "session", Generation: 1, + ManifestPath: filepath.Join(root, "manifest.json"), BaseBytes: 1, + BaseSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + DeltaPath: filepath.Join(root, "outside.jsonl"), + NativeSnapshot: NativeFile{Path: filepath.Join(root, "native.jsonl"), Bytes: 1, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + } + if err := writeSessionState(filepath.Join(directory, "state.json"), state); err != nil { + t.Fatal(err) + } + if _, err := LoadSessionState(filepath.Join(directory, "state.json")); err == nil { + t.Fatal("LoadSessionState should reject data paths outside the managed session directory") + } +} From 4589ffa41592edf80217da2195ee3f0971feca8f Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 01:45:25 +0800 Subject: [PATCH 10/33] feat: add guarded filesystem service lifecycle --- internal/cli/fs.go | 1 + internal/cli/fs_service.go | 382 +++++++++++++++++++++++++ internal/cli/fs_test.go | 102 ++++++- internal/mountfs/filesystem_test.go | 3 + internal/mountfs/host_cgofuse.go | 2 + internal/mountfs/host_stub.go | 2 + internal/service/mount_probe_darwin.go | 27 ++ internal/service/mount_probe_other.go | 19 ++ internal/service/service.go | 248 ++++++++++++++++ internal/service/service_test.go | 106 +++++++ 10 files changed, 887 insertions(+), 5 deletions(-) create mode 100644 internal/cli/fs_service.go create mode 100644 internal/service/mount_probe_darwin.go create mode 100644 internal/service/mount_probe_other.go create mode 100644 internal/service/service.go create mode 100644 internal/service/service_test.go diff --git a/internal/cli/fs.go b/internal/cli/fs.go index 7ae1a9f..8e58d86 100644 --- a/internal/cli/fs.go +++ b/internal/cli/fs.go @@ -87,6 +87,7 @@ func newFSCommand() *cobra.Command { command.AddCommand(newFSRollbackCommand()) command.AddCommand(newFSCompactCommand()) command.AddCommand(newFSRecoverCommand()) + command.AddCommand(newFSServiceCommand()) return command } diff --git a/internal/cli/fs_service.go b/internal/cli/fs_service.go new file mode 100644 index 0000000..eb8635c --- /dev/null +++ b/internal/cli/fs_service.go @@ -0,0 +1,382 @@ +package cli + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + + "github.com/jstar0/codexfold/internal/codex" + "github.com/jstar0/codexfold/internal/fsctl" + "github.com/jstar0/codexfold/internal/mountfs" + "github.com/jstar0/codexfold/internal/service" + "github.com/jstar0/codexfold/internal/vfs" + "github.com/spf13/cobra" +) + +const serviceLabel = "com.codexfold.fs" + +type FSServiceActionResult struct { + Action string `json:"action"` + Path string `json:"path,omitempty"` + DryRun bool `json:"dry_run"` +} + +type FSUpdatePreflightResult struct { + DoctorHealthy bool `json:"doctor_healthy"` + Compatibility FSCompatibilityResult `json:"compatibility"` + Decision service.UpdateDecision `json:"decision"` + QuarantinedSessions int `json:"quarantined_sessions"` +} + +func newFSServiceCommand() *cobra.Command { + command := &cobra.Command{Use: "service", Short: "Manage the per-user transparent filesystem service"} + command.AddCommand(newFSServiceInstallCommand()) + command.AddCommand(newFSServiceStartCommand()) + command.AddCommand(newFSServiceStopCommand()) + command.AddCommand(newFSServiceStatusCommand()) + command.AddCommand(newFSServiceUpdatePreflightCommand()) + return command +} + +func newFSServiceInstallCommand() *cobra.Command { + var codexHome, storeDir, mountPoint, binaryPath, plistPath, logDir string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "install", + Short: "Render and optionally bootstrap a per-user launchd service", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, store, mount, binary, plist, logs, err := resolveServicePaths(codexHome, storeDir, mountPoint, binaryPath, plistPath, logDir) + if err != nil { + return err + } + definition, err := service.RenderLaunchd(service.Options{Label: serviceLabel, BinaryPath: binary, CodexHome: home, StoreDir: store, MountPoint: mount, StdoutPath: filepath.Join(logs, "stdout.log"), StderrPath: filepath.Join(logs, "stderr.log")}) + if err != nil { + return err + } + if apply && (runtime.GOOS != "darwin" || !mountfs.Available()) { + return errors.New("service installation requires a FUSE-enabled macOS build and an authorized host prerequisite") + } + if apply { + if err := os.MkdirAll(logs, 0o700); err != nil { + return err + } + } + result, err := service.WriteDefinition(plist, definition, apply) + if err != nil { + return err + } + if apply { + manager := service.Manager{} + _ = manager.Bootout(command.Context(), plist) + if err := manager.Bootstrap(command.Context(), plist); err != nil { + return err + } + if err := manager.Kickstart(command.Context(), serviceLabel); err != nil { + return err + } + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "dry_run=%t path=%s bytes=%d\n", result.DryRun, result.Path, result.Bytes) + return err + }, + } + addServicePathFlags(command, &codexHome, &storeDir, &mountPoint, &binaryPath, &plistPath, &logDir) + command.Flags().BoolVar(&apply, "apply", false, "Write, bootstrap, and start the per-user service") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSServiceStartCommand() *cobra.Command { + return newFSServiceLifecycleCommand("start", func(ctx context.Context, manager service.Manager, plist string) error { + _ = manager.Bootout(ctx, plist) + if err := manager.Bootstrap(ctx, plist); err != nil { + return err + } + return manager.Kickstart(ctx, serviceLabel) + }) +} + +func newFSServiceStopCommand() *cobra.Command { + return newFSServiceLifecycleCommand("stop", func(ctx context.Context, manager service.Manager, plist string) error { + return manager.Bootout(ctx, plist) + }) +} + +func newFSServiceLifecycleCommand(action string, run func(context.Context, service.Manager, string) error) *cobra.Command { + var plistPath string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: action, + Short: action + " the per-user filesystem service", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + plist, err := resolvePlistPath(plistPath) + if err != nil { + return err + } + result := FSServiceActionResult{Action: action, Path: plist, DryRun: !apply} + if apply { + if runtime.GOOS != "darwin" { + return errors.New("launchd service lifecycle is available only on macOS") + } + if err := run(command.Context(), service.Manager{}, plist); err != nil { + return err + } + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "action=%s dry_run=%t path=%s\n", action, result.DryRun, plist) + return err + }, + } + command.Flags().StringVar(&plistPath, "plist", "", "LaunchAgent plist path") + command.Flags().BoolVar(&apply, "apply", false, "Execute the launchctl action") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSServiceStatusCommand() *cobra.Command { + var codexHome, mountPoint string + var jsonOutput bool + command := &cobra.Command{ + Use: "status", + Short: "Report daemon and mount health separately", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + status := service.Manager{}.Status(command.Context(), serviceLabel, defaultMountPoint(home, mountPoint)) + if jsonOutput { + return writeJSON(command, status) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "daemon=%t mount=%t daemon_error=%q mount_error=%q\n", status.DaemonRunning, status.MountHealthy, status.DaemonError, status.MountError) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSServiceUpdatePreflightCommand() *cobra.Command { + var codexHome, storeDir string + var compatibility compatibilityFlags + var automatic, promote, applyQuarantine, jsonOutput bool + command := &cobra.Command{ + Use: "update-preflight", + Short: "Gate service updates and optionally route unknown-version sessions to current native bytes", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) + doctorErr := requireStorageHealth(command.Context(), store) + compatibilityResult, err := evaluateCompatibility(command.Context(), store, compatibility) + if err != nil { + return err + } + fallbackReady, err := managedRoutesMatchCurrentBytes(command.Context(), home, store) + if err != nil { + fallbackReady = false + } + decision := service.EvaluateUpdate(service.UpdateInput{Capability: fsctl.StorageEngine, DoctorHealthy: doctorErr == nil, Compatibility: compatibilityResult.Evaluation, NativeFallbackReady: fallbackReady, Automatic: automatic, ExplicitPromotion: promote}) + result := FSUpdatePreflightResult{DoctorHealthy: doctorErr == nil, Compatibility: compatibilityResult, Decision: decision} + if decision.Quarantine && decision.RequiresNativeFallback && applyQuarantine { + count, err := quarantineManagedRoutes(command.Context(), home, store) + if err != nil { + return err + } + result.QuarantinedSessions = count + result.Decision = service.EvaluateUpdate(service.UpdateInput{Capability: fsctl.StorageEngine, DoctorHealthy: doctorErr == nil, Compatibility: compatibilityResult.Evaluation, NativeFallbackReady: true, Automatic: automatic, ExplicitPromotion: promote}) + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "allowed=%t quarantine=%t requires_fallback=%t doctor=%t quarantined=%d reason=%q\n", result.Decision.Allowed, result.Decision.Quarantine, result.Decision.RequiresNativeFallback, result.DoctorHealthy, result.QuarantinedSessions, result.Decision.Reason) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + addCompatibilityFlags(command, &compatibility) + command.Flags().BoolVar(&automatic, "automatic", false, "Evaluate an unattended update") + command.Flags().BoolVar(&promote, "promote", false, "Explicitly approve preview or canary promotion") + command.Flags().BoolVar(&applyQuarantine, "apply-quarantine", false, "Route managed sessions to verified current native bytes when clients are unknown") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func managedRoutesMatchCurrentBytes(ctx context.Context, home string, store string) (bool, error) { + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return false, err + } + sessions, err := codex.LoadSessions(home) + if err != nil { + return false, err + } + byID := make(map[string]codex.Session, len(sessions)) + for _, session := range sessions { + byID[session.ID] = session + } + for _, state := range states { + current, ok := byID[state.SessionID] + if !ok { + return false, fmt.Errorf("Codex route missing for managed session %s", state.SessionID) + } + managed, resolver, err := openManagedSession(ctx, store, state) + if err != nil { + return false, err + } + visible, err := hashManagedSession(ctx, managed) + _ = resolver.Close() + if err != nil { + return false, err + } + route, err := hashPath(current.RolloutPath) + if err != nil || route.Bytes != visible.Bytes || route.SHA256 != visible.SHA256 { + return false, nil + } + } + return true, nil +} + +func quarantineManagedRoutes(ctx context.Context, home string, store string) (int, error) { + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return 0, err + } + sessions, err := codex.LoadSessions(home) + if err != nil { + return 0, err + } + byID := make(map[string]codex.Session, len(sessions)) + for _, session := range sessions { + byID[session.ID] = session + } + count := 0 + for _, state := range states { + current, ok := byID[state.SessionID] + if !ok { + return count, fmt.Errorf("Codex route missing for managed session %s", state.SessionID) + } + managed, resolver, err := openManagedSession(ctx, store, state) + if err != nil { + return count, err + } + targetPath := filepath.Join(store, "fs", "sessions", state.SessionID, "quarantine-current.jsonl") + target, err := managed.MaterializeCurrent(ctx, targetPath, true) + _ = resolver.Close() + if err != nil { + return count, err + } + if filepath.Clean(current.RolloutPath) == filepath.Clean(target.Path) { + continue + } + if _, err := codex.RouteSession(ctx, codex.RouteOptions{CodexHome: home, SessionID: state.SessionID, ExpectedPath: current.RolloutPath, Target: codex.RouteTarget{Path: target.Path, Bytes: target.Bytes, SHA256: target.SHA256}}); err != nil { + return count, err + } + count++ + } + return count, nil +} + +func hashManagedSession(ctx context.Context, session *vfs.Session) (vfs.NativeFile, error) { + reader, err := session.OpenReader() + if err != nil { + return vfs.NativeFile{}, err + } + defer reader.Close() + hasher := sha256.New() + buffer := make([]byte, 1<<20) + var offset int64 + for offset < reader.Size() { + need := len(buffer) + if remaining := reader.Size() - offset; int64(need) > remaining { + need = int(remaining) + } + n, readErr := reader.ReadAt(ctx, buffer[:need], offset) + if n > 0 { + _, _ = hasher.Write(buffer[:n]) + offset += int64(n) + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + return vfs.NativeFile{}, readErr + } + if n == 0 { + break + } + } + if offset != reader.Size() { + return vfs.NativeFile{}, errors.New("managed session ended before its declared size") + } + return vfs.NativeFile{Bytes: offset, SHA256: hex.EncodeToString(hasher.Sum(nil))}, nil +} + +func addServicePathFlags(command *cobra.Command, codexHome, storeDir, mountPoint, binaryPath, plistPath, logDir *string) { + command.Flags().StringVar(codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().StringVar(mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + command.Flags().StringVar(binaryPath, "binary", "", "Absolute CodexFold binary path; defaults to the current executable") + command.Flags().StringVar(plistPath, "plist", "", "LaunchAgent plist path") + command.Flags().StringVar(logDir, "log-dir", "", "Service log directory; defaults to /service/logs") +} + +func resolveServicePaths(codexHome, storeDir, mountPoint, binaryPath, plistPath, logDir string) (string, string, string, string, string, string, error) { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return "", "", "", "", "", "", err + } + store := resolveFoldStore(home, storeDir) + mount := defaultMountPoint(home, mountPoint) + binary := binaryPath + if binary == "" { + binary, err = os.Executable() + if err != nil { + return "", "", "", "", "", "", err + } + } + binary, err = filepath.Abs(binary) + if err != nil { + return "", "", "", "", "", "", err + } + plist, err := resolvePlistPath(plistPath) + if err != nil { + return "", "", "", "", "", "", err + } + logs := logDir + if logs == "" { + logs = filepath.Join(store, "service", "logs") + } + logs, err = filepath.Abs(logs) + if err != nil { + return "", "", "", "", "", "", err + } + return home, store, mount, binary, plist, logs, nil +} + +func resolvePlistPath(explicit string) (string, error) { + if explicit != "" { + return filepath.Abs(explicit) + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, "Library", "LaunchAgents", serviceLabel+".plist"), nil +} diff --git a/internal/cli/fs_test.go b/internal/cli/fs_test.go index dd19977..d476a18 100644 --- a/internal/cli/fs_test.go +++ b/internal/cli/fs_test.go @@ -24,6 +24,8 @@ func TestRootExposesPackAndFilesystemCommands(t *testing.T) { {"pack", "build"}, {"pack", "doctor"}, {"fs", "status"}, {"fs", "doctor"}, {"fs", "compatibility"}, {"fs", "benchmark"}, {"fs", "serve"}, {"fs", "migrate"}, {"fs", "rollback"}, {"fs", "compact"}, {"fs", "recover"}, + {"fs", "service", "install"}, {"fs", "service", "start"}, {"fs", "service", "stop"}, + {"fs", "service", "status"}, {"fs", "service", "update-preflight"}, } { if _, _, err := root.Find(commandPath); err != nil { t.Fatalf("command %v should be exposed: %v", commandPath, err) @@ -31,6 +33,90 @@ func TestRootExposesPackAndFilesystemCommands(t *testing.T) { } } +func TestFSServiceInstallIsDryRunByDefaultAndApplyRequiresFuseBuild(t *testing.T) { + home, storeDir, _ := fsFixture(t, true) + plistPath := filepath.Join(home, "LaunchAgents", "com.codexfold.fs.plist") + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "service", "install", "--codex-home", home, "--store", storeDir, "--plist", plistPath, "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("service install dry-run: %v", err) + } + if _, err := os.Stat(plistPath); !os.IsNotExist(err) { + t.Fatalf("dry-run wrote plist: %v", err) + } + root = NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "service", "install", "--codex-home", home, "--store", storeDir, "--plist", plistPath, "--apply"}) + if err := root.Execute(); err == nil { + t.Fatal("default build should reject service installation without a FUSE host") + } +} + +func TestFSUpdatePreflightQuarantineRoutesLatestVisibleBytesNative(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + approvedCLI := approvedCLIContract(t, storeDir, "1.2.3") + mount := filepath.Join(home, "mount") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(mount, "session.jsonl") + original, _ := os.ReadFile(nativePath) + if err := os.WriteFile(target, original, 0o600); err != nil { + t.Fatal(err) + } + executeFS(t, []string{"fs", "migrate", "session", "--codex-home", home, "--store", storeDir, "--mount", mount, "--cli", approvedCLI, "--desktop-app", "none", "--apply"}) + state, err := managedState(storeDir, "session") + if err != nil { + t.Fatal(err) + } + managed, resolver, err := openManagedSession(context.Background(), storeDir, state) + if err != nil { + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + tail := []byte("{\"after_upgrade\":true}\n") + if _, err := writer.Append(context.Background(), tail); err != nil { + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + t.Fatal(err) + } + _ = writer.Close() + _ = resolver.Close() + unknownCLI := fakeCLI(t, "9.9.9") + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "service", "update-preflight", "--codex-home", home, "--store", storeDir, "--cli", unknownCLI, "--desktop-app", "none", "--apply-quarantine", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("update preflight: %v", err) + } + var result FSUpdatePreflightResult + if err := json.Unmarshal(output.Bytes(), &result); err != nil || !result.Decision.Quarantine || result.Decision.RequiresNativeFallback || result.QuarantinedSessions != 1 { + t.Fatalf("unexpected quarantine result: %#v err=%v output=%s", result, err, output.String()) + } + sessions, err := codex.LoadSessions(home) + if err != nil { + t.Fatal(err) + } + quarantineBytes, err := os.ReadFile(sessions[0].RolloutPath) + if err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), original...), tail...) + if !bytes.Equal(quarantineBytes, want) { + t.Fatalf("quarantine route is stale: got=%q want=%q", quarantineBytes, want) + } +} + func TestPackBuildAndDoctorCommandsUseFoldStore(t *testing.T) { home, storeDir, _ := fsFixture(t, true) var output bytes.Buffer @@ -324,11 +410,7 @@ func fsFixture(t *testing.T, archived bool) (string, string, string) { func approvedCLIContract(t *testing.T, storeDir string, version string) string { t.Helper() - cliPath := filepath.Join(t.TempDir(), "codex") - script := "#!/bin/sh\necho 'codex-cli " + version + "'\n" - if err := os.WriteFile(cliPath, []byte(script), 0o700); err != nil { - t.Fatal(err) - } + cliPath := fakeCLI(t, version) _, err := compat.Save(filepath.Join(storeDir, "compatibility"), compat.Contract{ Version: compat.ContractVersion, Platform: runtime.GOOS, ClientKind: "cli", ClientVersion: version, Operations: []compat.Operation{{Name: "read", Count: 1}}, @@ -340,6 +422,16 @@ func approvedCLIContract(t *testing.T, storeDir string, version string) string { return cliPath } +func fakeCLI(t *testing.T, version string) string { + t.Helper() + cliPath := filepath.Join(t.TempDir(), "codex") + script := "#!/bin/sh\necho 'codex-cli " + version + "'\n" + if err := os.WriteFile(cliPath, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + return cliPath +} + func executeFS(t *testing.T, args []string) { t.Helper() root := NewRootCommand() diff --git a/internal/mountfs/filesystem_test.go b/internal/mountfs/filesystem_test.go index 2b0c762..7c283b2 100644 --- a/internal/mountfs/filesystem_test.go +++ b/internal/mountfs/filesystem_test.go @@ -134,6 +134,9 @@ func TestFilesystemUpsertChangesNewOpensWithoutInvalidatingExistingHandles(t *te } func TestMountWithoutFuseBuildReturnsPrerequisiteError(t *testing.T) { + if Available() { + t.Fatal("default build should report the FUSE host as unavailable") + } err := Mount(context.Background(), HostOptions{MountPoint: t.TempDir(), Filesystem: New()}) if !errors.Is(err, ErrPrerequisite) { t.Fatalf("Mount error = %v, want ErrPrerequisite", err) diff --git a/internal/mountfs/host_cgofuse.go b/internal/mountfs/host_cgofuse.go index e596b4e..8deb862 100644 --- a/internal/mountfs/host_cgofuse.go +++ b/internal/mountfs/host_cgofuse.go @@ -18,6 +18,8 @@ type fuseFilesystem struct { core *Filesystem } +func Available() bool { return true } + func (f *fuseFilesystem) Getattr(name string, stat *fuse.Stat_t, _ uint64) int { attribute, errno := f.core.Getattr(name) if errno != 0 { diff --git a/internal/mountfs/host_stub.go b/internal/mountfs/host_stub.go index 6006738..2a366c8 100644 --- a/internal/mountfs/host_stub.go +++ b/internal/mountfs/host_stub.go @@ -4,4 +4,6 @@ package mountfs import "context" +func Available() bool { return false } + func mountHost(context.Context, HostOptions) error { return ErrPrerequisite } diff --git a/internal/service/mount_probe_darwin.go b/internal/service/mount_probe_darwin.go new file mode 100644 index 0000000..3476022 --- /dev/null +++ b/internal/service/mount_probe_darwin.go @@ -0,0 +1,27 @@ +//go:build darwin + +package service + +import ( + "errors" + "path/filepath" + "strings" + + "golang.org/x/sys/unix" +) + +func defaultMountProbe(path string) error { + var stat unix.Statfs_t + if err := unix.Statfs(path, &stat); err != nil { + return err + } + mountedAt := unix.ByteSliceToString(stat.Mntonname[:]) + filesystem := strings.ToLower(unix.ByteSliceToString(stat.Fstypename[:])) + if filepath.Clean(mountedAt) != filepath.Clean(path) { + return errors.New("path is not a mount root") + } + if !strings.Contains(filesystem, "fuse") { + return errors.New("mount root is not backed by FUSE") + } + return nil +} diff --git a/internal/service/mount_probe_other.go b/internal/service/mount_probe_other.go new file mode 100644 index 0000000..51f5d13 --- /dev/null +++ b/internal/service/mount_probe_other.go @@ -0,0 +1,19 @@ +//go:build !darwin + +package service + +import ( + "errors" + "os" +) + +func defaultMountProbe(path string) error { + info, err := os.Stat(path) + if err != nil { + return err + } + if !info.IsDir() { + return errors.New("mount path is not a directory") + } + return nil +} diff --git a/internal/service/service.go b/internal/service/service.go new file mode 100644 index 0000000..e290a36 --- /dev/null +++ b/internal/service/service.go @@ -0,0 +1,248 @@ +package service + +import ( + "bytes" + "context" + "encoding/xml" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/jstar0/codexfold/internal/compat" + "github.com/jstar0/codexfold/internal/fsctl" +) + +type Options struct { + Label string + BinaryPath string + CodexHome string + StoreDir string + MountPoint string + StdoutPath string + StderrPath string +} + +type InstallResult struct { + Path string `json:"path"` + DryRun bool `json:"dry_run"` + Bytes int `json:"bytes"` +} + +type Runner interface { + Run(context.Context, string, ...string) ([]byte, error) +} + +type ExecRunner struct{} + +func (ExecRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) { + return exec.CommandContext(ctx, name, args...).CombinedOutput() +} + +type Manager struct { + UID int + Runner Runner + MountProbe func(string) error +} + +type Status struct { + DaemonRunning bool `json:"daemon_running"` + MountHealthy bool `json:"mount_healthy"` + DaemonError string `json:"daemon_error,omitempty"` + MountError string `json:"mount_error,omitempty"` +} + +type UpdateInput struct { + Capability fsctl.Capability + DoctorHealthy bool + Compatibility compat.Evaluation + NativeFallbackReady bool + Automatic bool + ExplicitPromotion bool +} + +type UpdateDecision struct { + Allowed bool `json:"allowed"` + Quarantine bool `json:"quarantine"` + RequiresNativeFallback bool `json:"requires_native_fallback"` + Reason string `json:"reason,omitempty"` +} + +func RenderLaunchd(options Options) ([]byte, error) { + if err := validateOptions(options); err != nil { + return nil, err + } + arguments := []string{ + options.BinaryPath, "fs", "serve", "--apply", "--foreground=true", + "--codex-home", options.CodexHome, "--store", options.StoreDir, "--mount", options.MountPoint, + } + var output bytes.Buffer + output.WriteString("\n") + output.WriteString("\n") + output.WriteString("\n\n") + writePlistString(&output, "Label", options.Label) + output.WriteString(" ProgramArguments\n \n") + for _, argument := range arguments { + output.WriteString(" ") + _ = xml.EscapeText(&output, []byte(argument)) + output.WriteString("\n") + } + output.WriteString(" \n") + writePlistString(&output, "StandardOutPath", options.StdoutPath) + writePlistString(&output, "StandardErrorPath", options.StderrPath) + output.WriteString(" RunAtLoad\n \n") + output.WriteString(" KeepAlive\n \n") + output.WriteString(" ProcessType\n Background\n") + output.WriteString("\n\n") + return output.Bytes(), nil +} + +func WriteDefinition(path string, definition []byte, apply bool) (InstallResult, error) { + if !filepath.IsAbs(path) || len(definition) == 0 { + return InstallResult{}, errors.New("absolute definition path and non-empty definition are required") + } + result := InstallResult{Path: filepath.Clean(path), DryRun: !apply, Bytes: len(definition)} + if !apply { + return result, nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return InstallResult{}, err + } + temporary, err := os.CreateTemp(filepath.Dir(path), ".launchd-*.tmp") + if err != nil { + return InstallResult{}, err + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return InstallResult{}, err + } + if _, err := temporary.Write(definition); err != nil { + _ = temporary.Close() + return InstallResult{}, err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return InstallResult{}, err + } + if err := temporary.Close(); err != nil { + return InstallResult{}, err + } + if err := os.Rename(temporaryPath, path); err != nil { + return InstallResult{}, err + } + return result, nil +} + +func (m Manager) Bootstrap(ctx context.Context, plistPath string) error { + if !filepath.IsAbs(plistPath) { + return errors.New("absolute launchd plist path is required") + } + _, err := m.runner().Run(ctx, "launchctl", "bootstrap", m.domain(), plistPath) + return err +} + +func (m Manager) Bootout(ctx context.Context, plistPath string) error { + if !filepath.IsAbs(plistPath) { + return errors.New("absolute launchd plist path is required") + } + _, err := m.runner().Run(ctx, "launchctl", "bootout", m.domain(), plistPath) + return err +} + +func (m Manager) Kickstart(ctx context.Context, label string) error { + if !safeLabel(label) { + return errors.New("safe launchd label is required") + } + _, err := m.runner().Run(ctx, "launchctl", "kickstart", "-k", m.domain()+"/"+label) + return err +} + +func (m Manager) Status(ctx context.Context, label string, mountPoint string) Status { + result := Status{} + if _, err := m.runner().Run(ctx, "launchctl", "print", m.domain()+"/"+label); err != nil { + result.DaemonError = err.Error() + } else { + result.DaemonRunning = true + } + probe := m.MountProbe + if probe == nil { + probe = defaultMountProbe + } + if err := probe(mountPoint); err != nil { + result.MountError = err.Error() + } else { + result.MountHealthy = true + } + return result +} + +func EvaluateUpdate(input UpdateInput) UpdateDecision { + if !input.DoctorHealthy { + return UpdateDecision{Reason: "filesystem doctor is not healthy"} + } + if input.Compatibility.Quarantine || !input.Compatibility.Approved { + return UpdateDecision{Quarantine: true, RequiresNativeFallback: !input.NativeFallbackReady, Reason: "installed client version is not approved"} + } + if input.Automatic && (input.Capability == fsctl.FSEnginePreview || input.Capability == fsctl.PlatformCanary) { + return UpdateDecision{Reason: "automatic updates are disabled before platform production readiness"} + } + if (input.Capability == fsctl.FSEnginePreview || input.Capability == fsctl.PlatformCanary) && !input.ExplicitPromotion { + return UpdateDecision{Reason: "preview and canary updates require explicit promotion"} + } + return UpdateDecision{Allowed: true} +} + +func (m Manager) runner() Runner { + if m.Runner != nil { + return m.Runner + } + return ExecRunner{} +} + +func (m Manager) domain() string { + uid := m.UID + if uid <= 0 { + uid = os.Getuid() + } + return fmt.Sprintf("gui/%d", uid) +} + +func validateOptions(options Options) error { + if !safeLabel(options.Label) { + return errors.New("safe launchd label is required") + } + for name, path := range map[string]string{ + "binary": options.BinaryPath, "Codex home": options.CodexHome, "store": options.StoreDir, + "mount": options.MountPoint, "stdout": options.StdoutPath, "stderr": options.StderrPath, + } { + if !filepath.IsAbs(path) { + return fmt.Errorf("%s path must be absolute", name) + } + } + return nil +} + +func safeLabel(label string) bool { + if label == "" || strings.HasPrefix(label, ".") || strings.HasSuffix(label, ".") { + return false + } + for _, character := range label { + if character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' || character >= '0' && character <= '9' || character == '.' || character == '-' { + continue + } + return false + } + return true +} + +func writePlistString(output *bytes.Buffer, key string, value string) { + output.WriteString(" ") + _ = xml.EscapeText(output, []byte(key)) + output.WriteString("\n ") + _ = xml.EscapeText(output, []byte(value)) + output.WriteString("\n") +} diff --git a/internal/service/service_test.go b/internal/service/service_test.go new file mode 100644 index 0000000..00dc6b1 --- /dev/null +++ b/internal/service/service_test.go @@ -0,0 +1,106 @@ +package service + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/jstar0/codexfold/internal/compat" + "github.com/jstar0/codexfold/internal/fsctl" +) + +func TestRenderLaunchdUsesAbsoluteArgumentsAndContainsNoSessionContent(t *testing.T) { + root := t.TempDir() + definition, err := RenderLaunchd(Options{ + Label: "com.codexfold.fs", BinaryPath: filepath.Join(root, "bin", "codexfold"), + CodexHome: filepath.Join(root, "codex"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "logs", "stdout.log"), + StderrPath: filepath.Join(root, "logs", "stderr.log"), + }) + if err != nil { + t.Fatalf("RenderLaunchd: %v", err) + } + text := string(definition) + for _, required := range []string{"fs", "serve", "--apply", filepath.Join(root, "store"), filepath.Join(root, "mount")} { + if !strings.Contains(text, required) { + t.Fatalf("definition missing %q:\n%s", required, text) + } + } + if strings.Contains(text, "session_meta") || strings.Contains(text, "rollout") { + t.Fatalf("definition contains session content: %s", text) + } + if runtime.GOOS == "darwin" { + path := filepath.Join(root, "service.plist") + if err := os.WriteFile(path, definition, 0o600); err != nil { + t.Fatal(err) + } + if output, err := exec.Command("/usr/bin/plutil", "-lint", path).CombinedOutput(); err != nil { + t.Fatalf("plutil rejected definition: %v\n%s", err, output) + } + } + if _, err := RenderLaunchd(Options{Label: "com.codexfold.fs", BinaryPath: "codexfold", CodexHome: root, StoreDir: root, MountPoint: root, StdoutPath: filepath.Join(root, "out"), StderrPath: filepath.Join(root, "err")}); err == nil { + t.Fatal("relative binary path should be rejected") + } +} + +func TestManagerUsesOnlyPerUserLaunchctlAndSeparatesDaemonFromMount(t *testing.T) { + root := t.TempDir() + runner := &recordingRunner{outputs: map[string][]byte{"launchctl print gui/501/com.codexfold.fs": []byte("running")}} + manager := Manager{UID: 501, Runner: runner, MountProbe: func(string) error { return errors.New("mount unavailable") }} + plist := filepath.Join(root, "com.codexfold.fs.plist") + if err := os.WriteFile(plist, []byte("plist"), 0o600); err != nil { + t.Fatal(err) + } + if err := manager.Bootstrap(context.Background(), plist); err != nil { + t.Fatalf("Bootstrap: %v", err) + } + if err := manager.Kickstart(context.Background(), "com.codexfold.fs"); err != nil { + t.Fatalf("Kickstart: %v", err) + } + status := manager.Status(context.Background(), "com.codexfold.fs", filepath.Join(root, "mount")) + if !status.DaemonRunning || status.MountHealthy { + t.Fatalf("status did not separate daemon and mount: %#v", status) + } + joined := strings.Join(runner.calls, "\n") + if strings.Contains(joined, "sudo") || !strings.Contains(joined, "launchctl bootstrap gui/501") || !strings.Contains(joined, "launchctl kickstart -k gui/501/com.codexfold.fs") { + t.Fatalf("unexpected lifecycle commands:\n%s", joined) + } +} + +func TestEvaluateUpdateQuarantinesUnknownVersionsAndRejectsPreviewAutomation(t *testing.T) { + unknown := EvaluateUpdate(UpdateInput{Capability: fsctl.StorageEngine, DoctorHealthy: true, Compatibility: compat.Evaluation{Quarantine: true}, NativeFallbackReady: false}) + if unknown.Allowed || !unknown.Quarantine || !unknown.RequiresNativeFallback { + t.Fatalf("unknown version was not quarantined: %#v", unknown) + } + ready := EvaluateUpdate(UpdateInput{Capability: fsctl.StorageEngine, DoctorHealthy: true, Compatibility: compat.Evaluation{Quarantine: true}, NativeFallbackReady: true}) + if ready.Allowed || !ready.Quarantine || ready.RequiresNativeFallback { + t.Fatalf("quarantine should remain blocked after fallback: %#v", ready) + } + automatic := EvaluateUpdate(UpdateInput{Capability: fsctl.FSEnginePreview, DoctorHealthy: true, Compatibility: compat.Evaluation{Approved: true}, Automatic: true}) + if automatic.Allowed { + t.Fatalf("preview automatic update should be rejected: %#v", automatic) + } + manual := EvaluateUpdate(UpdateInput{Capability: fsctl.FSEnginePreview, DoctorHealthy: true, Compatibility: compat.Evaluation{Approved: true}, ExplicitPromotion: true}) + if !manual.Allowed { + t.Fatalf("explicit preview promotion should pass: %#v", manual) + } +} + +type recordingRunner struct { + calls []string + outputs map[string][]byte +} + +func (r *recordingRunner) Run(_ context.Context, name string, args ...string) ([]byte, error) { + call := strings.Join(append([]string{name}, args...), " ") + r.calls = append(r.calls, call) + if output, ok := r.outputs[call]; ok { + return output, nil + } + return nil, nil +} From a1ac76eb5523450dad232e1cc897edc1a5a7b226 Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 01:54:37 +0800 Subject: [PATCH 11/33] test: validate transparent filesystem engine preview --- docs/validation-fs-preview.md | 58 +++++++++ docs/validation-macos-canary.md | 28 +++++ internal/cli/fs.go | 4 +- internal/cli/fs_service.go | 5 +- internal/cli/fs_test.go | 4 + internal/testfs/corpus.go | 150 ++++++++++++++++++++++++ internal/testfs/corpus_test.go | 201 ++++++++++++++++++++++++++++++++ internal/testfs/faults.go | 30 +++++ internal/testfs/large_test.go | 99 ++++++++++++++++ internal/testfs/rss_darwin.go | 13 +++ internal/testfs/rss_linux.go | 13 +++ internal/testfs/rss_other.go | 5 + scripts/test-cross-platform.sh | 15 +++ 13 files changed, 621 insertions(+), 4 deletions(-) create mode 100644 docs/validation-fs-preview.md create mode 100644 docs/validation-macos-canary.md create mode 100644 internal/testfs/corpus.go create mode 100644 internal/testfs/corpus_test.go create mode 100644 internal/testfs/faults.go create mode 100644 internal/testfs/large_test.go create mode 100644 internal/testfs/rss_darwin.go create mode 100644 internal/testfs/rss_linux.go create mode 100644 internal/testfs/rss_other.go create mode 100755 scripts/test-cross-platform.sh diff --git a/docs/validation-fs-preview.md b/docs/validation-fs-preview.md new file mode 100644 index 0000000..5f7d2ec --- /dev/null +++ b/docs/validation-fs-preview.md @@ -0,0 +1,58 @@ +# Filesystem Engine Preview Validation + +This document records platform-neutral storage and session-engine evidence. It does not claim a real FUSE adapter, real Codex compatibility, a macOS canary, or production readiness. + +## Required Commands + +```sh +./scripts/test-cross-platform.sh +CODEXFOLD_RUN_LARGE_TEST=1 go test ./internal/testfs -run TestLargePreviewBenchmark -count=1 -v -timeout 30m +``` + +## Covered Behavior + +- Deterministic synthetic forks with shared history, independently different tails, repeated fields, repeated records, non-prefix duplicate content, a multi-block field, invalid JSONL, and an empty session. +- Complete SHA comparison and 10,000 deterministic random reads for every synthetic session. +- 100,000 append operations followed by `fsync`, reopen, and exact current-byte verification. +- Concurrent readers with one writer, random write copy-on-write, truncate, and generation-safe reopen. +- Journal interruption and recovery tests in `internal/vfs`, packed corruption tests in `internal/pack`, and optimistic route-race tests in `internal/codex`. +- Linux and Windows non-CGO compile-only checks. Cross-compiled test binaries are not executed on macOS. +- A generated 758 MiB rollout read entirely from packs after the loose-object directory is taken offline. + +## Status Boundary + +Passing these checks can justify only `fs-engine-preview`. The following remain separate authorization-gated evidence: + +- Root `fs_usage` traces from real Codex Desktop and CLI versions. +- A compiled and mounted `fuse && cgo` adapter with macFUSE authorized by the user. +- Real archived-session shadow and retained-source canaries. +- Desktop click, resume, send, tool, fork, archive, restart, sleep/wake, rollback, and upgrade quarantine behavior. +- Seven incident-free retention days before `production-ready:macos`. + +## Latest Result + +Run on 2026-07-12 using an Apple M4 Pro MacBook Pro with 12 CPU cores and 48 GiB RAM, macOS 26.5.1, and Go 1.26.4. + +The 758 MiB source was deliberately highly repetitive. The first benchmark followed Fold, Pack, and Shadow in the same process, so both native and packed data benefited from system caching. These values are a deterministic engine gate, not a claim about real Codex cold-cache workloads. + +| Metric | First pass | Warm pass | +| --- | ---: | ---: | +| Native sequential throughput | 16.03 GB/s | 15.87 GB/s | +| Virtual sequential throughput | 60.73 GB/s | 58.15 GB/s | +| Virtual/native ratio | 3.79x | 3.66x | +| Random read p50 | 0.584 us | 0.583 us | +| Random read p95 | 0.959 us | 0.958 us | +| Random read p99 | 1.625 us | 1.542 us | + +Additional results: + +- Fold: 35.23 s. +- Pack build: 0.055 s. +- Complete SHA plus 10,000 random-range shadow: 3.31 s. +- Go system memory: 160.04 MiB. +- Maximum RSS: 160.02 MiB. +- Configured decompressed block cache: 128 MiB. +- Loose-object directory offline during shadow and both benchmark passes: yes. +- 100,000 one-byte append calls followed by `fsync`: 2.18 s in the normal test build. + +The platform-neutral gates pass and justify `fs-engine-preview`. This result does not satisfy any Task 11 real-adapter or real-Codex gate. diff --git a/docs/validation-macos-canary.md b/docs/validation-macos-canary.md new file mode 100644 index 0000000..89912cb --- /dev/null +++ b/docs/validation-macos-canary.md @@ -0,0 +1,28 @@ +# macOS Adapter And Canary Validation + +## Current Status + +Blocked before adapter compilation and real Codex routing. No real session route has been changed. + +Observed on 2026-07-12: + +- Codex desktop bundle: `26.707.41301` build `5103`. +- Desktop-bundled CLI: `codex-cli 0.144.0-alpha.4`. +- CLI resolved from `PATH`: `codex-cli 0.142.5`. +- macFUSE or osxfuse package receipt: not present. +- macFUSE or osxfuse filesystem bundle: not present. +- `go build -tags fuse ./cmd/codexfold`: blocked by missing `fuse.h`. +- Root `fs_usage` trace: not attempted because elevation has not been explicitly authorized. + +## Required Authorization Sequence + +1. Explicitly authorize a root `fs_usage` capture for the installed desktop-bundled CLI, the `PATH` CLI, and Codex Desktop workflows. The captured contract stores only sanitized operation names, counts, safe flags, and the trace digest. +2. Reconcile every observed operation with the platform-neutral filesystem. Unsupported rename, unlink, lock, mapping, watcher, or open-mode behavior blocks the adapter. +3. Explicitly authorize macFUSE installation and its system extension. The project must not self-elevate or install it implicitly. +4. Build with `-tags fuse`, mount only a generated fixture namespace, and pass the exact-byte, random-read, append, random-write, truncate, fsync, rename/unlink-policy, daemon-kill, and remount tests. +5. Shadow 5 to 10 archived real sessions without changing routes or removing native files. +6. Route retained-source canaries only after trace, adapter, doctor, shadow, compatibility, and explicit apply gates pass. +7. Exercise desktop click, CLI resume, send, tool use, fork, archive, unarchive, daemon termination, remount, sleep/wake, host restart, rollback, and unknown-version quarantine. +8. Keep status at `platform-canary` for seven incident-free days before considering `production-ready:macos`. + +Fixture tests and `fs-engine-preview` evidence do not satisfy any item above. diff --git a/internal/cli/fs.go b/internal/cli/fs.go index 8e58d86..1f8ee0b 100644 --- a/internal/cli/fs.go +++ b/internal/cli/fs.go @@ -98,7 +98,7 @@ func newFSStatusCommand() *cobra.Command { Short: "Report the highest verified filesystem capability", Args: cobra.NoArgs, RunE: func(command *cobra.Command, _ []string) error { - status, err := fsctl.NewStatus(fsctl.StorageEngine, runtime.GOOS) + status, err := fsctl.NewStatus(verifiedCapability(), runtime.GOOS) if err != nil { return err } @@ -853,6 +853,8 @@ func defaultMountPoint(home string, explicit string) string { return filepath.Join(home, "fold-fs") } +func verifiedCapability() fsctl.Capability { return fsctl.FSEnginePreview } + func waitForTarget(ctx context.Context, target string, timeout time.Duration) (vfs.NativeFile, error) { if timeout <= 0 { timeout = 5 * time.Second diff --git a/internal/cli/fs_service.go b/internal/cli/fs_service.go index eb8635c..8e250aa 100644 --- a/internal/cli/fs_service.go +++ b/internal/cli/fs_service.go @@ -12,7 +12,6 @@ import ( "runtime" "github.com/jstar0/codexfold/internal/codex" - "github.com/jstar0/codexfold/internal/fsctl" "github.com/jstar0/codexfold/internal/mountfs" "github.com/jstar0/codexfold/internal/service" "github.com/jstar0/codexfold/internal/vfs" @@ -194,7 +193,7 @@ func newFSServiceUpdatePreflightCommand() *cobra.Command { if err != nil { fallbackReady = false } - decision := service.EvaluateUpdate(service.UpdateInput{Capability: fsctl.StorageEngine, DoctorHealthy: doctorErr == nil, Compatibility: compatibilityResult.Evaluation, NativeFallbackReady: fallbackReady, Automatic: automatic, ExplicitPromotion: promote}) + decision := service.EvaluateUpdate(service.UpdateInput{Capability: verifiedCapability(), DoctorHealthy: doctorErr == nil, Compatibility: compatibilityResult.Evaluation, NativeFallbackReady: fallbackReady, Automatic: automatic, ExplicitPromotion: promote}) result := FSUpdatePreflightResult{DoctorHealthy: doctorErr == nil, Compatibility: compatibilityResult, Decision: decision} if decision.Quarantine && decision.RequiresNativeFallback && applyQuarantine { count, err := quarantineManagedRoutes(command.Context(), home, store) @@ -202,7 +201,7 @@ func newFSServiceUpdatePreflightCommand() *cobra.Command { return err } result.QuarantinedSessions = count - result.Decision = service.EvaluateUpdate(service.UpdateInput{Capability: fsctl.StorageEngine, DoctorHealthy: doctorErr == nil, Compatibility: compatibilityResult.Evaluation, NativeFallbackReady: true, Automatic: automatic, ExplicitPromotion: promote}) + result.Decision = service.EvaluateUpdate(service.UpdateInput{Capability: verifiedCapability(), DoctorHealthy: doctorErr == nil, Compatibility: compatibilityResult.Evaluation, NativeFallbackReady: true, Automatic: automatic, ExplicitPromotion: promote}) } if jsonOutput { return writeJSON(command, result) diff --git a/internal/cli/fs_test.go b/internal/cli/fs_test.go index d476a18..f03da84 100644 --- a/internal/cli/fs_test.go +++ b/internal/cli/fs_test.go @@ -193,6 +193,10 @@ func TestFSStatusDoesNotClaimTransparentReadiness(t *testing.T) { if bytes.Contains(output.Bytes(), []byte("production-ready")) || bytes.Contains(output.Bytes(), []byte("platform-canary")) { t.Fatalf("status overclaimed readiness: %s", output.String()) } + var status fsctl.Status + if err := json.Unmarshal(output.Bytes(), &status); err != nil || status.Capability != fsctl.FSEnginePreview { + t.Fatalf("status did not report the verified engine preview: %#v err=%v", status, err) + } } func TestFSCompatibilityApprovesOnlyExactInstalledClientContract(t *testing.T) { diff --git a/internal/testfs/corpus.go b/internal/testfs/corpus.go new file mode 100644 index 0000000..71f3da9 --- /dev/null +++ b/internal/testfs/corpus.go @@ -0,0 +1,150 @@ +package testfs + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +type Options struct { + LargeFieldBytes int + RepeatedRecords int +} + +type Session struct { + ID string `json:"id"` + Path string `json:"path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` +} + +type Corpus struct { + Root string `json:"root"` + Sessions []Session `json:"sessions"` +} + +func Generate(root string, options Options) (Corpus, error) { + if root == "" { + return Corpus{}, errors.New("corpus root is required") + } + if options.LargeFieldBytes <= 0 { + options.LargeFieldBytes = 768 << 10 + } + if options.RepeatedRecords <= 0 { + options.RepeatedRecords = 64 + } + if err := os.MkdirAll(root, 0o700); err != nil { + return Corpus{}, err + } + large := make([]byte, options.LargeFieldBytes) + for index := range large { + large[index] = byte('a' + index%23) + } + repeated := []byte("{\"type\":\"event\",\"payload\":\"exact-repeated-record\"}\n") + prefixPath := filepath.Join(root, "shared-prefix.bin") + prefix, err := os.Create(prefixPath) + if err != nil { + return Corpus{}, err + } + writer := bufio.NewWriterSize(prefix, 1<<20) + _, _ = writer.WriteString("{\"type\":\"session_meta\",\"id\":\"synthetic\"}\n") + _, _ = writer.WriteString("{\"type\":\"large\",\"payload\":\"") + _, _ = writer.Write(large) + _, _ = writer.WriteString("\"}\n") + for index := 0; index < options.RepeatedRecords; index++ { + _, _ = writer.Write(repeated) + } + _, _ = writer.WriteString("not-json-but-valid-rollout-bytes\n") + if err := writer.Flush(); err != nil { + _ = prefix.Close() + return Corpus{}, err + } + if err := prefix.Sync(); err != nil { + _ = prefix.Close() + return Corpus{}, err + } + if err := prefix.Close(); err != nil { + return Corpus{}, err + } + prefixData, err := os.ReadFile(prefixPath) + if err != nil { + return Corpus{}, err + } + definitions := []struct { + id string + body []byte + }{ + {id: "fork-a", body: append(append([]byte(nil), prefixData...), []byte("{\"tail\":\"a\"}\n")...)}, + {id: "fork-b", body: append(append([]byte(nil), prefixData...), []byte("{\"tail\":\"b\"}\n")...)}, + {id: "reordered", body: append(append([]byte(nil), repeated...), append(prefixData, repeated...)...)}, + {id: "empty", body: []byte{}}, + } + corpus := Corpus{Root: root, Sessions: make([]Session, 0, len(definitions))} + for _, definition := range definitions { + path := filepath.Join(root, definition.id+".jsonl") + if err := os.WriteFile(path, definition.body, 0o600); err != nil { + return Corpus{}, err + } + digest := sha256.Sum256(definition.body) + corpus.Sessions = append(corpus.Sessions, Session{ID: definition.id, Path: path, Bytes: int64(len(definition.body)), SHA256: hex.EncodeToString(digest[:])}) + } + _ = os.Remove(prefixPath) + return corpus, nil +} + +func GenerateRollout(path string, targetBytes int64) (Session, error) { + if path == "" || targetBytes < 0 { + return Session{}, errors.New("rollout path and non-negative target size are required") + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return Session{}, err + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return Session{}, err + } + hasher := sha256.New() + writer := bufio.NewWriterSize(io.MultiWriter(file, hasher), 4<<20) + record := []byte("{\"type\":\"event\",\"payload\":\"synthetic-repeat-0123456789abcdefghijklmnopqrstuvwxyz\"}\n") + var written int64 + for targetBytes-written >= int64(len(record)) { + n, writeErr := writer.Write(record) + written += int64(n) + if writeErr != nil { + _ = file.Close() + return Session{}, writeErr + } + } + if remaining := targetBytes - written; remaining > 0 { + padding := make([]byte, remaining) + for index := range padding { + padding[index] = byte('A' + index%26) + } + n, writeErr := writer.Write(padding) + written += int64(n) + if writeErr != nil { + _ = file.Close() + return Session{}, writeErr + } + } + if err := writer.Flush(); err != nil { + _ = file.Close() + return Session{}, err + } + if err := file.Sync(); err != nil { + _ = file.Close() + return Session{}, err + } + if err := file.Close(); err != nil { + return Session{}, err + } + if written != targetBytes { + return Session{}, fmt.Errorf("generated %d bytes, want %d", written, targetBytes) + } + return Session{ID: "large", Path: path, Bytes: written, SHA256: hex.EncodeToString(hasher.Sum(nil))}, nil +} diff --git a/internal/testfs/corpus_test.go b/internal/testfs/corpus_test.go new file mode 100644 index 0000000..a137907 --- /dev/null +++ b/internal/testfs/corpus_test.go @@ -0,0 +1,201 @@ +package testfs + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "math/rand" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/jstar0/codexfold/internal/codex" + "github.com/jstar0/codexfold/internal/fold" + "github.com/jstar0/codexfold/internal/fsctl" + "github.com/jstar0/codexfold/internal/pack" + "github.com/jstar0/codexfold/internal/vfs" +) + +func TestGenerateIsDeterministicAndContainsForkAndNonPrefixDuplication(t *testing.T) { + first, err := Generate(filepath.Join(t.TempDir(), "first"), Options{}) + if err != nil { + t.Fatal(err) + } + second, err := Generate(filepath.Join(t.TempDir(), "second"), Options{}) + if err != nil { + t.Fatal(err) + } + if len(first.Sessions) != len(second.Sessions) || len(first.Sessions) < 4 { + t.Fatalf("unexpected corpora: %#v %#v", first, second) + } + for index := range first.Sessions { + if first.Sessions[index].ID != second.Sessions[index].ID || first.Sessions[index].SHA256 != second.Sessions[index].SHA256 || first.Sessions[index].Bytes != second.Sessions[index].Bytes { + t.Fatalf("corpus is not deterministic at %d: %#v %#v", index, first.Sessions[index], second.Sessions[index]) + } + } + if first.Sessions[0].SHA256 == first.Sessions[1].SHA256 || first.Sessions[2].Bytes <= first.Sessions[0].Bytes { + t.Fatalf("fork and reordered fixtures are not distinct: %#v", first.Sessions) + } +} + +func TestPackedCorpusShadowRandomReadsAndWritableSessionStress(t *testing.T) { + root := t.TempDir() + corpus, err := Generate(filepath.Join(root, "corpus"), Options{LargeFieldBytes: 768 << 10, RepeatedRecords: 128}) + if err != nil { + t.Fatal(err) + } + store := filepath.Join(root, "store") + for _, fixture := range corpus.Sessions { + _, err := fold.Fold(context.Background(), codex.Session{ID: fixture.ID, RolloutPath: fixture.Path, Archived: true}, fold.FoldOptions{StoreDir: store, Apply: true, FieldThreshold: 32}) + if err != nil { + t.Fatalf("fold %s: %v", fixture.ID, err) + } + } + if _, err := pack.Build(context.Background(), store, pack.BuildOptions{}); err != nil { + t.Fatalf("pack build: %v", err) + } + resolver, err := pack.Open(store, pack.OpenOptions{CacheBytes: 16 << 20}) + if err != nil { + t.Fatal(err) + } + defer resolver.Close() + for _, fixture := range corpus.Sessions { + manifest, err := fold.LoadManifest(store, fixture.ID) + if err != nil { + t.Fatal(err) + } + view, err := vfs.NewView(manifest, resolver) + if err != nil { + t.Fatal(err) + } + shadow, err := fsctl.Shadow(context.Background(), fixture.Path, view, fsctl.ShadowOptions{BlockBytes: 64 << 10, RandomReads: 10000, Seed: 42}) + if err != nil || !shadow.Verified { + t.Fatalf("shadow %s: %#v err=%v", fixture.ID, shadow, err) + } + } + fixture := corpus.Sessions[0] + manifest, _ := fold.LoadManifest(store, fixture.ID) + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{Root: store, ManifestPath: fold.ManifestPath(store, fixture.ID), Manifest: manifest, Reader: resolver, NativeSnapshot: vfs.NativeFile{Path: fixture.Path, Bytes: fixture.Bytes, SHA256: fixture.SHA256}}) + if err != nil { + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + start := time.Now() + for index := 0; index < 100000; index++ { + if _, err := writer.Append(context.Background(), []byte("x")); err != nil { + t.Fatalf("append %d: %v", index, err) + } + } + if err := writer.Sync(); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + if testing.Verbose() { + t.Logf("100000 appends: %s", time.Since(start)) + } + current, err := managed.MaterializeCurrent(context.Background(), filepath.Join(root, "current.jsonl"), false) + if err != nil || current.Bytes != fixture.Bytes+100000 { + t.Fatalf("append result: %#v err=%v", current, err) + } + verifyConcurrentReadAndWrite(t, managed) + writer, err = managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + if _, err := writer.WriteAt(context.Background(), []byte("PATCH"), 7); err != nil { + t.Fatal(err) + } + if err := writer.Truncate(context.Background(), current.Bytes/2); err != nil { + t.Fatal(err) + } + _ = writer.Close() + info, err := managed.VisibleInfo() + if err != nil || info.Size != current.Bytes/2 { + t.Fatalf("COW/truncate result: %#v err=%v", info, err) + } +} + +func TestGenerateRolloutWritesExactRequestedBytes(t *testing.T) { + fixture, err := GenerateRollout(filepath.Join(t.TempDir(), "large.jsonl"), (17<<20)+37) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(fixture.Path) + if err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(data) + if int64(len(data)) != fixture.Bytes || hex.EncodeToString(digest[:]) != fixture.SHA256 { + t.Fatalf("generated rollout metadata differs: %#v", fixture) + } +} + +func TestFaultHookFiresExactlyOnceAtRequestedPhase(t *testing.T) { + faults := NewFaults("state-publish") + if err := faults.Hook("prepare"); err != nil { + t.Fatal(err) + } + if err := faults.Hook("state-publish"); err == nil || !faults.Fired() { + t.Fatalf("fault did not fire: %v", err) + } + if err := faults.Hook("state-publish"); err != nil { + t.Fatalf("fault fired twice: %v", err) + } +} + +func verifyConcurrentReadAndWrite(t *testing.T, managed *vfs.Session) { + t.Helper() + writer, err := managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + var wait sync.WaitGroup + errorsSeen := make(chan error, 5) + for worker := 0; worker < 5; worker++ { + wait.Add(1) + go func(seed int64) { + defer wait.Done() + random := rand.New(rand.NewSource(seed)) + for index := 0; index < 1000; index++ { + reader, err := managed.OpenReader() + if err != nil { + errorsSeen <- err + return + } + if reader.Size() > 0 { + offset := random.Int63n(reader.Size()) + buffer := make([]byte, 1) + if _, err := reader.ReadAt(context.Background(), buffer, offset); err != nil && !errors.Is(err, io.EOF) { + _ = reader.Close() + errorsSeen <- err + return + } + } + _ = reader.Close() + } + }(int64(worker + 1)) + } + for index := 0; index < 1000; index++ { + if _, err := writer.Append(context.Background(), []byte("y")); err != nil { + t.Fatal(err) + } + } + if err := writer.Sync(); err != nil { + t.Fatal(err) + } + _ = writer.Close() + wait.Wait() + close(errorsSeen) + for err := range errorsSeen { + t.Fatal(err) + } +} diff --git a/internal/testfs/faults.go b/internal/testfs/faults.go new file mode 100644 index 0000000..574f225 --- /dev/null +++ b/internal/testfs/faults.go @@ -0,0 +1,30 @@ +package testfs + +import ( + "errors" + "sync" +) + +type Faults struct { + mu sync.Mutex + phase string + fired bool +} + +func NewFaults(phase string) *Faults { return &Faults{phase: phase} } + +func (f *Faults) Hook(phase string) error { + f.mu.Lock() + defer f.mu.Unlock() + if !f.fired && phase == f.phase { + f.fired = true + return errors.New("injected fault at " + phase) + } + return nil +} + +func (f *Faults) Fired() bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.fired +} diff --git a/internal/testfs/large_test.go b/internal/testfs/large_test.go new file mode 100644 index 0000000..a14032e --- /dev/null +++ b/internal/testfs/large_test.go @@ -0,0 +1,99 @@ +package testfs + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/jstar0/codexfold/internal/codex" + "github.com/jstar0/codexfold/internal/fold" + "github.com/jstar0/codexfold/internal/fsctl" + "github.com/jstar0/codexfold/internal/pack" + "github.com/jstar0/codexfold/internal/vfs" +) + +type largeGateReport struct { + SourceBytes int64 `json:"source_bytes"` + FoldDuration time.Duration `json:"fold_duration"` + PackDuration time.Duration `json:"pack_duration"` + ShadowDuration time.Duration `json:"shadow_duration"` + First fsctl.BenchmarkReport `json:"first"` + Warm fsctl.BenchmarkReport `json:"warm"` + MaxRSSBytes uint64 `json:"max_rss_bytes"` + PackCacheBytes int64 `json:"pack_cache_bytes"` + LooseObjectsOff bool `json:"loose_objects_offline"` +} + +func TestLargePreviewBenchmark(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_LARGE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_LARGE_TEST=1 to run the 758 MiB preview gate") + } + root := t.TempDir() + const sourceBytes = int64(758 << 20) + fixture, err := GenerateRollout(filepath.Join(root, "large.jsonl"), sourceBytes) + if err != nil { + t.Fatal(err) + } + store := filepath.Join(root, "store") + foldStart := time.Now() + if _, err := fold.Fold(context.Background(), codex.Session{ID: fixture.ID, RolloutPath: fixture.Path, Archived: true}, fold.FoldOptions{StoreDir: store, Apply: true, FieldThreshold: 1 << 20}); err != nil { + t.Fatal(err) + } + foldDuration := time.Since(foldStart) + packStart := time.Now() + if _, err := pack.Build(context.Background(), store, pack.BuildOptions{}); err != nil { + t.Fatal(err) + } + packDuration := time.Since(packStart) + const cacheBytes = int64(128 << 20) + resolver, err := pack.Open(store, pack.OpenOptions{CacheBytes: cacheBytes}) + if err != nil { + t.Fatal(err) + } + defer resolver.Close() + manifest, err := fold.LoadManifest(store, fixture.ID) + if err != nil { + t.Fatal(err) + } + view, err := vfs.NewView(manifest, resolver) + if err != nil { + t.Fatal(err) + } + objects := filepath.Join(store, "objects") + offlineObjects := filepath.Join(store, "objects.offline") + if err := os.Rename(objects, offlineObjects); err != nil { + t.Fatal(err) + } + shadowStart := time.Now() + shadow, err := fsctl.Shadow(context.Background(), fixture.Path, view, fsctl.ShadowOptions{BlockBytes: 4 << 20, RandomReads: 10000, Seed: 42}) + if err != nil || !shadow.Verified { + t.Fatalf("shadow: %#v err=%v", shadow, err) + } + shadowDuration := time.Since(shadowStart) + runtime.GC() + options := fsctl.BenchmarkOptions{SequentialBlockBytes: 4 << 20, RandomBlockBytes: 4 << 10, RandomReads: 10000, Seed: 42} + first, err := fsctl.Benchmark(context.Background(), fixture.Path, view, options) + if err != nil { + t.Fatal(err) + } + warm, err := fsctl.Benchmark(context.Background(), fixture.Path, view, options) + if err != nil { + t.Fatal(err) + } + report := largeGateReport{SourceBytes: sourceBytes, FoldDuration: foldDuration, PackDuration: packDuration, ShadowDuration: shadowDuration, First: first, Warm: warm, MaxRSSBytes: maxRSSBytes(), PackCacheBytes: cacheBytes, LooseObjectsOff: true} + encoded, _ := json.MarshalIndent(report, "", " ") + t.Logf("large preview gate:\n%s", encoded) + if first.Virtual.BytesPerSecond < 500<<20 || first.Virtual.BytesPerSecond < first.Native.BytesPerSecond*0.70 { + t.Fatalf("first virtual throughput gate failed: native=%.0f virtual=%.0f", first.Native.BytesPerSecond, first.Virtual.BytesPerSecond) + } + if warm.Virtual.BytesPerSecond < 500<<20 || warm.Virtual.BytesPerSecond < warm.Native.BytesPerSecond*0.80 { + t.Fatalf("warm virtual throughput gate failed: native=%.0f virtual=%.0f", warm.Native.BytesPerSecond, warm.Virtual.BytesPerSecond) + } + if report.MaxRSSBytes != 0 && report.MaxRSSBytes > 512<<20 { + t.Fatalf("max RSS exceeded 512 MiB: %d", report.MaxRSSBytes) + } +} diff --git a/internal/testfs/rss_darwin.go b/internal/testfs/rss_darwin.go new file mode 100644 index 0000000..4dd3744 --- /dev/null +++ b/internal/testfs/rss_darwin.go @@ -0,0 +1,13 @@ +//go:build darwin + +package testfs + +import "golang.org/x/sys/unix" + +func maxRSSBytes() uint64 { + var usage unix.Rusage + if err := unix.Getrusage(unix.RUSAGE_SELF, &usage); err != nil || usage.Maxrss < 0 { + return 0 + } + return uint64(usage.Maxrss) +} diff --git a/internal/testfs/rss_linux.go b/internal/testfs/rss_linux.go new file mode 100644 index 0000000..f74be85 --- /dev/null +++ b/internal/testfs/rss_linux.go @@ -0,0 +1,13 @@ +//go:build linux + +package testfs + +import "golang.org/x/sys/unix" + +func maxRSSBytes() uint64 { + var usage unix.Rusage + if err := unix.Getrusage(unix.RUSAGE_SELF, &usage); err != nil || usage.Maxrss < 0 { + return 0 + } + return uint64(usage.Maxrss) * 1024 +} diff --git a/internal/testfs/rss_other.go b/internal/testfs/rss_other.go new file mode 100644 index 0000000..7b4f05a --- /dev/null +++ b/internal/testfs/rss_other.go @@ -0,0 +1,5 @@ +//go:build !darwin && !linux + +package testfs + +func maxRSSBytes() uint64 { return 0 } diff --git a/scripts/test-cross-platform.sh b/scripts/test-cross-platform.sh new file mode 100755 index 0000000..20d447a --- /dev/null +++ b/scripts/test-cross-platform.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu + +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$root" + +go test ./... -count=1 +go test -race ./... -count=1 +go vet ./... +go build ./cmd/codexfold + +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /tmp/codexfold-linux-amd64 ./cmd/codexfold +CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o /tmp/codexfold-windows-amd64.exe ./cmd/codexfold +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test -c -o /tmp/codexfold-testfs-linux.test ./internal/testfs +CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go test -c -o /tmp/codexfold-testfs-windows.test.exe ./internal/testfs From dc07ab1cfbf7ee5c3fc1c924c87097f6e26882fc Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 01:59:30 +0800 Subject: [PATCH 12/33] fix: require verified fuse mount before routing --- internal/cli/fs.go | 44 +++++++++++++++++++++++++++---------- internal/cli/fs_test.go | 37 +++++++++++++++++++++++++++++++ internal/service/service.go | 4 +++- 3 files changed, 73 insertions(+), 12 deletions(-) diff --git a/internal/cli/fs.go b/internal/cli/fs.go index 1f8ee0b..6c5f13f 100644 --- a/internal/cli/fs.go +++ b/internal/cli/fs.go @@ -21,6 +21,7 @@ import ( "github.com/jstar0/codexfold/internal/fsctl" "github.com/jstar0/codexfold/internal/mountfs" "github.com/jstar0/codexfold/internal/pack" + "github.com/jstar0/codexfold/internal/service" "github.com/jstar0/codexfold/internal/vfs" "github.com/spf13/cobra" ) @@ -76,6 +77,8 @@ type compatibilityFlags struct { desktopPath string } +var mountHealthProbe = service.ProbeMount + func newFSCommand() *cobra.Command { command := &cobra.Command{Use: "fs", Short: "Operate the transparent session filesystem"} command.AddCommand(newFSStatusCommand()) @@ -369,8 +372,8 @@ func newFSMigrateCommand() *cobra.Command { if len(compatibilityResult.DetectionErrors) != 0 || !compatibilityResult.Evaluation.Approved { return errors.New("installed Codex client versions are not covered by compatibility contracts") } - if info, err := os.Stat(mount); err != nil || !info.IsDir() { - return errors.New("filesystem mount point is not available") + if err := mountHealthProbe(mount); err != nil { + return fmt.Errorf("filesystem mount point is not healthy: %w", err) } if _, err := vfs.OpenSession(command.Context(), vfs.SessionOptions{Root: store, ManifestPath: fold.ManifestPath(store, session.ID), Manifest: manifest, Reader: resolver, NativeSnapshot: native}); err != nil { return err @@ -621,13 +624,18 @@ func newFSRecoverCommand() *cobra.Command { } func addCompatibilityFlags(command *cobra.Command, flags *compatibilityFlags) { + defaults := defaultCompatibilityFlags() command.Flags().StringVar(&flags.contractsPath, "contracts", "", "Compatibility contract directory; defaults to /compatibility") - command.Flags().StringVar(&flags.cliPath, "cli", "codex", "Codex CLI path, or 'none' to skip CLI evaluation") - defaultDesktop := "none" + command.Flags().StringVar(&flags.cliPath, "cli", defaults.cliPath, "Codex CLI path, or 'none' to skip CLI evaluation") + command.Flags().StringVar(&flags.desktopPath, "desktop-app", defaults.desktopPath, "Codex desktop application path, or 'none' to skip desktop evaluation") +} + +func defaultCompatibilityFlags() compatibilityFlags { + desktop := "none" if runtime.GOOS == "darwin" { - defaultDesktop = "/Applications/ChatGPT.app" + desktop = "/Applications/ChatGPT.app" } - command.Flags().StringVar(&flags.desktopPath, "desktop-app", defaultDesktop, "Codex desktop application path, or 'none' to skip desktop evaluation") + return compatibilityFlags{cliPath: "codex", desktopPath: desktop} } func evaluateCompatibility(ctx context.Context, store string, flags compatibilityFlags) (FSCompatibilityResult, error) { @@ -752,12 +760,17 @@ func requireStorageHealth(ctx context.Context, store string) error { } func fsDoctor(ctx context.Context, home string, store string, mount string) fsctl.DoctorReport { + serviceStatus := service.Manager{}.Status(ctx, serviceLabel, mount) checks := []fsctl.Check{ - {Component: fsctl.ComponentDaemon, Run: func(context.Context) error { return errors.New("managed service lifecycle is not installed") }}, + {Component: fsctl.ComponentDaemon, Run: func(context.Context) error { + if !serviceStatus.DaemonRunning { + return errors.New(serviceStatus.DaemonError) + } + return nil + }}, {Component: fsctl.ComponentMount, Run: func(context.Context) error { - info, err := os.Stat(mount) - if err != nil || !info.IsDir() { - return errors.New("filesystem mount point is unavailable") + if !serviceStatus.MountHealthy { + return errors.New(serviceStatus.MountError) } return nil }}, @@ -841,7 +854,16 @@ func fsDoctor(ctx context.Context, home string, store string, mount string) fsct } return nil }}, - fsctl.Check{Component: fsctl.ComponentClient, Run: func(context.Context) error { return errors.New("run fs compatibility with explicit client contracts") }}, + fsctl.Check{Component: fsctl.ComponentClient, Run: func(ctx context.Context) error { + result, err := evaluateCompatibility(ctx, store, defaultCompatibilityFlags()) + if err != nil { + return err + } + if len(result.DetectionErrors) != 0 || !result.Evaluation.Approved { + return errors.New("installed Codex clients are not covered by exact compatibility contracts") + } + return nil + }}, ) return fsctl.Doctor(ctx, checks) } diff --git a/internal/cli/fs_test.go b/internal/cli/fs_test.go index f03da84..2ee9292 100644 --- a/internal/cli/fs_test.go +++ b/internal/cli/fs_test.go @@ -57,6 +57,7 @@ func TestFSServiceInstallIsDryRunByDefaultAndApplyRequiresFuseBuild(t *testing.T } func TestFSUpdatePreflightQuarantineRoutesLatestVisibleBytesNative(t *testing.T) { + allowFixtureMount(t) home, storeDir, nativePath := fsFixture(t, true) approvedCLI := approvedCLIContract(t, storeDir, "1.2.3") mount := filepath.Join(home, "mount") @@ -181,6 +182,33 @@ func TestFSMigrateApplyFailsClosedWithoutMountedTarget(t *testing.T) { } } +func TestFSMigrateApplyRejectsPlainDirectoryThatOnlyLooksLikeMount(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + mount := filepath.Join(home, "plain-directory") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(nativePath) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(mount, "session.jsonl"), data, 0o600); err != nil { + t.Fatal(err) + } + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "migrate", "session", "--codex-home", home, "--store", storeDir, "--mount", mount, "--cli", cliPath, "--desktop-app", "none", "--apply"}) + if err := root.Execute(); err == nil { + t.Fatal("plain directory should not satisfy the FUSE mount health gate") + } + sessions, _ := codex.LoadSessions(home) + if sessions[0].RolloutPath != nativePath { + t.Fatalf("failed mount gate changed route to %q", sessions[0].RolloutPath) + } +} + func TestFSStatusDoesNotClaimTransparentReadiness(t *testing.T) { root := NewRootCommand() var output bytes.Buffer @@ -217,6 +245,7 @@ func TestFSCompatibilityApprovesOnlyExactInstalledClientContract(t *testing.T) { } func TestFSMigrateApplyInitializesManagedStateAndRoutesVerifiedTarget(t *testing.T) { + allowFixtureMount(t) home, storeDir, nativePath := fsFixture(t, true) cliPath := approvedCLIContract(t, storeDir, "1.2.3") mount := filepath.Join(home, "mount") @@ -249,6 +278,7 @@ func TestFSMigrateApplyInitializesManagedStateAndRoutesVerifiedTarget(t *testing } func TestFSRollbackUsesLatestVisibleBytesAfterVirtualAppend(t *testing.T) { + allowFixtureMount(t) home, storeDir, nativePath := fsFixture(t, true) cliPath := approvedCLIContract(t, storeDir, "1.2.3") mount := filepath.Join(home, "mount") @@ -446,3 +476,10 @@ func executeFS(t *testing.T, args []string) { t.Fatalf("execute %v: %v", args, err) } } + +func allowFixtureMount(t *testing.T) { + t.Helper() + previous := mountHealthProbe + mountHealthProbe = func(string) error { return nil } + t.Cleanup(func() { mountHealthProbe = previous }) +} diff --git a/internal/service/service.go b/internal/service/service.go index e290a36..0d2c3a4 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -170,7 +170,7 @@ func (m Manager) Status(ctx context.Context, label string, mountPoint string) St } probe := m.MountProbe if probe == nil { - probe = defaultMountProbe + probe = ProbeMount } if err := probe(mountPoint); err != nil { result.MountError = err.Error() @@ -180,6 +180,8 @@ func (m Manager) Status(ctx context.Context, label string, mountPoint string) St return result } +func ProbeMount(path string) error { return defaultMountProbe(path) } + func EvaluateUpdate(input UpdateInput) UpdateDecision { if !input.DoctorHealthy { return UpdateDecision{Reason: "filesystem doctor is not healthy"} From 07661c2245cede21b5fc7bf6a0006fc9711045c3 Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 02:13:20 +0800 Subject: [PATCH 13/33] test: measure cache-bypass preview performance --- docs/validation-fs-preview.md | 32 ++++++++------ internal/fsctl/benchmark.go | 20 ++++++--- internal/fsctl/fsctl_test.go | 15 +++++++ internal/fsctl/nocache_darwin.go | 14 ++++++ internal/fsctl/nocache_other.go | 7 +++ internal/pack/doctor.go | 2 +- internal/pack/nocache_darwin.go | 14 ++++++ internal/pack/nocache_other.go | 7 +++ internal/pack/pack_test.go | 21 +++++++++ internal/pack/resolver.go | 31 ++++++++++---- internal/testfs/large_test.go | 73 ++++++++++++++++++++------------ internal/testfs/resource.go | 20 +++++++++ internal/testfs/rss_darwin.go | 12 ++++-- internal/testfs/rss_linux.go | 12 ++++-- internal/testfs/rss_other.go | 2 +- 15 files changed, 218 insertions(+), 64 deletions(-) create mode 100644 internal/fsctl/nocache_darwin.go create mode 100644 internal/fsctl/nocache_other.go create mode 100644 internal/pack/nocache_darwin.go create mode 100644 internal/pack/nocache_other.go create mode 100644 internal/testfs/resource.go diff --git a/docs/validation-fs-preview.md b/docs/validation-fs-preview.md index 5f7d2ec..010af44 100644 --- a/docs/validation-fs-preview.md +++ b/docs/validation-fs-preview.md @@ -33,26 +33,30 @@ Passing these checks can justify only `fs-engine-preview`. The following remain Run on 2026-07-12 using an Apple M4 Pro MacBook Pro with 12 CPU cores and 48 GiB RAM, macOS 26.5.1, and Go 1.26.4. -The 758 MiB source was deliberately highly repetitive. The first benchmark followed Fold, Pack, and Shadow in the same process, so both native and packed data benefited from system caching. These values are a deterministic engine gate, not a claim about real Codex cold-cache workloads. +The 758 MiB source was deliberately highly repetitive. The cold pass requested and successfully applied macOS `F_NOCACHE` to both the native rollout file and every opened pack file, with an empty process-level decompressed block cache. It is a cache-bypass gate, not a disk-power-cycle or root-level system-cache purge. These values are deterministic engine evidence, not a claim that every real Codex workload has the same compression or reuse ratio. -| Metric | First pass | Warm pass | +| Metric | Cold cache-bypass pass | Warm pass | | --- | ---: | ---: | -| Native sequential throughput | 16.03 GB/s | 15.87 GB/s | -| Virtual sequential throughput | 60.73 GB/s | 58.15 GB/s | -| Virtual/native ratio | 3.79x | 3.66x | -| Random read p50 | 0.584 us | 0.583 us | -| Random read p95 | 0.959 us | 0.958 us | -| Random read p99 | 1.625 us | 1.542 us | +| Native sequential throughput | 14.58 GB/s | 16.46 GB/s | +| Virtual sequential throughput | 34.56 GB/s | 54.69 GB/s | +| Virtual/native ratio | 2.37x | 3.32x | +| Random read p50 | 0.708 us | 0.625 us | +| Random read p95 | 1.041 us | 0.958 us | +| Random read p99 | 1.292 us | 1.250 us | Additional results: -- Fold: 35.23 s. -- Pack build: 0.055 s. -- Complete SHA plus 10,000 random-range shadow: 3.31 s. -- Go system memory: 160.04 MiB. -- Maximum RSS: 160.02 MiB. +- Fold: 44.69 s. +- Pack build: 0.065 s. +- Complete SHA plus 10,000 random-range shadow: 3.51 s. +- Go system memory: 135.45 MiB. +- Maximum RSS: 135.70 MiB. +- User CPU for the complete heavy gate: 69.72 s. +- System CPU for the complete heavy gate: 15.46 s. - Configured decompressed block cache: 128 MiB. -- Loose-object directory offline during shadow and both benchmark passes: yes. +- Native `F_NOCACHE` applied during the cold pass: yes. +- Pack-file `F_NOCACHE` applied during the cold pass: yes. +- Loose-object directory offline during cold benchmark, shadow, and warm benchmark: yes. - 100,000 one-byte append calls followed by `fsync`: 2.18 s in the normal test build. The platform-neutral gates pass and justify `fs-engine-preview`. This result does not satisfy any Task 11 real-adapter or real-Codex gate. diff --git a/internal/fsctl/benchmark.go b/internal/fsctl/benchmark.go index 041a9a9..8380a29 100644 --- a/internal/fsctl/benchmark.go +++ b/internal/fsctl/benchmark.go @@ -18,6 +18,7 @@ type BenchmarkOptions struct { RandomBlockBytes int RandomReads int Seed int64 + BypassOSCache bool } type SequentialMetric struct { @@ -34,10 +35,12 @@ type RandomMetric struct { } type BenchmarkReport struct { - Native SequentialMetric `json:"native"` - Virtual SequentialMetric `json:"virtual"` - Random RandomMetric `json:"random"` - GoSysBytes uint64 `json:"go_sys_bytes"` + Native SequentialMetric `json:"native"` + Virtual SequentialMetric `json:"virtual"` + Random RandomMetric `json:"random"` + GoSysBytes uint64 `json:"go_sys_bytes"` + OSCacheBypassRequested bool `json:"os_cache_bypass_requested"` + OSCacheBypassApplied bool `json:"os_cache_bypass_applied"` } func Benchmark(ctx context.Context, nativePath string, virtual Readable, options BenchmarkOptions) (BenchmarkReport, error) { @@ -55,6 +58,13 @@ func Benchmark(ctx context.Context, nativePath string, virtual Readable, options return BenchmarkReport{}, err } defer native.Close() + bypassApplied := false + if options.BypassOSCache { + bypassApplied, err = configureNoCache(native) + if err != nil { + return BenchmarkReport{}, err + } + } info, err := native.Stat() if err != nil { return BenchmarkReport{}, err @@ -76,7 +86,7 @@ func Benchmark(ctx context.Context, nativePath string, virtual Readable, options } var memory runtime.MemStats runtime.ReadMemStats(&memory) - return BenchmarkReport{Native: nativeMetric, Virtual: virtualMetric, Random: randomMetric, GoSysBytes: memory.Sys}, nil + return BenchmarkReport{Native: nativeMetric, Virtual: virtualMetric, Random: randomMetric, GoSysBytes: memory.Sys, OSCacheBypassRequested: options.BypassOSCache, OSCacheBypassApplied: bypassApplied}, nil } func benchmarkNativeSequential(ctx context.Context, file *os.File, size int64, blockBytes int) (SequentialMetric, error) { diff --git a/internal/fsctl/fsctl_test.go b/internal/fsctl/fsctl_test.go index 0115f97..71c935d 100644 --- a/internal/fsctl/fsctl_test.go +++ b/internal/fsctl/fsctl_test.go @@ -89,6 +89,21 @@ func TestBenchmarkMeasuresNativeAndVirtualReads(t *testing.T) { } } +func TestBenchmarkRecordsRequestedOSCacheBypass(t *testing.T) { + path := filepath.Join(t.TempDir(), "native.jsonl") + data := []byte("cache-bypass") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + report, err := Benchmark(context.Background(), path, byteReader(data), BenchmarkOptions{BypassOSCache: true, RandomReads: 1}) + if err != nil { + t.Fatal(err) + } + if !report.OSCacheBypassRequested { + t.Fatalf("benchmark did not record cache-bypass request: %#v", report) + } +} + type byteReader []byte func (r byteReader) Size() int64 { return int64(len(r)) } diff --git a/internal/fsctl/nocache_darwin.go b/internal/fsctl/nocache_darwin.go new file mode 100644 index 0000000..0f999d0 --- /dev/null +++ b/internal/fsctl/nocache_darwin.go @@ -0,0 +1,14 @@ +//go:build darwin + +package fsctl + +import ( + "os" + + "golang.org/x/sys/unix" +) + +func configureNoCache(file *os.File) (bool, error) { + _, err := unix.FcntlInt(file.Fd(), unix.F_NOCACHE, 1) + return err == nil, err +} diff --git a/internal/fsctl/nocache_other.go b/internal/fsctl/nocache_other.go new file mode 100644 index 0000000..834b9c1 --- /dev/null +++ b/internal/fsctl/nocache_other.go @@ -0,0 +1,7 @@ +//go:build !darwin + +package fsctl + +import "os" + +func configureNoCache(*os.File) (bool, error) { return false, nil } diff --git a/internal/pack/doctor.go b/internal/pack/doctor.go index 8fff362..f7bcdaf 100644 --- a/internal/pack/doctor.go +++ b/internal/pack/doctor.go @@ -64,7 +64,7 @@ func Doctor(ctx context.Context, storeDir string) (DoctorResult, error) { } func verifyGeneration(ctx context.Context, directory string, index Index) error { - resolver, err := openGeneration(directory, 0) + resolver, err := openGeneration(directory, 0, false) if err != nil { return err } diff --git a/internal/pack/nocache_darwin.go b/internal/pack/nocache_darwin.go new file mode 100644 index 0000000..41463c7 --- /dev/null +++ b/internal/pack/nocache_darwin.go @@ -0,0 +1,14 @@ +//go:build darwin + +package pack + +import ( + "os" + + "golang.org/x/sys/unix" +) + +func configureNoCache(file *os.File) (bool, error) { + _, err := unix.FcntlInt(file.Fd(), unix.F_NOCACHE, 1) + return err == nil, err +} diff --git a/internal/pack/nocache_other.go b/internal/pack/nocache_other.go new file mode 100644 index 0000000..29e79a3 --- /dev/null +++ b/internal/pack/nocache_other.go @@ -0,0 +1,7 @@ +//go:build !darwin + +package pack + +import "os" + +func configureNoCache(*os.File) (bool, error) { return false, nil } diff --git a/internal/pack/pack_test.go b/internal/pack/pack_test.go index 783c8d1..8e9a1e3 100644 --- a/internal/pack/pack_test.go +++ b/internal/pack/pack_test.go @@ -70,6 +70,27 @@ func TestBuildAndResolverReadExactRandomRanges(t *testing.T) { } } +func TestResolverSupportsOSCacheBypassOption(t *testing.T) { + root := t.TempDir() + refs := putObjects(t, root, []byte("cache-bypass-object")) + writeManifest(t, root, "session", refs) + if _, err := Build(context.Background(), root, BuildOptions{}); err != nil { + t.Fatal(err) + } + resolver, err := Open(root, OpenOptions{BypassOSCache: true}) + if err != nil { + t.Fatal(err) + } + defer resolver.Close() + buffer := make([]byte, refs[0].RawBytes) + if _, err := resolver.ReadAt(context.Background(), refs[0], buffer, 0); err != nil { + t.Fatal(err) + } + if string(buffer) != "cache-bypass-object" { + t.Fatalf("unexpected bytes: %q", buffer) + } +} + func TestBuildInterruptionKeepsPreviousGenerationCurrent(t *testing.T) { root := t.TempDir() refs := putObjects(t, root, []byte("first-generation")) diff --git a/internal/pack/resolver.go b/internal/pack/resolver.go index f361b8a..06e9f78 100644 --- a/internal/pack/resolver.go +++ b/internal/pack/resolver.go @@ -19,16 +19,18 @@ import ( ) type OpenOptions struct { - CacheBytes int64 + CacheBytes int64 + BypassOSCache bool } type Resolver struct { - directory string - index Index - objects map[string]Object - packs map[string]*os.File - cache *blockCache - closeOnce sync.Once + directory string + index Index + objects map[string]Object + packs map[string]*os.File + cache *blockCache + bypassOSCacheApplied bool + closeOnce sync.Once } func Open(storeDir string, options OpenOptions) (*Resolver, error) { @@ -43,10 +45,10 @@ func Open(storeDir string, options OpenOptions) (*Resolver, error) { if options.CacheBytes == 0 { options.CacheBytes = defaultCacheBytes } - return openGeneration(filepath.Join(storeDir, "packs", generation), options.CacheBytes) + return openGeneration(filepath.Join(storeDir, "packs", generation), options.CacheBytes, options.BypassOSCache) } -func openGeneration(directory string, cacheBytes int64) (*Resolver, error) { +func openGeneration(directory string, cacheBytes int64, bypassOSCache bool) (*Resolver, error) { data, err := os.ReadFile(filepath.Join(directory, "index.json")) if err != nil { return nil, fmt.Errorf("read pack index: %w", err) @@ -74,6 +76,15 @@ func openGeneration(directory string, cacheBytes int64) (*Resolver, error) { _ = resolver.Close() return nil, fmt.Errorf("open pack %s: %w", block.Pack, err) } + if bypassOSCache { + applied, err := configureNoCache(file) + if err != nil { + _ = file.Close() + _ = resolver.Close() + return nil, fmt.Errorf("disable OS cache for pack %s: %w", block.Pack, err) + } + resolver.bypassOSCacheApplied = resolver.bypassOSCacheApplied || applied + } resolver.packs[block.Pack] = file } } @@ -93,6 +104,8 @@ func openGeneration(directory string, cacheBytes int64) (*Resolver, error) { return resolver, nil } +func (r *Resolver) OSCacheBypassApplied() bool { return r.bypassOSCacheApplied } + func (r *Resolver) ReadAt(ctx context.Context, ref fold.ObjectRef, destination []byte, offset int64) (int, error) { if offset < 0 { return 0, errors.New("negative object read offset") diff --git a/internal/testfs/large_test.go b/internal/testfs/large_test.go index a14032e..a414f51 100644 --- a/internal/testfs/large_test.go +++ b/internal/testfs/large_test.go @@ -17,21 +17,25 @@ import ( ) type largeGateReport struct { - SourceBytes int64 `json:"source_bytes"` - FoldDuration time.Duration `json:"fold_duration"` - PackDuration time.Duration `json:"pack_duration"` - ShadowDuration time.Duration `json:"shadow_duration"` - First fsctl.BenchmarkReport `json:"first"` - Warm fsctl.BenchmarkReport `json:"warm"` - MaxRSSBytes uint64 `json:"max_rss_bytes"` - PackCacheBytes int64 `json:"pack_cache_bytes"` - LooseObjectsOff bool `json:"loose_objects_offline"` + SourceBytes int64 `json:"source_bytes"` + FoldDuration time.Duration `json:"fold_duration"` + PackDuration time.Duration `json:"pack_duration"` + ShadowDuration time.Duration `json:"shadow_duration"` + Cold fsctl.BenchmarkReport `json:"cold"` + Warm fsctl.BenchmarkReport `json:"warm"` + MaxRSSBytes uint64 `json:"max_rss_bytes"` + UserCPU time.Duration `json:"user_cpu"` + SystemCPU time.Duration `json:"system_cpu"` + PackCacheBytes int64 `json:"pack_cache_bytes"` + PackCacheBypassApplied bool `json:"pack_os_cache_bypass_applied"` + LooseObjectsOff bool `json:"loose_objects_offline"` } func TestLargePreviewBenchmark(t *testing.T) { if os.Getenv("CODEXFOLD_RUN_LARGE_TEST") != "1" { t.Skip("set CODEXFOLD_RUN_LARGE_TEST=1 to run the 758 MiB preview gate") } + usageBefore := processResourceUsage() root := t.TempDir() const sourceBytes = int64(758 << 20) fixture, err := GenerateRollout(filepath.Join(root, "large.jsonl"), sourceBytes) @@ -50,45 +54,62 @@ func TestLargePreviewBenchmark(t *testing.T) { } packDuration := time.Since(packStart) const cacheBytes = int64(128 << 20) - resolver, err := pack.Open(store, pack.OpenOptions{CacheBytes: cacheBytes}) + manifest, err := fold.LoadManifest(store, fixture.ID) if err != nil { t.Fatal(err) } - defer resolver.Close() - manifest, err := fold.LoadManifest(store, fixture.ID) + objects := filepath.Join(store, "objects") + offlineObjects := filepath.Join(store, "objects.offline") + if err := os.Rename(objects, offlineObjects); err != nil { + t.Fatal(err) + } + coldResolver, err := pack.Open(store, pack.OpenOptions{CacheBytes: cacheBytes, BypassOSCache: true}) if err != nil { t.Fatal(err) } - view, err := vfs.NewView(manifest, resolver) + coldView, err := vfs.NewView(manifest, coldResolver) if err != nil { + _ = coldResolver.Close() t.Fatal(err) } - objects := filepath.Join(store, "objects") - offlineObjects := filepath.Join(store, "objects.offline") - if err := os.Rename(objects, offlineObjects); err != nil { + runtime.GC() + coldOptions := fsctl.BenchmarkOptions{SequentialBlockBytes: 4 << 20, RandomBlockBytes: 4 << 10, RandomReads: 10000, Seed: 42, BypassOSCache: true} + cold, err := fsctl.Benchmark(context.Background(), fixture.Path, coldView, coldOptions) + packBypassApplied := coldResolver.OSCacheBypassApplied() + _ = coldResolver.Close() + if err != nil { + t.Fatal(err) + } + if runtime.GOOS == "darwin" && (!cold.OSCacheBypassApplied || !packBypassApplied) { + t.Fatalf("macOS cold gate did not apply F_NOCACHE: native=%t pack=%t", cold.OSCacheBypassApplied, packBypassApplied) + } + warmResolver, err := pack.Open(store, pack.OpenOptions{CacheBytes: cacheBytes}) + if err != nil { + t.Fatal(err) + } + defer warmResolver.Close() + warmView, err := vfs.NewView(manifest, warmResolver) + if err != nil { t.Fatal(err) } shadowStart := time.Now() - shadow, err := fsctl.Shadow(context.Background(), fixture.Path, view, fsctl.ShadowOptions{BlockBytes: 4 << 20, RandomReads: 10000, Seed: 42}) + shadow, err := fsctl.Shadow(context.Background(), fixture.Path, warmView, fsctl.ShadowOptions{BlockBytes: 4 << 20, RandomReads: 10000, Seed: 42}) if err != nil || !shadow.Verified { t.Fatalf("shadow: %#v err=%v", shadow, err) } shadowDuration := time.Since(shadowStart) runtime.GC() - options := fsctl.BenchmarkOptions{SequentialBlockBytes: 4 << 20, RandomBlockBytes: 4 << 10, RandomReads: 10000, Seed: 42} - first, err := fsctl.Benchmark(context.Background(), fixture.Path, view, options) - if err != nil { - t.Fatal(err) - } - warm, err := fsctl.Benchmark(context.Background(), fixture.Path, view, options) + warmOptions := fsctl.BenchmarkOptions{SequentialBlockBytes: 4 << 20, RandomBlockBytes: 4 << 10, RandomReads: 10000, Seed: 42} + warm, err := fsctl.Benchmark(context.Background(), fixture.Path, warmView, warmOptions) if err != nil { t.Fatal(err) } - report := largeGateReport{SourceBytes: sourceBytes, FoldDuration: foldDuration, PackDuration: packDuration, ShadowDuration: shadowDuration, First: first, Warm: warm, MaxRSSBytes: maxRSSBytes(), PackCacheBytes: cacheBytes, LooseObjectsOff: true} + usage := subtractUsage(processResourceUsage(), usageBefore) + report := largeGateReport{SourceBytes: sourceBytes, FoldDuration: foldDuration, PackDuration: packDuration, ShadowDuration: shadowDuration, Cold: cold, Warm: warm, MaxRSSBytes: usage.MaxRSSBytes, UserCPU: usage.UserCPU, SystemCPU: usage.SystemCPU, PackCacheBytes: cacheBytes, PackCacheBypassApplied: packBypassApplied, LooseObjectsOff: true} encoded, _ := json.MarshalIndent(report, "", " ") t.Logf("large preview gate:\n%s", encoded) - if first.Virtual.BytesPerSecond < 500<<20 || first.Virtual.BytesPerSecond < first.Native.BytesPerSecond*0.70 { - t.Fatalf("first virtual throughput gate failed: native=%.0f virtual=%.0f", first.Native.BytesPerSecond, first.Virtual.BytesPerSecond) + if cold.Virtual.BytesPerSecond < 500<<20 || cold.Virtual.BytesPerSecond < cold.Native.BytesPerSecond*0.70 { + t.Fatalf("cold virtual throughput gate failed: native=%.0f virtual=%.0f", cold.Native.BytesPerSecond, cold.Virtual.BytesPerSecond) } if warm.Virtual.BytesPerSecond < 500<<20 || warm.Virtual.BytesPerSecond < warm.Native.BytesPerSecond*0.80 { t.Fatalf("warm virtual throughput gate failed: native=%.0f virtual=%.0f", warm.Native.BytesPerSecond, warm.Virtual.BytesPerSecond) diff --git a/internal/testfs/resource.go b/internal/testfs/resource.go new file mode 100644 index 0000000..b4f345c --- /dev/null +++ b/internal/testfs/resource.go @@ -0,0 +1,20 @@ +package testfs + +import "time" + +type resourceUsage struct { + MaxRSSBytes uint64 + UserCPU time.Duration + SystemCPU time.Duration +} + +func subtractUsage(after resourceUsage, before resourceUsage) resourceUsage { + result := after + if after.UserCPU >= before.UserCPU { + result.UserCPU = after.UserCPU - before.UserCPU + } + if after.SystemCPU >= before.SystemCPU { + result.SystemCPU = after.SystemCPU - before.SystemCPU + } + return result +} diff --git a/internal/testfs/rss_darwin.go b/internal/testfs/rss_darwin.go index 4dd3744..dd6ec2a 100644 --- a/internal/testfs/rss_darwin.go +++ b/internal/testfs/rss_darwin.go @@ -2,12 +2,16 @@ package testfs -import "golang.org/x/sys/unix" +import ( + "time" -func maxRSSBytes() uint64 { + "golang.org/x/sys/unix" +) + +func processResourceUsage() resourceUsage { var usage unix.Rusage if err := unix.Getrusage(unix.RUSAGE_SELF, &usage); err != nil || usage.Maxrss < 0 { - return 0 + return resourceUsage{} } - return uint64(usage.Maxrss) + return resourceUsage{MaxRSSBytes: uint64(usage.Maxrss), UserCPU: time.Duration(unix.TimevalToNsec(usage.Utime)), SystemCPU: time.Duration(unix.TimevalToNsec(usage.Stime))} } diff --git a/internal/testfs/rss_linux.go b/internal/testfs/rss_linux.go index f74be85..c95e983 100644 --- a/internal/testfs/rss_linux.go +++ b/internal/testfs/rss_linux.go @@ -2,12 +2,16 @@ package testfs -import "golang.org/x/sys/unix" +import ( + "time" -func maxRSSBytes() uint64 { + "golang.org/x/sys/unix" +) + +func processResourceUsage() resourceUsage { var usage unix.Rusage if err := unix.Getrusage(unix.RUSAGE_SELF, &usage); err != nil || usage.Maxrss < 0 { - return 0 + return resourceUsage{} } - return uint64(usage.Maxrss) * 1024 + return resourceUsage{MaxRSSBytes: uint64(usage.Maxrss) * 1024, UserCPU: time.Duration(unix.TimevalToNsec(usage.Utime)), SystemCPU: time.Duration(unix.TimevalToNsec(usage.Stime))} } diff --git a/internal/testfs/rss_other.go b/internal/testfs/rss_other.go index 7b4f05a..758214d 100644 --- a/internal/testfs/rss_other.go +++ b/internal/testfs/rss_other.go @@ -2,4 +2,4 @@ package testfs -func maxRSSBytes() uint64 { return 0 } +func processResourceUsage() resourceUsage { return resourceUsage{} } From 2d90db750f9b33018a02d554ef051384212e7cad Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 02:48:20 +0800 Subject: [PATCH 14/33] feat: validate real fuse-t filesystem adapter --- internal/cli/fs_test.go | 25 ++- internal/mountfs/filesystem.go | 32 +++- internal/mountfs/filesystem_test.go | 50 +++++- internal/mountfs/fuse_integration_test.go | 202 ++++++++++++++++++++++ internal/mountfs/host_cgofuse.go | 3 + internal/service/mount_probe_darwin.go | 19 +- 6 files changed, 319 insertions(+), 12 deletions(-) create mode 100644 internal/mountfs/fuse_integration_test.go diff --git a/internal/cli/fs_test.go b/internal/cli/fs_test.go index 2ee9292..19835c5 100644 --- a/internal/cli/fs_test.go +++ b/internal/cli/fs_test.go @@ -14,6 +14,7 @@ import ( "github.com/jstar0/codexfold/internal/compat" "github.com/jstar0/codexfold/internal/fold" "github.com/jstar0/codexfold/internal/fsctl" + "github.com/jstar0/codexfold/internal/mountfs" "github.com/jstar0/codexfold/internal/pack" "github.com/jstar0/codexfold/internal/vfs" ) @@ -51,7 +52,15 @@ func TestFSServiceInstallIsDryRunByDefaultAndApplyRequiresFuseBuild(t *testing.T root.SetOut(&bytes.Buffer{}) root.SetErr(&bytes.Buffer{}) root.SetArgs([]string{"fs", "service", "install", "--codex-home", home, "--store", storeDir, "--plist", plistPath, "--apply"}) - if err := root.Execute(); err == nil { + err := root.Execute() + if mountfs.Available() { + if err != nil { + t.Fatalf("FUSE build should install the service definition: %v", err) + } + if _, statErr := os.Stat(plistPath); statErr != nil { + t.Fatalf("service definition was not written: %v", statErr) + } + } else if err == nil { t.Fatal("default build should reject service installation without a FUSE host") } } @@ -397,12 +406,14 @@ func TestFSReadOnlyCommandsRunWithoutClaimingMountHealth(t *testing.T) { t.Fatalf("%v overclaimed readiness: %s", args, output.String()) } } - root := NewRootCommand() - root.SetOut(&bytes.Buffer{}) - root.SetErr(&bytes.Buffer{}) - root.SetArgs([]string{"fs", "serve", "--codex-home", home, "--store", storeDir, "--mount", filepath.Join(home, "mount"), "--apply"}) - if err := root.Execute(); err == nil { - t.Fatal("default build should not claim the FUSE prerequisite is available") + if !mountfs.Available() { + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "serve", "--codex-home", home, "--store", storeDir, "--mount", filepath.Join(home, "mount"), "--apply"}) + if err := root.Execute(); err == nil { + t.Fatal("default build should not claim the FUSE prerequisite is available") + } } status, _ := fsctl.NewStatus(fsctl.StorageEngine, runtime.GOOS) if status.Capability != fsctl.StorageEngine { diff --git a/internal/mountfs/filesystem.go b/internal/mountfs/filesystem.go index 97148e7..df5fb8f 100644 --- a/internal/mountfs/filesystem.go +++ b/internal/mountfs/filesystem.go @@ -159,7 +159,15 @@ func (f *Filesystem) Write(handleID uint64, data []byte, offset int64) (int, sys if handle.append { n, err = handle.write.Append(context.Background(), data) } else { - n, err = handle.write.WriteAt(context.Background(), data, offset) + info, infoErr := handle.session.VisibleInfo() + if infoErr != nil { + return 0, errnoFor(infoErr) + } + if offset == info.Size { + n, err = handle.write.Append(context.Background(), data) + } else { + n, err = handle.write.WriteAt(context.Background(), data, offset) + } } if err != nil { return n, errnoFor(err) @@ -193,6 +201,16 @@ func (f *Filesystem) TruncatePath(name string, size int64) syscall.Errno { if errno != 0 { return errno } + if handle := f.lockActiveWriter(session); handle != nil { + defer handle.mu.Unlock() + if err := handle.write.Truncate(context.Background(), size); err != nil { + return errnoFor(err) + } + if handle.read != nil { + return refreshReader(handle) + } + return 0 + } writer, err := session.OpenWriter() if err != nil { return errnoFor(err) @@ -205,6 +223,18 @@ func (f *Filesystem) TruncatePath(name string, size int64) syscall.Errno { return errnoFor(closeErr) } +func (f *Filesystem) lockActiveWriter(session *vfs.Session) *fileHandle { + f.mu.RLock() + defer f.mu.RUnlock() + for _, handle := range f.handles { + if handle.session == session && handle.write != nil { + handle.mu.Lock() + return handle + } + } + return nil +} + func (f *Filesystem) Fsync(handleID uint64) syscall.Errno { handle, errno := f.handle(handleID) if errno != 0 { diff --git a/internal/mountfs/filesystem_test.go b/internal/mountfs/filesystem_test.go index 7c283b2..242b2d8 100644 --- a/internal/mountfs/filesystem_test.go +++ b/internal/mountfs/filesystem_test.go @@ -90,6 +90,54 @@ func TestFilesystemRandomWriteTruncateAndWriterExclusion(t *testing.T) { } } +func TestFilesystemWriteAtVisibleEOFUsesDeltaWithoutCopyOnWrite(t *testing.T) { + source := []byte("first\nsecond\nthird\n") + session := mountSessionFixture(t, "session", source) + filesystem := New() + if err := filesystem.AddSession("session", session); err != nil { + t.Fatal(err) + } + handle, errno := filesystem.Open("/session.jsonl", os.O_RDWR) + if errno != 0 { + t.Fatalf("Open writer errno=%v", errno) + } + t.Cleanup(func() { _ = filesystem.Release(handle) }) + tail := []byte("tail\n") + if n, errno := filesystem.Write(handle, tail, int64(len(source))); errno != 0 || n != len(tail) { + t.Fatalf("Write at EOF = %d errno=%v", n, errno) + } + if state := session.State(); state.BackingPath != "" { + t.Fatalf("EOF write created copy-on-write backing %q", state.BackingPath) + } else if info, err := os.Stat(state.DeltaPath); err != nil || info.Size() != int64(len(tail)) { + t.Fatalf("delta after EOF write: info=%#v err=%v", info, err) + } + current := make([]byte, len(source)+len(tail)) + if n, errno := filesystem.Read(handle, current, 0); errno != 0 || n != len(current) { + t.Fatalf("Read after EOF write = %d errno=%v", n, errno) + } + want := append(append([]byte(nil), source...), tail...) + if !bytes.Equal(current, want) { + t.Fatalf("visible bytes differ: got=%q want=%q", current, want) + } +} + +func TestFilesystemPathTruncateUsesTheActiveWriter(t *testing.T) { + filesystem, source := mountFixture(t) + handle, errno := filesystem.Open("/session.jsonl", os.O_RDWR) + if errno != 0 { + t.Fatalf("Open writer errno=%v", errno) + } + t.Cleanup(func() { _ = filesystem.Release(handle) }) + wantSize := int64(len(source) - 3) + if errno := filesystem.TruncatePath("/session.jsonl", wantSize); errno != 0 { + t.Fatalf("TruncatePath with active writer errno=%v", errno) + } + attribute, errno := filesystem.Getattr("/session.jsonl") + if errno != 0 || attribute.Size != wantSize { + t.Fatalf("Getattr after path truncate = %#v errno=%v", attribute, errno) + } +} + func TestFilesystemRejectsUnsafeAndManagementMutations(t *testing.T) { filesystem, _ := mountFixture(t) if _, errno := filesystem.Open("/../session.jsonl", os.O_RDONLY); errno != syscall.ENOENT { @@ -135,7 +183,7 @@ func TestFilesystemUpsertChangesNewOpensWithoutInvalidatingExistingHandles(t *te func TestMountWithoutFuseBuildReturnsPrerequisiteError(t *testing.T) { if Available() { - t.Fatal("default build should report the FUSE host as unavailable") + t.Skip("FUSE-enabled builds are covered by the gated real mount test") } err := Mount(context.Background(), HostOptions{MountPoint: t.TempDir(), Filesystem: New()}) if !errors.Is(err, ErrPrerequisite) { diff --git a/internal/mountfs/fuse_integration_test.go b/internal/mountfs/fuse_integration_test.go new file mode 100644 index 0000000..2270405 --- /dev/null +++ b/internal/mountfs/fuse_integration_test.go @@ -0,0 +1,202 @@ +//go:build darwin && fuse && cgo + +package mountfs + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/jstar0/codexfold/internal/fold" + "github.com/jstar0/codexfold/internal/service" + "github.com/jstar0/codexfold/internal/vfs" +) + +func TestRealFuseMountNativeFileOperations(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real macFUSE adapter test") + } + root := t.TempDir() + source := []byte("first\nsecond\nthird\n") + digest := sha256.Sum256(source) + digestHex := hex.EncodeToString(digest[:]) + nativePath := filepath.Join(root, "native.jsonl") + if err := os.WriteFile(nativePath, source, 0o600); err != nil { + t.Fatal(err) + } + manifest := fold.Manifest{ + Version: fold.ManifestVersion, Kind: fold.ManifestKind, + Session: fold.ManifestSession{ID: "fixture", RolloutPath: nativePath, Archived: true}, + Source: fold.ManifestSource{Bytes: int64(len(source)), SHA256: digestHex}, + Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: digestHex, RawBytes: int64(len(source))}}}, + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, + Reader: fuseFixtureReader{digestHex: source}, NativeSnapshot: vfs.NativeFile{Path: nativePath, Bytes: int64(len(source)), SHA256: digestHex}, + }) + if err != nil { + t.Fatal(err) + } + filesystem := New() + if err := filesystem.AddSession("fixture", managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealMount(t, mountPoint, filesystem) + target := filepath.Join(mountPoint, "fixture.jsonl") + entries, err := os.ReadDir(mountPoint) + if err != nil || len(entries) != 1 || entries[0].Name() != "fixture.jsonl" { + t.Fatalf("native directory listing = %#v err=%v", entries, err) + } + initialInfo, err := os.Stat(target) + if err != nil || initialInfo.Size() != int64(len(source)) || initialInfo.Mode().Perm() != 0o600 { + t.Fatalf("initial native stat = %#v err=%v", initialInfo, err) + } + read, err := os.ReadFile(target) + if err != nil || string(read) != string(source) { + t.Fatalf("native read differs: %q err=%v", read, err) + } + appendFile, err := os.OpenFile(target, os.O_RDWR|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + tail := []byte("tail\n") + if _, err := appendFile.Write(tail); err != nil { + _ = appendFile.Close() + t.Fatal(err) + } + if err := appendFile.Sync(); err != nil { + _ = appendFile.Close() + t.Fatal(err) + } + if err := appendFile.Close(); err != nil { + t.Fatal(err) + } + read, err = os.ReadFile(target) + want := append(append([]byte(nil), source...), tail...) + if err != nil || string(read) != string(want) { + t.Fatalf("append read differs: %q err=%v", read, err) + } + randomFile, err := os.OpenFile(target, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := randomFile.WriteAt([]byte("PATCH"), 2); err != nil { + _ = randomFile.Close() + t.Fatal(err) + } + if err := randomFile.Truncate(int64(len(want) - 3)); err != nil { + _ = randomFile.Close() + t.Fatal(err) + } + if err := randomFile.Sync(); err != nil { + _ = randomFile.Close() + t.Fatal(err) + } + if err := randomFile.Close(); err != nil { + t.Fatal(err) + } + info, err := os.Stat(target) + if err != nil || info.Size() != int64(len(want)-3) { + t.Fatalf("stat after truncate: %#v err=%v", info, err) + } + if err := os.Rename(target, filepath.Join(mountPoint, "renamed.jsonl")); err == nil { + t.Fatal("rename should fail closed until a real Codex trace requires it") + } + mutated := append([]byte(nil), want...) + copy(mutated[2:], []byte("PATCH")) + mutated = mutated[:len(mutated)-3] + read, err = os.ReadFile(target) + if err != nil || !bytes.Equal(read, mutated) { + t.Fatalf("random-write/truncate read differs: %q err=%v", read, err) + } + stopMount() + waitForRealUnmount(t, mountPoint) + + stopRemount := startRealMount(t, mountPoint, filesystem) + read, err = os.ReadFile(target) + if err != nil || !bytes.Equal(read, mutated) { + t.Fatalf("remount read differs: %q err=%v", read, err) + } + stopRemount() + waitForRealUnmount(t, mountPoint) +} + +func startRealMount(t *testing.T, mountPoint string, filesystem *Filesystem) func() { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + mountDone := make(chan error, 1) + go func() { + mountDone <- Mount(ctx, HostOptions{MountPoint: mountPoint, Filesystem: filesystem, Foreground: true}) + }() + var stopOnce sync.Once + stopMount := func() { + stopOnce.Do(func() { + cancel() + select { + case err := <-mountDone: + if err != nil && !errors.Is(err, context.Canceled) { + t.Errorf("mount shutdown: %v", err) + } + case <-time.After(10 * time.Second): + t.Error("mount did not stop after cancellation") + } + }) + } + t.Cleanup(stopMount) + waitForRealMount(t, mountPoint, mountDone) + return stopMount +} + +func waitForRealMount(t *testing.T, mountPoint string, mountDone <-chan error) { + t.Helper() + deadline := time.Now().Add(20 * time.Second) + for time.Now().Before(deadline) { + if err := service.ProbeMount(mountPoint); err == nil { + return + } + select { + case err := <-mountDone: + t.Fatalf("mount exited before becoming healthy: %v", err) + case <-time.After(100 * time.Millisecond): + } + } + t.Fatal("FUSE mount did not become healthy") +} + +func waitForRealUnmount(t *testing.T, mountPoint string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if err := service.ProbeMount(mountPoint); err != nil { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("FUSE mount remained active after shutdown") +} + +type fuseFixtureReader map[string][]byte + +func (r fuseFixtureReader) ReadAt(_ context.Context, ref fold.ObjectRef, destination []byte, offset int64) (int, error) { + data := r[ref.SHA256] + if offset >= int64(len(data)) { + return 0, io.EOF + } + n := copy(destination, data[offset:]) + if n < len(destination) { + return n, io.EOF + } + return n, nil +} diff --git a/internal/mountfs/host_cgofuse.go b/internal/mountfs/host_cgofuse.go index 8deb862..209cee4 100644 --- a/internal/mountfs/host_cgofuse.go +++ b/internal/mountfs/host_cgofuse.go @@ -158,6 +158,9 @@ func mountHost(ctx context.Context, options HostOptions) (result error) { }() mounted := host.Mount(options.MountPoint, arguments) close(done) + if err := ctx.Err(); err != nil { + return err + } if !mounted { return errors.New("FUSE host exited without mounting") } diff --git a/internal/service/mount_probe_darwin.go b/internal/service/mount_probe_darwin.go index 3476022..3377503 100644 --- a/internal/service/mount_probe_darwin.go +++ b/internal/service/mount_probe_darwin.go @@ -17,11 +17,24 @@ func defaultMountProbe(path string) error { } mountedAt := unix.ByteSliceToString(stat.Mntonname[:]) filesystem := strings.ToLower(unix.ByteSliceToString(stat.Fstypename[:])) - if filepath.Clean(mountedAt) != filepath.Clean(path) { + mountedFrom := strings.ToLower(unix.ByteSliceToString(stat.Mntfromname[:])) + requestedPath := canonicalMountPath(path) + actualPath := canonicalMountPath(mountedAt) + if actualPath != requestedPath { return errors.New("path is not a mount root") } - if !strings.Contains(filesystem, "fuse") { - return errors.New("mount root is not backed by FUSE") + macFUSE := strings.Contains(filesystem, "fuse") + fuseT := filesystem == "nfs" && strings.HasPrefix(mountedFrom, "fuse-t:") + if !macFUSE && !fuseT { + return errors.New("mount root is not backed by a supported FUSE provider") } return nil } + +func canonicalMountPath(path string) string { + resolved, err := filepath.EvalSymlinks(path) + if err == nil { + return filepath.Clean(resolved) + } + return filepath.Clean(path) +} From ea04f2d304ecc641277ceffef01f4c0b45ac4179 Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 03:12:32 +0800 Subject: [PATCH 15/33] fix: sanitize native filesystem contracts --- internal/compat/compat_test.go | 18 ++++++++++++++---- internal/compat/fsusage.go | 4 ++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/internal/compat/compat_test.go b/internal/compat/compat_test.go index ce1af99..43a0ba4 100644 --- a/internal/compat/compat_test.go +++ b/internal/compat/compat_test.go @@ -12,16 +12,17 @@ import ( func TestParseFSUsageProducesSanitizedOperationContract(t *testing.T) { trace := strings.Join([]string{ - "12:00:00.000 open F=3 (R_____) /Users/example/.codex/sessions/private.jsonl codex.123", + "12:00:00.000 open F=3 (R__________X___) /Users/example/.codex/sessions/private.jsonl codex.123", "12:00:00.001 read F=3 B=4096 /Users/example/.codex/sessions/private.jsonl codex.123", - "12:00:00.002 fsync F=3 /Users/example/.codex/sessions/private.jsonl codex.123", - "12:00:00.003 read F=3 B=4096 /Users/example/.codex/sessions/private.jsonl codex.123", + "12:00:00.002 fcntl F=3 codex.123", + "12:00:00.003 fsync F=3 /Users/example/.codex/sessions/private.jsonl codex.123", + "12:00:00.004 read F=3 B=4096 /Users/example/.codex/sessions/private.jsonl codex.123", }, "\n") contract, err := ParseFSUsage(strings.NewReader(trace), ContractOptions{Platform: "darwin", ClientKind: "cli", ClientVersion: "0.1.0"}) if err != nil { t.Fatalf("ParseFSUsage returned error: %v", err) } - if contract.TraceSHA256 == "" || len(contract.Operations) != 3 { + if contract.TraceSHA256 == "" || len(contract.Operations) != 4 { t.Fatalf("unexpected contract: %#v", contract) } encoded, err := json.Marshal(contract) @@ -34,6 +35,15 @@ func TestParseFSUsageProducesSanitizedOperationContract(t *testing.T) { if contract.Operations[1].Name != "read" || contract.Operations[1].Count != 2 { t.Fatalf("operation aggregation differs: %#v", contract.Operations) } + if got := contract.Operations[0].Signatures; len(got) != 1 || got[0].Value != "(R__________X___)" { + t.Fatalf("open signatures should contain only stable flags: %#v", got) + } + if got := contract.Operations[1].Signatures; len(got) != 0 { + t.Fatalf("read signatures leaked volatile descriptor or byte counts: %#v", got) + } + if got := contract.Operations[2].Signatures; len(got) != 1 || got[0].Value != "" { + t.Fatalf("fcntl signatures should preserve the stable command: %#v", got) + } } func TestEvaluateQuarantinesUnknownClientVersion(t *testing.T) { diff --git a/internal/compat/fsusage.go b/internal/compat/fsusage.go index a959ca3..ea03d9f 100644 --- a/internal/compat/fsusage.go +++ b/internal/compat/fsusage.go @@ -17,8 +17,8 @@ type ContractOptions struct { ClientVersion string } -var operationPattern = regexp.MustCompile(`(?i)\b(open|openat|close|read|pread|write|pwrite|fsync|fdatasync|stat|stat64|fstat|mmap|truncate|ftruncate|rename|unlink|flock|fcntl|clonefile)\b`) -var signaturePattern = regexp.MustCompile(`\b(?:F|B|O|FLAGS)=[A-Za-z0-9_()+-]+`) +var operationPattern = regexp.MustCompile(`(?i)\b(open|openat|close|read|pread|readv|write|pwrite|writev|fsync|fdatasync|stat|stat64|lstat|lstat64|fstat|fstat64|mmap|truncate|ftruncate|rename|renameat|unlink|unlinkat|flock|fcntl|clonefile|getattrlist)\b`) +var signaturePattern = regexp.MustCompile(`\([A-Z_]{4,32}\)|<[A-Z0-9_=+-]+>`) func ParseFSUsage(reader io.Reader, options ContractOptions) (Contract, error) { scanner := bufio.NewScanner(reader) From 34a83a61e2d4cbcbcf03f5eec8de4e7d30c98102 Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 14:22:03 +0800 Subject: [PATCH 16/33] fix: preserve native fallback during client quarantine --- docs/validation-macos-canary.md | 82 +++++++++++++---- internal/cli/fs.go | 31 ++++++- internal/cli/fs_service.go | 27 ++++++ internal/cli/fs_test.go | 106 ++++++++++++++++++++++ internal/mountfs/filesystem.go | 42 ++++++++- internal/mountfs/filesystem_test.go | 23 +++++ internal/mountfs/fuse_integration_test.go | 32 +++++++ internal/vfs/handles.go | 7 ++ internal/vfs/session_test.go | 22 +++++ 9 files changed, 349 insertions(+), 23 deletions(-) diff --git a/docs/validation-macos-canary.md b/docs/validation-macos-canary.md index 89912cb..da85e76 100644 --- a/docs/validation-macos-canary.md +++ b/docs/validation-macos-canary.md @@ -2,27 +2,73 @@ ## Current Status -Blocked before adapter compilation and real Codex routing. No real session route has been changed. +The FUSE-T macOS adapter and an isolated real Codex CLI canary have passed. The project remains at `fs-engine-preview`; it has not earned `platform-canary` or production status. Observed on 2026-07-12: - Codex desktop bundle: `26.707.41301` build `5103`. - Desktop-bundled CLI: `codex-cli 0.144.0-alpha.4`. - CLI resolved from `PATH`: `codex-cli 0.142.5`. -- macFUSE or osxfuse package receipt: not present. -- macFUSE or osxfuse filesystem bundle: not present. -- `go build -tags fuse ./cmd/codexfold`: blocked by missing `fuse.h`. -- Root `fs_usage` trace: not attempted because elevation has not been explicitly authorized. - -## Required Authorization Sequence - -1. Explicitly authorize a root `fs_usage` capture for the installed desktop-bundled CLI, the `PATH` CLI, and Codex Desktop workflows. The captured contract stores only sanitized operation names, counts, safe flags, and the trace digest. -2. Reconcile every observed operation with the platform-neutral filesystem. Unsupported rename, unlink, lock, mapping, watcher, or open-mode behavior blocks the adapter. -3. Explicitly authorize macFUSE installation and its system extension. The project must not self-elevate or install it implicitly. -4. Build with `-tags fuse`, mount only a generated fixture namespace, and pass the exact-byte, random-read, append, random-write, truncate, fsync, rename/unlink-policy, daemon-kill, and remount tests. -5. Shadow 5 to 10 archived real sessions without changing routes or removing native files. -6. Route retained-source canaries only after trace, adapter, doctor, shadow, compatibility, and explicit apply gates pass. -7. Exercise desktop click, CLI resume, send, tool use, fork, archive, unarchive, daemon termination, remount, sleep/wake, host restart, rollback, and unknown-version quarantine. -8. Keep status at `platform-canary` for seven incident-free days before considering `production-ready:macos`. - -Fixture tests and `fs-engine-preview` evidence do not satisfy any item above. +- FUSE adapter: FUSE-T `1.2.7`; macFUSE is not required or supported by this validation route. +- `CGO_ENABLED=1 go test -tags fuse ./...`: passed. +- The real FUSE-T fixture test passed mount, list, stat, exact reads, reopen, EOF append, fsync, random-write copy-on-write, truncate, rejected rename, unmount, and remount. +- A session added to the store after mount became readable through on-demand loading without remounting. +- An equal-length truncate from real Codex remained a no-op and did not hydrate a writable backing file. + +## Real Shadow Evidence + +Nine archived real sessions were copied into an isolated validation store without changing their Codex routes or deleting their source files. The set covered small and medium sessions, one session around 23 MiB, and one real fork parent/child pair. + +Results: + +- 9 of 9 complete-file SHA-256 comparisons passed. +- 90,000 of 90,000 random-range comparisons passed. +- The generated pack contained 1,182 objects. +- Pack doctor reported zero issues. + +These results validate exact reconstruction and random reads. They do not validate Desktop behavior or long-running service reliability. + +## Isolated Real Codex Canary + +The canary used an isolated Codex home and state database. It did not modify the user's real Codex routes. + +The validated sequence was: + +1. Start the real FUSE-T filesystem service. +2. Migrate one retained-source session through the product command. +3. Verify the complete mounted file and 10,000 random ranges. +4. Route only the isolated SQLite record to the mounted JSONL. +5. Resume with the unmodified desktop-bundled Codex CLI and append through `append.delta` without creating a complete backing file. +6. Stop, remount, resume again, run a shell tool, and append again without creating a complete backing file. +7. Roll back to a verified ordinary JSONL containing the latest visible bytes. +8. Resume and append successfully from that native fallback. + +The rollback safety regression also covers a native fallback that becomes newer than managed state. Unknown-version quarantine must preserve that current native route and must not overwrite it with stale managed bytes. + +A direct `SIGTERM` stopped the foreground service and removed the mount cleanly. A PTY `Ctrl-C` experiment left a reparented process once; this was a test-harness behavior and is not used as lifecycle evidence. + +## Remaining Gates + +The following gates are still open: + +- Complete native syscall contract for the `PATH` CLI after its launcher re-exec. +- Complete native syscall contract for Codex Desktop. +- Direct Desktop click and continued conversation against a virtual session. +- A real Codex fork created and continued while the source session is virtual. +- Sleep/wake and host-restart recovery. +- Unknown-version quarantine in the retained-source real canary path, beyond the isolated regression. +- Retained-source canary routes in the real Codex home. +- Seven incident-free days after reaching `platform-canary`. + +Until every applicable gate passes, the project must keep the capability at `fs-engine-preview`, retain original JSONL files, and avoid changing real Codex routes. + +## Reproducible Test Commands + +```sh +go test ./... +CGO_ENABLED=1 go test -tags fuse ./... -count=1 -timeout 5m +go test -race ./internal/mountfs ./internal/vfs ./internal/cli ./internal/service +CODEXFOLD_RUN_FUSE_TEST=1 CGO_ENABLED=1 \ + go test -tags fuse ./internal/mountfs \ + -run '^TestRealFuseMountNativeFileOperations$' -count=1 -v +``` diff --git a/internal/cli/fs.go b/internal/cli/fs.go index 6c5f13f..823a378 100644 --- a/internal/cli/fs.go +++ b/internal/cli/fs.go @@ -12,6 +12,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "time" "github.com/jstar0/codexfold/internal/cdc" @@ -261,7 +262,33 @@ func newFSServeCommand() *cobra.Command { defer cancel() closers := make([]io.Closer, 0) known := make(map[string]uint64) + var loadMu sync.Mutex + openState := func(state vfs.SessionState) (*vfs.Session, error) { + managed, resolver, err := openManagedSession(ctx, store, state) + if err != nil { + return nil, err + } + closers = append(closers, resolver) + known[state.SessionID] = state.Generation + return managed, nil + } + filesystem.SetSessionLoader(func(sessionID string) (*vfs.Session, error) { + loadMu.Lock() + defer loadMu.Unlock() + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return nil, err + } + for _, state := range states { + if state.SessionID == sessionID { + return openState(state) + } + } + return nil, os.ErrNotExist + }) load := func() error { + loadMu.Lock() + defer loadMu.Unlock() states, err := vfs.DiscoverSessionStates(store) if err != nil { return err @@ -270,15 +297,13 @@ func newFSServeCommand() *cobra.Command { if known[state.SessionID] == state.Generation { continue } - managed, resolver, err := openManagedSession(ctx, store, state) + managed, err := openState(state) if err != nil { return err } - closers = append(closers, resolver) if err := filesystem.UpsertSession(state.SessionID, managed); err != nil { return err } - known[state.SessionID] = state.Generation } return nil } diff --git a/internal/cli/fs_service.go b/internal/cli/fs_service.go index 8e250aa..8d1a3a4 100644 --- a/internal/cli/fs_service.go +++ b/internal/cli/fs_service.go @@ -238,6 +238,12 @@ func managedRoutesMatchCurrentBytes(ctx context.Context, home string, store stri if !ok { return false, fmt.Errorf("Codex route missing for managed session %s", state.SessionID) } + if isGeneratedNativeFallbackPath(current.RolloutPath, store, state.SessionID) { + if _, err := hashPath(current.RolloutPath); err != nil { + return false, err + } + continue + } managed, resolver, err := openManagedSession(ctx, store, state) if err != nil { return false, err @@ -274,6 +280,12 @@ func quarantineManagedRoutes(ctx context.Context, home string, store string) (in if !ok { return count, fmt.Errorf("Codex route missing for managed session %s", state.SessionID) } + if isGeneratedNativeFallbackPath(current.RolloutPath, store, state.SessionID) { + if _, err := hashPath(current.RolloutPath); err != nil { + return count, err + } + continue + } managed, resolver, err := openManagedSession(ctx, store, state) if err != nil { return count, err @@ -295,6 +307,21 @@ func quarantineManagedRoutes(ctx context.Context, home string, store string) (in return count, nil } +func isGeneratedNativeFallbackPath(path string, store string, sessionID string) bool { + if path == "" || store == "" || sessionID == "" { + return false + } + if filepath.Clean(filepath.Dir(path)) != filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) { + return false + } + switch filepath.Base(path) { + case "fallback-current.jsonl", "quarantine-current.jsonl": + return true + default: + return false + } +} + func hashManagedSession(ctx context.Context, session *vfs.Session) (vfs.NativeFile, error) { reader, err := session.OpenReader() if err != nil { diff --git a/internal/cli/fs_test.go b/internal/cli/fs_test.go index 19835c5..f3fa2dc 100644 --- a/internal/cli/fs_test.go +++ b/internal/cli/fs_test.go @@ -336,6 +336,112 @@ func TestFSRollbackUsesLatestVisibleBytesAfterVirtualAppend(t *testing.T) { } } +func TestFSUpdatePreflightPreservesNewerNativeFallbackAfterRollback(t *testing.T) { + allowFixtureMount(t) + home, storeDir, nativePath := fsFixture(t, true) + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + mount := filepath.Join(home, "mount") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(mount, "session.jsonl") + original, err := os.ReadFile(nativePath) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target, original, 0o600); err != nil { + t.Fatal(err) + } + executeFS(t, []string{"fs", "migrate", "session", "--codex-home", home, "--store", storeDir, "--mount", mount, "--cli", cliPath, "--desktop-app", "none", "--apply"}) + + state, err := managedState(storeDir, "session") + if err != nil { + t.Fatal(err) + } + managed, resolver, err := openManagedSession(context.Background(), storeDir, state) + if err != nil { + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + managedTail := []byte("{\"managed_tail\":true}\n") + if _, err := writer.Append(context.Background(), managedTail); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + _ = writer.Close() + _ = resolver.Close() + + executeFS(t, []string{"fs", "rollback", "session", "--codex-home", home, "--store", storeDir, "--apply"}) + sessions, err := codex.LoadSessions(home) + if err != nil { + t.Fatal(err) + } + fallbackPath := sessions[0].RolloutPath + if filepath.Base(fallbackPath) != "fallback-current.jsonl" { + t.Fatalf("rollback did not use the generated fallback: %s", fallbackPath) + } + + nativeTail := []byte("{\"native_tail\":true}\n") + fallback, err := os.OpenFile(fallbackPath, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + if _, err := fallback.Write(nativeTail); err != nil { + _ = fallback.Close() + t.Fatal(err) + } + if err := fallback.Sync(); err != nil { + _ = fallback.Close() + t.Fatal(err) + } + if err := fallback.Close(); err != nil { + t.Fatal(err) + } + want := append(append(append([]byte(nil), original...), managedTail...), nativeTail...) + beforeRoute := fallbackPath + + unknownCLI := fakeCLI(t, "9.9.9") + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "service", "update-preflight", "--codex-home", home, "--store", storeDir, "--cli", unknownCLI, "--desktop-app", "none", "--apply-quarantine", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("update preflight: %v", err) + } + var result FSUpdatePreflightResult + if err := json.Unmarshal(output.Bytes(), &result); err != nil { + t.Fatalf("decode preflight result: %v output=%s", err, output.String()) + } + if !result.Decision.Quarantine || result.Decision.RequiresNativeFallback || result.QuarantinedSessions != 0 { + t.Fatalf("unexpected fallback preflight result: %#v output=%s", result, output.String()) + } + sessions, err = codex.LoadSessions(home) + if err != nil { + t.Fatal(err) + } + if sessions[0].RolloutPath != beforeRoute { + t.Fatalf("preflight replaced newer native fallback: got=%s want=%s", sessions[0].RolloutPath, beforeRoute) + } + got, err := os.ReadFile(fallbackPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("preflight changed newer native fallback: got=%q want=%q", got, want) + } +} + func TestFSCompactCommitsNewExactGeneration(t *testing.T) { home, storeDir, nativePath := fsFixture(t, true) manifest, err := fold.LoadManifest(storeDir, "session") diff --git a/internal/mountfs/filesystem.go b/internal/mountfs/filesystem.go index df5fb8f..7aa3e8f 100644 --- a/internal/mountfs/filesystem.go +++ b/internal/mountfs/filesystem.go @@ -31,9 +31,11 @@ type fileHandle struct { type Filesystem struct { mu sync.RWMutex + loadMu sync.Mutex sessions map[string]*vfs.Session handles map[uint64]*fileHandle next uint64 + loader func(string) (*vfs.Session, error) } func New() *Filesystem { @@ -45,11 +47,12 @@ func (f *Filesystem) AddSession(sessionID string, session *vfs.Session) error { return errors.New("safe session ID and session are required") } f.mu.Lock() - defer f.mu.Unlock() if _, exists := f.sessions[sessionID]; exists { + f.mu.Unlock() return errors.New("session is already mounted") } f.sessions[sessionID] = session + f.mu.Unlock() return nil } @@ -63,6 +66,12 @@ func (f *Filesystem) UpsertSession(sessionID string, session *vfs.Session) error return nil } +func (f *Filesystem) SetSessionLoader(loader func(string) (*vfs.Session, error)) { + f.mu.Lock() + f.loader = loader + f.mu.Unlock() +} + func (f *Filesystem) ReadDir(name string) ([]string, syscall.Errno) { if cleanPath(name) != "/" { return nil, syscall.ENOTDIR @@ -305,10 +314,39 @@ func (f *Filesystem) sessionForPath(name string) (*vfs.Session, syscall.Errno) { sessionID := strings.TrimSuffix(strings.TrimPrefix(cleaned, "/"), ".jsonl") f.mu.RLock() session := f.sessions[sessionID] + loader := f.loader f.mu.RUnlock() - if session == nil { + if session != nil { + return session, 0 + } + if loader == nil { return nil, syscall.ENOENT } + f.loadMu.Lock() + defer f.loadMu.Unlock() + f.mu.RLock() + session = f.sessions[sessionID] + loader = f.loader + f.mu.RUnlock() + if session != nil { + return session, 0 + } + if loader == nil { + return nil, syscall.ENOENT + } + loaded, err := loader(sessionID) + if err != nil { + return nil, errnoFor(err) + } + if loaded == nil { + return nil, syscall.EIO + } + f.mu.Lock() + if session = f.sessions[sessionID]; session == nil { + f.sessions[sessionID] = loaded + session = loaded + } + f.mu.Unlock() return session, 0 } diff --git a/internal/mountfs/filesystem_test.go b/internal/mountfs/filesystem_test.go index 242b2d8..28835a6 100644 --- a/internal/mountfs/filesystem_test.go +++ b/internal/mountfs/filesystem_test.go @@ -138,6 +138,29 @@ func TestFilesystemPathTruncateUsesTheActiveWriter(t *testing.T) { } } +func TestFilesystemLoadsAMissingSessionOnceOnFirstAccess(t *testing.T) { + source := []byte("loaded") + session := mountSessionFixture(t, "loaded", source) + filesystem := New() + loads := 0 + filesystem.SetSessionLoader(func(sessionID string) (*vfs.Session, error) { + loads++ + if sessionID != "loaded" { + return nil, os.ErrNotExist + } + return session, nil + }) + for attempt := 0; attempt < 2; attempt++ { + attribute, errno := filesystem.Getattr("/loaded.jsonl") + if errno != 0 || attribute.Size != int64(len(source)) { + t.Fatalf("Getattr attempt %d = %#v errno=%v", attempt, attribute, errno) + } + } + if loads != 1 { + t.Fatalf("session loader calls = %d, want 1", loads) + } +} + func TestFilesystemRejectsUnsafeAndManagementMutations(t *testing.T) { filesystem, _ := mountFixture(t) if _, errno := filesystem.Open("/../session.jsonl", os.O_RDONLY); errno != syscall.ENOENT { diff --git a/internal/mountfs/fuse_integration_test.go b/internal/mountfs/fuse_integration_test.go index 2270405..921a21d 100644 --- a/internal/mountfs/fuse_integration_test.go +++ b/internal/mountfs/fuse_integration_test.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" "sync" + "sync/atomic" "testing" "time" @@ -67,6 +68,18 @@ func TestRealFuseMountNativeFileOperations(t *testing.T) { if err != nil || string(read) != string(source) { t.Fatalf("native read differs: %q err=%v", read, err) } + hotSource := []byte("hot\n") + hotSession := mountSessionFixture(t, "hot", hotSource) + var hotAvailable atomic.Bool + filesystem.SetSessionLoader(func(sessionID string) (*vfs.Session, error) { + if sessionID != "hot" || !hotAvailable.Load() { + return nil, os.ErrNotExist + } + return hotSession, nil + }) + hotAvailable.Store(true) + hotTarget := filepath.Join(mountPoint, "hot.jsonl") + waitForRealFile(t, hotTarget, hotSource) appendFile, err := os.OpenFile(target, os.O_RDWR|os.O_APPEND, 0) if err != nil { t.Fatal(err) @@ -187,6 +200,25 @@ func waitForRealUnmount(t *testing.T, mountPoint string) { t.Fatal("FUSE mount remained active after shutdown") } +func waitForRealFile(t *testing.T, path string, want []byte) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(path) + if err == nil { + if !bytes.Equal(data, want) { + t.Fatalf("hot-loaded file differs: got=%q want=%q", data, want) + } + return + } + if !os.IsNotExist(err) { + t.Fatalf("read hot-loaded file: %v", err) + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("hot-loaded file did not become visible") +} + type fuseFixtureReader map[string][]byte func (r fuseFixtureReader) ReadAt(_ context.Context, ref fold.ObjectRef, destination []byte, offset int64) (int, error) { diff --git a/internal/vfs/handles.go b/internal/vfs/handles.go index 2cad82d..699b29f 100644 --- a/internal/vfs/handles.go +++ b/internal/vfs/handles.go @@ -173,6 +173,13 @@ func (h *WriteHandle) Truncate(ctx context.Context, size int64) error { if size < 0 { return errors.New("negative truncate size") } + visible, err := h.session.VisibleInfo() + if err != nil { + return err + } + if size == visible.Size { + return nil + } path, err := h.session.ensureBacking(ctx) if err != nil { return err diff --git a/internal/vfs/session_test.go b/internal/vfs/session_test.go index d769379..e75e534 100644 --- a/internal/vfs/session_test.go +++ b/internal/vfs/session_test.go @@ -152,6 +152,28 @@ func TestSessionTruncateTransitionsToBacking(t *testing.T) { } } +func TestSessionEqualLengthTruncateDoesNotCreateBacking(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + if err := writer.Truncate(context.Background(), int64(len(source))); err != nil { + t.Fatalf("equal-length Truncate: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("Close writer: %v", err) + } + if state := session.State(); state.BackingPath != "" { + t.Fatalf("equal-length truncate created backing: %#v", state) + } + if info, err := os.Stat(session.State().DeltaPath); err != nil || info.Size() != 0 { + t.Fatalf("equal-length truncate changed delta: info=%#v err=%v", info, err) + } +} + func TestSessionInterruptedCopyOnWriteKeepsPreviousGeneration(t *testing.T) { root := t.TempDir() manifest, reader, source := sessionFixture(t, root) From aac49d95f5e513ab17acd521fe71b584f7bf1b89 Mon Sep 17 00:00:00 2001 From: King Star Date: Sun, 12 Jul 2026 14:36:26 +0800 Subject: [PATCH 17/33] docs: record macOS namespace compatibility boundary --- ...11-transparent-session-filesystem-design.md | 9 +++++---- docs/validation-macos-canary.md | 18 ++++++++++++++++-- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md index 5d21667..1109b1b 100644 --- a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md +++ b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md @@ -200,7 +200,7 @@ The engine must implement and test: | `flush/release` | Release handle state without triggering unsafe inline compaction | | `truncate` | Transition to writable backing before changing visible size | | random `write/pwrite` | Transition to writable backing before mutation | -| `rename/unlink` | Implement native-equivalent behavior when Codex tracing shows that the client uses it; otherwise management operations use explicit APIs | +| `rename/unlink` | Implement native-equivalent behavior when Codex tracing shows that the client uses it; Codex archive/unarchive currently requires canonical active/archive directories inside one virtual namespace | | `mmap` | Supported when the platform adapter can provide coherent read pages; otherwise platform readiness is blocked | | file locks | Preserve the lock behavior observed in the native Codex trace | @@ -259,10 +259,11 @@ The trace suite covers listing, opening, scrolling old history, resume, sending ### macOS -- Current reference adapter: macFUSE. +- Current reference adapter: FUSE-T `1.2.7`. - Service: user launch service with keep-alive and mount health monitoring. -- Required tests: APFS native baseline, Apple Silicon, Codex Desktop, Codex CLI, sleep/wake, network changes, user logout/login, daemon kill, mount restart, and Codex upgrade. -- If macFUSE remains the selected adapter, its installation and system-extension approval are separate user-authorized deployment steps. Development before that approval uses the platform-neutral engine and adapter mocks. +- Required tests: APFS native baseline, Apple Silicon, Codex Desktop, Codex CLI, canonical `sessions` and `archived_sessions` namespace moves, sleep/wake, network changes, user logout/login, daemon kill, mount restart, and Codex upgrade. +- FUSE-T is the validated userspace host for this project; macFUSE is not a prerequisite for the current macOS route. +- The current flat mount is insufficient for production because Codex moves archived rollouts between canonical directories. Platform readiness requires a directory-level namespace or an equivalent mechanism that keeps those moves native-compatible. ### Linux diff --git a/docs/validation-macos-canary.md b/docs/validation-macos-canary.md index da85e76..642a210 100644 --- a/docs/validation-macos-canary.md +++ b/docs/validation-macos-canary.md @@ -2,7 +2,7 @@ ## Current Status -The FUSE-T macOS adapter and an isolated real Codex CLI canary have passed. The project remains at `fs-engine-preview`; it has not earned `platform-canary` or production status. +The FUSE-T macOS adapter and an isolated real Codex CLI canary have passed for read, append, resume, fork, and child-session enrollment. The project remains at `fs-engine-preview`; it has not earned `platform-canary` or production status. Observed on 2026-07-12: @@ -14,6 +14,9 @@ Observed on 2026-07-12: - The real FUSE-T fixture test passed mount, list, stat, exact reads, reopen, EOF append, fsync, random-write copy-on-write, truncate, rejected rename, unmount, and remount. - A session added to the store after mount became readable through on-demand loading without remounting. - An equal-length truncate from real Codex remained a no-op and did not hydrate a writable backing file. +- A real fork from a virtual parent created a native child session; parent and child then resumed independently without cross-contamination. +- The child was archived while native, folded and packed, then enrolled through the mounted filesystem after remount. +- After an isolated database-only archive-flag reset, the enrolled child resumed through its virtual path and appended successfully. ## Real Shadow Evidence @@ -43,6 +46,15 @@ The validated sequence was: 7. Roll back to a verified ordinary JSONL containing the latest visible bytes. 8. Resume and append successfully from that native fallback. +The additional fork sequence was: + +1. Route the parent to the mounted virtual JSONL. +2. Run the unmodified CLI `fork` command with a real prompt. +3. Confirm the child was created as an ordinary native rollout while the parent stayed virtual. +4. Resume the native child and the virtual parent separately. +5. Archive the native child, fold and pack it, remount, and migrate the child through the real CLI route. +6. Resume the migrated child through the virtual path after an isolated database-only archive-flag reset. + The rollback safety regression also covers a native fallback that becomes newer than managed state. Unknown-version quarantine must preserve that current native route and must not overwrite it with stale managed bytes. A direct `SIGTERM` stopped the foreground service and removed the mount cleanly. A PTY `Ctrl-C` experiment left a reparented process once; this was a test-harness behavior and is not used as lifecycle evidence. @@ -54,7 +66,9 @@ The following gates are still open: - Complete native syscall contract for the `PATH` CLI after its launcher re-exec. - Complete native syscall contract for Codex Desktop. - Direct Desktop click and continued conversation against a virtual session. -- A real Codex fork created and continued while the source session is virtual. +- A real Codex fork created and continued while the source session is virtual: the CLI path passes; Desktop remains open. +- Transparent archive/unarchive for virtual routes. The current flat `/.jsonl` mount fails official `unarchive` because Codex requires canonical `sessions/YYYY/MM/DD/...` and `archived_sessions/...` paths and moves the rollout between them. A database-only archive-flag reset is test-only evidence and is not an implementation. +- A directory-level virtual namespace that keeps active and archived canonical paths inside one filesystem, or an equivalent native-compatible mechanism. - Sleep/wake and host-restart recovery. - Unknown-version quarantine in the retained-source real canary path, beyond the isolated regression. - Retained-source canary routes in the real Codex home. From a1944d32ec6b9381d9ebc00975074861df5452d2 Mon Sep 17 00:00:00 2001 From: King Star Date: Mon, 13 Jul 2026 16:56:56 +0800 Subject: [PATCH 18/33] feat: harden canonical session namespace and recovery --- ...arent-session-filesystem-implementation.md | 20 +- ...1-transparent-session-filesystem-design.md | 8 +- docs/validation-macos-canary.md | 75 +- internal/cli/fs.go | 374 +++++++++- internal/cli/fs_compatibility_import.go | 72 ++ internal/cli/fs_namespace.go | 187 +++++ internal/cli/fs_reconcile.go | 110 +++ internal/cli/fs_service.go | 501 ++++++++++++-- internal/cli/fs_test.go | 644 +++++++++++++++++- internal/compat/compat_test.go | 11 + internal/compat/fsusage.go | 2 +- internal/mountfs/filesystem.go | 503 +++++++++++++- internal/mountfs/filesystem_test.go | 382 +++++++++++ internal/mountfs/fuse_integration_test.go | 110 ++- internal/mountfs/host.go | 39 +- internal/mountfs/host_cgofuse.go | 333 ++++++++- internal/mountfs/host_safety_test.go | 58 ++ internal/mountid/identity.go | 34 + internal/reconcile/reconcile.go | 442 ++++++++++++ internal/reconcile/reconcile_test.go | 133 ++++ internal/reconcile/repair.go | 369 ++++++++++ internal/reconcile/repair_test.go | 98 +++ internal/service/mount_probe_darwin.go | 13 + internal/service/process_lock.go | 64 ++ internal/service/process_lock_unix.go | 22 + internal/service/process_lock_windows.go | 23 + internal/service/service.go | 63 +- internal/service/service_test.go | 68 +- internal/sessionns/activation.go | 331 +++++++++ internal/sessionns/activation_test.go | 259 +++++++ internal/sessionns/routing_guard.go | 125 ++++ .../activate-canonical-after-codex-exit.sh | 114 ++++ scripts/prepare-isolated-codex-home.sh | 48 ++ .../tests/test-prepare-isolated-codex-home.sh | 32 + .../tests/test-public-scripts-sanitized.sh | 13 + 35 files changed, 5524 insertions(+), 156 deletions(-) create mode 100644 internal/cli/fs_compatibility_import.go create mode 100644 internal/cli/fs_namespace.go create mode 100644 internal/cli/fs_reconcile.go create mode 100644 internal/mountfs/host_safety_test.go create mode 100644 internal/mountid/identity.go create mode 100644 internal/reconcile/reconcile.go create mode 100644 internal/reconcile/reconcile_test.go create mode 100644 internal/reconcile/repair.go create mode 100644 internal/reconcile/repair_test.go create mode 100644 internal/service/process_lock.go create mode 100644 internal/service/process_lock_unix.go create mode 100644 internal/service/process_lock_windows.go create mode 100644 internal/sessionns/activation.go create mode 100644 internal/sessionns/activation_test.go create mode 100644 internal/sessionns/routing_guard.go create mode 100755 scripts/activate-canonical-after-codex-exit.sh create mode 100755 scripts/prepare-isolated-codex-home.sh create mode 100755 scripts/tests/test-prepare-isolated-codex-home.sh create mode 100755 scripts/tests/test-public-scripts-sanitized.sh diff --git a/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md b/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md index b3e8fd0..ad4821e 100644 --- a/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md +++ b/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md @@ -6,7 +6,7 @@ **Architecture:** Extend Fold V1 with a block-addressable packed resolver, then place a platform-neutral exact-byte session engine above it. The engine composes an immutable manifest base with an append delta or verified writable backing; platform adapters only translate native file operations. Migration, compatibility quarantine, fallback, and promotion remain explicit journaled transactions, with real Codex routing disabled until shadow and platform gates pass. -**Tech Stack:** Go 1.26, zstd, Cobra, modernc SQLite, `cgofuse` v1.6.0 behind platform/build tags, macFUSE as the initial macOS adapter candidate. +**Tech Stack:** Go 1.26, zstd, Cobra, modernc SQLite, `cgofuse` v1.6.0 behind platform/build tags, and FUSE-T 1.2.7 as the validated macOS host. ## Global Constraints @@ -25,7 +25,8 @@ - `TF-013`: status output uses only `storage-engine`, `fs-engine-preview`, `platform-canary`, `production-ready:`, and `cross-platform-ready`. - `TF-014`: a migration snapshot is not deleted before `production-ready:` and the per-session retention gate. - `TF-015`: unknown client versions enter compatibility quarantine and cannot write a virtual route before current bytes are automatically routed to verified native backing. -- `TF-016`: macFUSE or another privileged prerequisite is not installed without explicit user authorization. +- `TF-016`: FUSE-T or another privileged prerequisite is not installed without explicit user authorization. +- `TF-017`: canonical namespace activation requires a verified CodexFold mount identity, write-sealed unmounted backing, route normalization, and a watcher that tolerates canonical and mount-alias spellings. - Mock and fixture evidence never satisfies a gate that names real Codex, a real adapter, client upgrade, host restart, or canary retention. - Public code and documentation contain no private paths, domains, credentials, real session IDs, or private control-plane dependency. @@ -49,6 +50,7 @@ | `TF-014` | 4, 6 | 10, 11 | | `TF-015` | 4, 6 | 10, 11 | | `TF-016` | 7, 9 | 11 | +| `TF-017` | 7, 9 | 10, 11 | --- @@ -467,7 +469,7 @@ Keep all path validation, handle ownership, session lookups, and error mapping i - [ ] **Step 4: Add `cgofuse` v1.6.0 behind explicit build constraints** -`host_cgofuse.go` uses `//go:build fuse && cgo` and translates cgofuse callbacks to the neutral filesystem. `host_stub.go` uses `//go:build !fuse || !cgo` and returns a typed prerequisite error. Default `go test ./...` and cross-compilation must not require macFUSE headers. +`host_cgofuse.go` uses `//go:build fuse && cgo` and translates cgofuse callbacks to the neutral filesystem. `host_stub.go` uses `//go:build !fuse || !cgo` and returns a typed prerequisite error. Default `go test ./...` and cross-compilation must not require an installed FUSE host. - [ ] **Step 5: Run default, race, and cross-platform compile tests** @@ -481,7 +483,7 @@ CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go test -c -o /tmp/codexfold-mountfs-win go test ./... -count=1 ``` -Expected: all PASS without installed macFUSE. +Expected: all PASS without an installed FUSE host. - [ ] **Step 6: Commit** @@ -571,7 +573,7 @@ Expected: FAIL because service APIs are absent. - [ ] **Step 3: Implement service lifecycle without self-elevation** -Render a per-user launchd plist and use `launchctl bootstrap/bootout/kickstart` only after an explicit apply command. Detect prerequisites but never install macFUSE or request elevation from library code. +Render a per-user launchd plist and use `launchctl bootstrap/bootout/kickstart` only after an explicit apply command. Detect prerequisites but never install FUSE-T or request elevation from library code. - [ ] **Step 4: Implement update compatibility guard** @@ -650,7 +652,7 @@ git commit -m "test: validate transparent filesystem engine preview" **Files:** - Create: `docs/validation-macos-canary.md` -- Modify only after approval: local macFUSE prerequisite and user launch service outside the public repository. +- Modify only after approval: local FUSE-T prerequisite and user launch service outside the public repository. - Modify only after all gates pass: selected Codex state rows through `codexfold fs migrate --apply`. **Interfaces:** @@ -665,7 +667,7 @@ With explicit elevation approval, run sanitized `fs_usage` tracing for list/open Add or correct platform operation tests before changing adapter code. Any unsupported observed operation blocks installation and migration. -- [ ] **Step 3: Request and apply macFUSE authorization** +- [ ] **Step 3: Request and apply the selected FUSE host authorization** Install the selected prerequisite only after explicit approval, build with `-tags fuse`, mount a temporary fixture namespace, and run fstest/fsx-equivalent plus the project operation suite. A mount alone is not success. @@ -692,9 +694,9 @@ No private path, session ID, trace content, credential, or control-plane name ma ## Plan Self-Review -- `TF-001` through `TF-016` each map to implementation and verification tasks. +- `TF-001` through `TF-017` each map to implementation and verification tasks. - Real-client, real-adapter, restart, upgrade, and retention gates remain in Task 11 and cannot be satisfied by Task 10 fixtures. -- macFUSE is a candidate and explicit authorization boundary, not a baked-in product promise. +- FUSE-T is the validated macOS host and remains an explicit authorization boundary; Linux and Windows adapters are certified independently. - The stale migration snapshot is never used as current fallback after virtual writes diverge. - The default build remains portable and does not require installed FUSE headers. - No task changes a real Codex route before shadow, compatibility, doctor, and explicit apply gates pass. diff --git a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md index 1109b1b..2f1db89 100644 --- a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md +++ b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md @@ -30,6 +30,7 @@ Every implementation plan, task, test report, release note, and control-plane st | `TF-014` | Native fallback deletion is disabled until platform production readiness and per-session retention gates pass. | | `TF-015` | A Codex client version without passing compatibility evidence enters compatibility quarantine: enrollment and destructive automation pause, and an already-routed session is automatically switched to a byte-verified current native writable backing before that client version may write. Routine client upgrades require no manual session preparation. | | `TF-016` | Platform filesystem prerequisites requiring elevated or system-extension approval are installed only after explicit user authorization. | +| `TF-017` | A canonical mount may never degrade into a writable ordinary directory or expose stale session files. The unmounted backing directory is empty and write-sealed, activation requires a live CodexFold mount identity, and service start succeeds only after the daemon and operational mount probe are both healthy. Desktop realpath rewrites from `CODEX_HOME/sessions` or `archived_sessions` into the mount alias are synchronously normalized in the Codex state database, and the route watcher accepts either spelling without exiting. | ### Decision Hierarchy @@ -46,7 +47,7 @@ An implementation change is therefore acceptable only when its requirement cover ### Drift Control - Each implementation-plan task lists the requirement IDs it implements or verifies. -- Each plan starts with a complete requirement-to-task coverage table for `TF-001` through `TF-016`. +- Each plan starts with a complete requirement-to-task coverage table for `TF-001` through `TF-017`. - A requirement with no implementation or verification task blocks plan approval. - Completion reports list fresh evidence by requirement ID and state any unmet ID explicitly. - Mock, fixture, or synthetic evidence cannot satisfy a requirement that names real Codex, a real platform adapter, a client upgrade, a host restart, or a canary period. @@ -332,6 +333,10 @@ After platform production readiness, enrollment is policy-driven rather than man - Startup recovery resolves every pending journal entry before accepting mounts. - A corrupt object, pack frame, manifest, or index blocks the affected session and preserves its native fallback. - Mount health failure blocks new migration and compaction. +- The ordinary directory underneath a canonical mount is empty, is never used as session storage, and remains non-writable whenever the mount is absent. +- Every mount instance exposes a process-generated identity through an operational read; provider type or `statfs` alone is not mount-health evidence. +- Namespace activation requires the live mount identity and may not accept a plain directory containing look-alike `sessions` trees. +- A store has one filesystem-host process lock. Service installation and restart return success only after launchd reports a running process and the mount identity is readable. - Database and global-state changes use optimistic revalidation and rollback. - A session with an active writer is never folded, removed, migrated, or rolled back. - A detected Codex Desktop or CLI version change immediately enters compatibility quarantine and schedules the native-operation compatibility suite. @@ -403,6 +408,7 @@ A single SHA-256 mismatch, unexpected Codex file operation, unresolved crash-rec - Daemon process health and mount health as separate states. - Mount path ownership, adapter version, and mounted-generation identity. +- Empty and write-sealed ordinary mount backing state whenever the adapter is not mounted. - Active pack generation, every resolver entry, object boundaries, stored checksum, raw length, and object SHA-256. - Manifest generation validity and complete virtual reconstruction. - Delta path, size, mtime, digest, synchronization state, and writer lease. diff --git a/docs/validation-macos-canary.md b/docs/validation-macos-canary.md index 642a210..42538af 100644 --- a/docs/validation-macos-canary.md +++ b/docs/validation-macos-canary.md @@ -2,21 +2,43 @@ ## Current Status -The FUSE-T macOS adapter and an isolated real Codex CLI canary have passed for read, append, resume, fork, and child-session enrollment. The project remains at `fs-engine-preview`; it has not earned `platform-canary` or production status. +The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have passed for read, append, resume, fork, child-session enrollment, canonical archive/unarchive moves, launchd restart, rollback, namespace deactivation, and unknown-version quarantine. The project remains at `fs-engine-preview` because the user Codex home is intentionally not enrolled and sleep or full host restart has not been exercised. + +Additional failure-containment evidence on 2026-07-13: + +- The mount backing directory rejects symlinks and any ordinary files before the host starts. +- The ordinary backing directory is mode `0500` while unmounted, contains no namespace entries, and rejects attempts to create a session branch. +- A live mount exposes a per-process random `.codexfold-health` generation. The mount probe requires both the supported FUSE provider and a successful read of that identity. +- Canonical namespace activation rejects a plain directory even when it contains `sessions` and `archived_sessions` look-alikes. +- A store-wide advisory process lock rejects a second filesystem host. +- Service install and start wait for both a running launchd process and a readable mount identity; failed readiness is booted out rather than reported as success. +- An isolated launchd canary passed canonical activation, transparent native passthrough, clean stop, write-sealed downtime, restart, `SIGKILL`, automatic PID replacement, post-restart append, final stop, and namespace deactivation. +- Codex Desktop `26.707.61608+5200` was observed resolving the canonical `sessions` symlink to `fold-fs/sessions` and writing that real path back to SQLite. Without a guard, the route watcher exited and Desktop later removed the stale thread row. Canonical activation now installs synchronous SQLite insert/update triggers that normalize both active and archived mount aliases back to `CODEX_HOME`, including Unicode paths. +- The route watcher independently accepts exact mount aliases and maps them to the same virtual route instead of terminating the filesystem service. +- A repaired isolated Desktop canary opened the complete history, appended `CODEXFOLD_ROUTE_GUARD_RESTART_OK`, survived a forced Desktop `SIGKILL`, reopened the same task, displayed the marker, retained the canonical SQLite path, and kept the FUSE service running. +- The sanitized Desktop trace was imported as the exact `26.707.61608+5200` contract with `statfs`, `getattr`, `read`, `open`, `rename`, `utimens`, `readdir`, `write`, `fsync`, `flush`, and `release`. Together with PATH `codex-cli 0.144.1`, compatibility evaluation is approved and not quarantined. +- The configured third-party Responses provider rejected model turns because its hosted image tool conflicted with the local `image_gen.imagegen` tool. The user append, filesystem durability, restart, and route normalization checks completed before that provider-level rejection; no model-reply claim is made for this focused canary. Observed on 2026-07-12: -- Codex desktop bundle: `26.707.41301` build `5103`. +- Codex desktop bundle: `26.707.51957` build `5175`. - Desktop-bundled CLI: `codex-cli 0.144.0-alpha.4`. - CLI resolved from `PATH`: `codex-cli 0.142.5`. +- Exact-version contracts passed together for the desktop-bundled CLI, the PATH CLI, and Desktop. - FUSE adapter: FUSE-T `1.2.7`; macFUSE is not required or supported by this validation route. - `CGO_ENABLED=1 go test -tags fuse ./...`: passed. -- The real FUSE-T fixture test passed mount, list, stat, exact reads, reopen, EOF append, fsync, random-write copy-on-write, truncate, rejected rename, unmount, and remount. +- The real FUSE-T fixture tests passed mount, list, stat, exact reads, reopen, EOF append, fsync, random-write copy-on-write, truncate, unmount, remount, and canonical managed rename in both directions. +- Managed extended attributes use hidden native carrier files, and AppleDouble sidecars follow managed archive/unarchive moves in both directions. +- Canonical mode exposes `sessions/YYYY/MM/DD/...` and `archived_sessions/...` in one FUSE-T namespace while passing unmanaged and newly created rollouts through a native backing tree. +- Canonical mode rejects a missing or relative native backing root instead of interpreting an empty root as the current working directory. +- The route watcher reads Codex state once per polling cycle and only updates a mounted session when its generation or canonical route changes. +- Canonical migration uses a two-phase cutover: it stages a hidden hard-link or cross-volume copy, waits for a daemon acknowledgement for the exact generation and route, then removes the native directory entry. A failed cutover retires the managed state and discards only the staged copy. +- The canonical migrate and rollback commands wait up to 15 seconds by default, but return immediately after the exact target bytes are verified. - A session added to the store after mount became readable through on-demand loading without remounting. - An equal-length truncate from real Codex remained a no-op and did not hydrate a writable backing file. - A real fork from a virtual parent created a native child session; parent and child then resumed independently without cross-contamination. - The child was archived while native, folded and packed, then enrolled through the mounted filesystem after remount. -- After an isolated database-only archive-flag reset, the enrolled child resumed through its virtual path and appended successfully. +- After an isolated database-only archive-flag reset, the enrolled child resumed through its virtual path and appended successfully. That reset remains historical evidence only; canonical mode no longer needs it for archive/unarchive. ## Real Shadow Evidence @@ -28,12 +50,13 @@ Results: - 90,000 of 90,000 random-range comparisons passed. - The generated pack contained 1,182 objects. - Pack doctor reported zero issues. +- A 2026-07-13 direct read-only scan of one current archived rollout processed 538,542 bytes and 192 records with zero invalid JSON records, zero missing sessions, and no file change during the scan. These results validate exact reconstruction and random reads. They do not validate Desktop behavior or long-running service reliability. ## Isolated Real Codex Canary -The canary used an isolated Codex home and state database. It did not modify the user's real Codex routes. +The canary used an isolated Codex home and state database. It did not modify the user's real Codex routes. The final full-flow run used the desktop-bundled `codex-cli 0.144.0-alpha.4` and a clean isolated root; the later focused route-guard run used Desktop `26.707.61608+5200` and PATH `codex-cli 0.144.1`. `scripts/prepare-isolated-codex-home.sh` copies the current `config.toml`, `auth.json`, and optional `models_cache.json` byte-for-byte and uses APFS clones for static plugin assets, so the canary uses the current provider configuration without allowing canary writes to modify the source home. The validated sequence was: @@ -45,6 +68,7 @@ The validated sequence was: 6. Stop, remount, resume again, run a shell tool, and append again without creating a complete backing file. 7. Roll back to a verified ordinary JSONL containing the latest visible bytes. 8. Resume and append successfully from that native fallback. +9. Repeat canonical migration with the two-phase daemon acknowledgement, then verify the source entry is removed only after the matching mounted route is live. The additional fork sequence was: @@ -53,24 +77,44 @@ The additional fork sequence was: 3. Confirm the child was created as an ordinary native rollout while the parent stayed virtual. 4. Resume the native child and the virtual parent separately. 5. Archive the native child, fold and pack it, remount, and migrate the child through the real CLI route. -6. Resume the migrated child through the virtual path after an isolated database-only archive-flag reset. +6. Resume the migrated child through the virtual path and verify parent/child append isolation by file size and marker content. + +The Desktop sequence was: + +1. Prepare the isolated Codex home from the current `config.toml` and `auth.json`, verify both SHA-256 values match before launch, and clone static plugin assets with copy-on-write isolation. +2. Start a separate Desktop process with an isolated Electron data directory and verify its child app-server has the isolated `CODEX_HOME`. +3. Open the managed session through `codex://threads/` and verify the virtual history is displayed. +4. Send a real message through the Desktop composer and verify the reply appears in the UI and only the append delta grows; no complete writable backing file is created. +5. Use Desktop's `Continue in new task from here` action, continue the child, and verify the child is native while the virtual parent remains unchanged by the child marker. +6. Import the sanitized Desktop operation trace as the exact `26.707.51957+5175` compatibility contract. + +A separate provider check started the unmodified desktop-bundled CLI with the prepared isolated home. The provider's model endpoint returned its non-OpenAI model catalog, the rollout recorded the configured provider and model, and the Responses request completed with `CODEXFOLD_FIXED_PROVIDER_OK`. Codex may rewrite home-relative plugin paths inside the isolated copy after startup; it did not replace the configured model provider. + +The canonical namespace sequence was: + +1. Expose the isolated Codex home's `sessions` and `archived_sessions` directories through one FUSE-T mount, with a separate native backing tree for unmanaged files. +2. Start with a managed parent at its canonical archived route. +3. Run the official `codex unarchive ` command and verify the SQLite route, archived flag, and mounted path moved to `sessions/YYYY/MM/DD/...`. +4. Resume the unarchived session with the unmodified desktop-bundled CLI and receive the requested canary marker. +5. Run the official `codex archive ` command and verify the route returned to `archived_sessions/...` with no active managed JSONL and no native JSONL duplicate. +6. Stop and restart the filesystem service, then repeat unarchive, resume, and archive successfully while the destination date directories exist only in the native backing tree. +7. Roll back the managed parent to a native JSONL, resume it through the unmodified CLI, stop the service, deactivate the namespace, and resume again from ordinary directories. + +The adapter exposes `setxattr`, `getxattr`, `listxattr`, and `removexattr` for managed files through hidden native carrier files. macOS AppleDouble metadata sidecars are moved with their managed rollout during archive and unarchive. The real FUSE-T integration test verifies that the xattr survives both moves and that stale sidecars do not remain at the old route. The rollback safety regression also covers a native fallback that becomes newer than managed state. Unknown-version quarantine must preserve that current native route and must not overwrite it with stale managed bytes. +The unknown-version canary used a fourth clean isolated home. A fake `codex-cli 9.9.9` triggered quarantine, materialized current bytes into `fs/fallbacks//quarantine-current.jsonl`, updated SQLite to that ordinary JSONL, retired the managed state, kept the daemon and mount healthy, and allowed namespace deactivation without changing the fallback route. + +The per-user launchd service was installed against an isolated home. It recovered a healthy FUSE-T mount after both `SIGTERM` and `SIGKILL`. Service installation initially blocked because `launchctl kickstart -k` waited for the old FUSE process; the lifecycle now uses non-destructive `kickstart`, returns promptly, and reports daemon and mount health separately. + A direct `SIGTERM` stopped the foreground service and removed the mount cleanly. A PTY `Ctrl-C` experiment left a reparented process once; this was a test-harness behavior and is not used as lifecycle evidence. ## Remaining Gates The following gates are still open: -- Complete native syscall contract for the `PATH` CLI after its launcher re-exec. -- Complete native syscall contract for Codex Desktop. -- Direct Desktop click and continued conversation against a virtual session. -- A real Codex fork created and continued while the source session is virtual: the CLI path passes; Desktop remains open. -- Transparent archive/unarchive for virtual routes. The current flat `/.jsonl` mount fails official `unarchive` because Codex requires canonical `sessions/YYYY/MM/DD/...` and `archived_sessions/...` paths and moves the rollout between them. A database-only archive-flag reset is test-only evidence and is not an implementation. -- A directory-level virtual namespace that keeps active and archived canonical paths inside one filesystem, or an equivalent native-compatible mechanism. -- Sleep/wake and host-restart recovery. -- Unknown-version quarantine in the retained-source real canary path, beyond the isolated regression. +- Sleep/wake and full host-restart recovery. These disruptive checks were not run against the user's active machine. - Retained-source canary routes in the real Codex home. - Seven incident-free days after reaching `platform-canary`. @@ -83,6 +127,5 @@ go test ./... CGO_ENABLED=1 go test -tags fuse ./... -count=1 -timeout 5m go test -race ./internal/mountfs ./internal/vfs ./internal/cli ./internal/service CODEXFOLD_RUN_FUSE_TEST=1 CGO_ENABLED=1 \ - go test -tags fuse ./internal/mountfs \ - -run '^TestRealFuseMountNativeFileOperations$' -count=1 -v + go test -tags fuse ./internal/mountfs -count=1 -v ``` diff --git a/internal/cli/fs.go b/internal/cli/fs.go index 823a378..45f4a4e 100644 --- a/internal/cli/fs.go +++ b/internal/cli/fs.go @@ -50,11 +50,12 @@ type FSServeResult struct { } type FSRollbackResult struct { - SessionID string `json:"session_id"` - From string `json:"from"` - Target vfs.NativeFile `json:"target"` - DryRun bool `json:"dry_run"` - Routed bool `json:"routed"` + SessionID string `json:"session_id"` + From string `json:"from"` + Target vfs.NativeFile `json:"target"` + RetiredState string `json:"retired_state,omitempty"` + DryRun bool `json:"dry_run"` + Routed bool `json:"routed"` } type FSCompactResult struct { @@ -85,12 +86,16 @@ func newFSCommand() *cobra.Command { command.AddCommand(newFSStatusCommand()) command.AddCommand(newFSDoctorCommand()) command.AddCommand(newFSCompatibilityCommand()) + command.AddCommand(newFSCompatibilityImportCommand()) command.AddCommand(newFSBenchmarkCommand()) command.AddCommand(newFSServeCommand()) command.AddCommand(newFSMigrateCommand()) command.AddCommand(newFSRollbackCommand()) command.AddCommand(newFSCompactCommand()) command.AddCommand(newFSRecoverCommand()) + command.AddCommand(newFSRepairRolloutCommand()) + command.AddCommand(newFSReconcileRolloutCommand()) + command.AddCommand(newFSNamespaceCommand()) command.AddCommand(newFSServiceCommand()) return command } @@ -230,6 +235,9 @@ func newFSServeCommand() *cobra.Command { var mountPoint string var apply bool var foreground bool + var canonicalNamespace bool + var nativeRoot string + var operationTracePath string var jsonOutput bool command := &cobra.Command{ Use: "serve", @@ -240,6 +248,12 @@ func newFSServeCommand() *cobra.Command { if err != nil { return err } + if canonicalNamespace { + if nativeRoot == "" || !filepath.IsAbs(nativeRoot) { + return errors.New("canonical namespace requires an absolute native root") + } + nativeRoot = filepath.Clean(nativeRoot) + } store := resolveFoldStore(home, storeDir) mount := defaultMountPoint(home, mountPoint) states, err := vfs.DiscoverSessionStates(store) @@ -254,14 +268,40 @@ func newFSServeCommand() *cobra.Command { _, err = fmt.Fprintf(command.OutOrStdout(), "dry_run=true mount=%s sessions=%d\n", mount, len(states)) return err } + processLock, err := service.AcquireProcessLock(filepath.Join(store, "fs", "service.lock")) + if err != nil { + return err + } + defer processLock.Close() if err := os.MkdirAll(mount, 0o700); err != nil { return err } + var operationRecorder func(string) + if operationTracePath != "" { + recorder, closer, err := newOperationRecorder(operationTracePath) + if err != nil { + return err + } + operationRecorder = recorder + defer closer.Close() + } + if canonicalNamespace { + for _, directory := range []string{"sessions", "archived_sessions"} { + if err := os.MkdirAll(filepath.Join(nativeRoot, directory), 0o700); err != nil { + return err + } + } + } filesystem := mountfs.New() + if canonicalNamespace { + filesystem = mountfs.NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + } ctx, cancel := context.WithCancel(command.Context()) defer cancel() closers := make([]io.Closer, 0) known := make(map[string]uint64) + knownRoutes := make(map[string]string) var loadMu sync.Mutex openState := func(state vfs.SessionState) (*vfs.Session, error) { managed, resolver, err := openManagedSession(ctx, store, state) @@ -269,7 +309,6 @@ func newFSServeCommand() *cobra.Command { return nil, err } closers = append(closers, resolver) - known[state.SessionID] = state.Generation return managed, nil } filesystem.SetSessionLoader(func(sessionID string) (*vfs.Session, error) { @@ -281,7 +320,11 @@ func newFSServeCommand() *cobra.Command { } for _, state := range states { if state.SessionID == sessionID { - return openState(state) + managed, err := openState(state) + if err == nil { + known[state.SessionID] = state.Generation + } + return managed, err } } return nil, os.ErrNotExist @@ -293,7 +336,56 @@ func newFSServeCommand() *cobra.Command { if err != nil { return err } + routes := make(map[string]string) + if canonicalNamespace { + routes, err = discoverCanonicalRoutes(home, mount, store, states, codex.LoadSessions) + if err != nil { + return err + } + } + seen := make(map[string]struct{}, len(states)) for _, state := range states { + seen[state.SessionID] = struct{}{} + if canonicalNamespace { + route, exists := routes[state.SessionID] + if !exists { + if _, mounted := known[state.SessionID]; mounted { + if err := filesystem.RemoveSession(state.SessionID); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + delete(known, state.SessionID) + delete(knownRoutes, state.SessionID) + } + continue + } + generation, generationKnown := known[state.SessionID] + if generationKnown && generation == state.Generation { + if knownRoutes[state.SessionID] == route { + continue + } + if err := filesystem.MoveSessionAt(state.SessionID, route); err != nil { + return err + } + knownRoutes[state.SessionID] = route + if err := writeMountAcknowledgement(store, state.SessionID, state.Generation, route); err != nil { + return err + } + continue + } + managed, err := openState(state) + if err != nil { + return err + } + if err := filesystem.UpsertSessionAt(state.SessionID, route, managed); err != nil { + return err + } + known[state.SessionID] = state.Generation + knownRoutes[state.SessionID] = route + if err := writeMountAcknowledgement(store, state.SessionID, state.Generation, route); err != nil { + return err + } + continue + } if known[state.SessionID] == state.Generation { continue } @@ -304,6 +396,17 @@ func newFSServeCommand() *cobra.Command { if err := filesystem.UpsertSession(state.SessionID, managed); err != nil { return err } + known[state.SessionID] = state.Generation + } + for sessionID := range known { + if _, exists := seen[sessionID]; exists { + continue + } + if err := filesystem.RemoveSession(sessionID); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + delete(known, sessionID) + delete(knownRoutes, sessionID) } return nil } @@ -329,7 +432,7 @@ func newFSServeCommand() *cobra.Command { } } }() - mountErr := mountfs.Mount(ctx, mountfs.HostOptions{MountPoint: mount, Filesystem: filesystem, Foreground: foreground}) + mountErr := mountfs.Mount(ctx, mountfs.HostOptions{MountPoint: mount, Filesystem: filesystem, Foreground: foreground, OperationRecorder: operationRecorder}) cancel() <-watcherDone for _, closer := range closers { @@ -348,6 +451,9 @@ func newFSServeCommand() *cobra.Command { command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") command.Flags().BoolVar(&apply, "apply", false, "Start the filesystem host") command.Flags().BoolVar(&foreground, "foreground", true, "Keep the FUSE host in the foreground") + command.Flags().BoolVar(&canonicalNamespace, "canonical-namespace", false, "Expose sessions and archived_sessions as a shared virtual namespace") + command.Flags().StringVar(&nativeRoot, "native-root", "", "Backing root for unmanaged canonical session files") + command.Flags().StringVar(&operationTracePath, "operation-trace", "", "Absolute path for sanitized FUSE operation names") command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output for dry-run") return command } @@ -356,8 +462,9 @@ func newFSMigrateCommand() *cobra.Command { var codexHome string var storeDir string var mountPoint string + var nativeRoot string var mountWait time.Duration - var apply bool + var apply, canonicalNamespace, compatibilityCanary bool var jsonOutput bool var compatibility compatibilityFlags command := &cobra.Command{ @@ -378,40 +485,126 @@ func newFSMigrateCommand() *cobra.Command { if !session.Archived { return errors.New("only archived sessions are eligible for filesystem migration") } - shadow, err := fsctl.Shadow(command.Context(), session.RolloutPath, view, fsctl.ShadowOptions{RandomReads: 10000, Seed: 1}) + mount := defaultMountPoint(home, mountPoint) + sourcePath := session.RolloutPath + target := filepath.Join(mount, session.ID+".jsonl") + if canonicalNamespace { + if nativeRoot == "" { + nativeRoot = filepath.Join(home, "fold-native") + } + sourcePath, err = canonicalNativeRoute(home, nativeRoot, session.RolloutPath) + if err != nil { + return err + } + target, err = canonicalMountRoute(home, mount, session.RolloutPath) + if err != nil { + return err + } + } + shadow, err := fsctl.Shadow(command.Context(), sourcePath, view, fsctl.ShadowOptions{RandomReads: 10000, Seed: 1}) if err != nil { return err } - mount := defaultMountPoint(home, mountPoint) - target := filepath.Join(mount, session.ID+".jsonl") - native := vfs.NativeFile{Path: session.RolloutPath, Bytes: shadow.Bytes, SHA256: shadow.SHA256} + native := vfs.NativeFile{Path: sourcePath, Bytes: shadow.Bytes, SHA256: shadow.SHA256} result := FSMigrateResult{SessionID: session.ID, Native: native, Target: target, Shadow: shadow, DryRun: !apply} if apply { if err := requireStorageHealth(command.Context(), store); err != nil { return err } - compatibilityResult, err := evaluateCompatibility(command.Context(), store, compatibility) - if err != nil { - return err - } - if len(compatibilityResult.DetectionErrors) != 0 || !compatibilityResult.Evaluation.Approved { - return errors.New("installed Codex client versions are not covered by compatibility contracts") + if compatibilityCanary { + userHome, err := os.UserHomeDir() + if err != nil { + return err + } + if err := validateCompatibilityCanary(home, filepath.Join(userHome, ".codex"), store, canonicalNamespace, compatibility); err != nil { + return err + } + } else { + compatibilityResult, err := evaluateCompatibility(command.Context(), store, compatibility) + if err != nil { + return err + } + if len(compatibilityResult.DetectionErrors) != 0 || !compatibilityResult.Evaluation.Approved { + return errors.New("installed Codex client versions are not covered by compatibility contracts") + } } if err := mountHealthProbe(mount); err != nil { return fmt.Errorf("filesystem mount point is not healthy: %w", err) } - if _, err := vfs.OpenSession(command.Context(), vfs.SessionOptions{Root: store, ManifestPath: fold.ManifestPath(store, session.ID), Manifest: manifest, Reader: resolver, NativeSnapshot: native}); err != nil { - return err + canonicalSource := "" + canonicalRoute := "" + if canonicalNamespace { + if _, err := os.Stat(filepath.Join(store, "fs", "sessions", session.ID, "state.json")); err == nil { + return errors.New("session is already managed") + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + canonicalSource = native.Path + canonicalRoute, err = canonicalNamespaceRoute(home, mount, session.RolloutPath) + if err != nil { + return err + } + retained, err := retainCanonicalSnapshot(store, session.ID, native) + if err != nil { + return err + } + native = retained + result.Native = retained + } + rollbackCanonicalMigration := func(cause error) error { + if !canonicalNamespace { + return cause + } + if _, err := os.Stat(filepath.Join(store, "fs", "sessions", session.ID)); err == nil { + if _, retireErr := retireManagedState(store, session.ID); retireErr != nil { + return errors.Join(cause, retireErr) + } + } + if err := restoreCanonicalSnapshotSource(canonicalSource, native.Path); err != nil { + return errors.Join(cause, err) + } + return cause + } + managed, err := vfs.OpenSession(command.Context(), vfs.SessionOptions{Root: store, ManifestPath: fold.ManifestPath(store, session.ID), Manifest: manifest, Reader: resolver, NativeSnapshot: native}) + if err != nil { + return rollbackCanonicalMigration(err) + } + if canonicalNamespace { + if err := waitForMountAcknowledgement(command.Context(), store, session.ID, managed.State().Generation, canonicalRoute, mountWait); err != nil { + return rollbackCanonicalMigration(fmt.Errorf("wait for canonical mount acknowledgement: %w", err)) + } + sessions, err := codex.LoadSessions(home) + if err != nil { + return rollbackCanonicalMigration(err) + } + current, err := findSession(sessions, session.ID) + if err != nil || filepath.Clean(current.RolloutPath) != filepath.Clean(session.RolloutPath) { + return rollbackCanonicalMigration(errors.New("canonical Codex route changed during migration")) + } + if err := finalizeCanonicalSnapshotSource(canonicalSource, native); err != nil { + return rollbackCanonicalMigration(err) + } } targetFile, err := waitForTarget(command.Context(), target, mountWait) if err != nil { - return fmt.Errorf("verify mounted target: %w", err) + return rollbackCanonicalMigration(fmt.Errorf("verify mounted target: %w", err)) } if targetFile.Bytes != shadow.Bytes || targetFile.SHA256 != shadow.SHA256 { - return errors.New("mounted target differs from the shadow-verified native session") + return rollbackCanonicalMigration(errors.New("mounted target differs from the shadow-verified native session")) } - if _, err := codex.RouteSession(command.Context(), codex.RouteOptions{CodexHome: home, SessionID: session.ID, ExpectedPath: session.RolloutPath, Target: codex.RouteTarget{Path: target, Bytes: targetFile.Bytes, SHA256: targetFile.SHA256}}); err != nil { - return err + if !canonicalNamespace { + if _, err := codex.RouteSession(command.Context(), codex.RouteOptions{CodexHome: home, SessionID: session.ID, ExpectedPath: session.RolloutPath, Target: codex.RouteTarget{Path: target, Bytes: targetFile.Bytes, SHA256: targetFile.SHA256}}); err != nil { + return err + } + } else { + sessions, err := codex.LoadSessions(home) + if err != nil { + return err + } + current, err := findSession(sessions, session.ID) + if err != nil || filepath.Clean(current.RolloutPath) != filepath.Clean(session.RolloutPath) { + return rollbackCanonicalMigration(errors.New("canonical Codex route changed during migration")) + } } result.Routed = true result.DryRun = false @@ -426,8 +619,11 @@ func newFSMigrateCommand() *cobra.Command { command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") - command.Flags().DurationVar(&mountWait, "mount-wait", 5*time.Second, "Maximum wait for the mounted session target") + command.Flags().BoolVar(&canonicalNamespace, "canonical-namespace", false, "Enroll the session at its canonical Codex path without changing SQLite routing") + command.Flags().StringVar(&nativeRoot, "native-root", "", "Canonical native snapshot root; defaults to /fold-native") + command.Flags().DurationVar(&mountWait, "mount-wait", 15*time.Second, "Maximum wait for the mounted session target") command.Flags().BoolVar(&apply, "apply", false, "Enroll and route the session after all gates pass") + command.Flags().BoolVar(&compatibilityCanary, "compatibility-canary", false, "Allow an isolated canonical canary with both client checks explicitly skipped") addCompatibilityFlags(command, &compatibility) command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") return command @@ -436,8 +632,11 @@ func newFSMigrateCommand() *cobra.Command { func newFSRollbackCommand() *cobra.Command { var codexHome string var storeDir string + var mountPoint string + var nativeRoot string var targetPath string - var apply bool + var mountWait time.Duration + var apply, canonicalNamespace bool var jsonOutput bool command := &cobra.Command{ Use: "rollback ", @@ -461,11 +660,49 @@ func newFSRollbackCommand() *cobra.Command { if err != nil { return err } - if targetPath == "" { - targetPath = filepath.Join(store, "fs", "sessions", state.SessionID, "fallback-current.jsonl") + currentNativeFallback := !canonicalNamespace && isGeneratedNativeFallbackPath(current.RolloutPath, store, state.SessionID) + mount := defaultMountPoint(home, mountPoint) + if canonicalNamespace { + if nativeRoot == "" { + nativeRoot = filepath.Join(home, "fold-native") + } + canonicalTarget, err := canonicalNativeRoute(home, nativeRoot, current.RolloutPath) + if err != nil { + return err + } + if targetPath != "" && filepath.Clean(targetPath) != filepath.Clean(canonicalTarget) { + return errors.New("canonical rollback target must remain inside the retained native namespace") + } + targetPath = canonicalTarget + } else if targetPath == "" { + targetPath = filepath.Join(store, "fs", "fallbacks", state.SessionID, "fallback-current.jsonl") } result := FSRollbackResult{SessionID: state.SessionID, From: current.RolloutPath, Target: vfs.NativeFile{Path: filepath.Clean(targetPath)}, DryRun: !apply} if apply { + if currentNativeFallback { + target, err := hashPath(current.RolloutPath) + if err != nil { + return err + } + retiredState, err := retireManagedState(store, state.SessionID) + if err != nil { + return err + } + result.Target = target + result.RetiredState = retiredState + result.Routed = true + result.DryRun = false + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "session=%s dry_run=%t routed=%t from=%s target=%s\n", result.SessionID, result.DryRun, result.Routed, result.From, result.Target.Path) + return err + } + if canonicalNamespace { + if err := mountHealthProbe(mount); err != nil { + return fmt.Errorf("canonical filesystem mount is not healthy: %w", err) + } + } managed, resolver, err := openManagedSession(command.Context(), store, state) if err != nil { return err @@ -475,8 +712,35 @@ func newFSRollbackCommand() *cobra.Command { if err != nil { return err } - if _, err := codex.RouteSession(command.Context(), codex.RouteOptions{CodexHome: home, SessionID: state.SessionID, ExpectedPath: current.RolloutPath, Target: codex.RouteTarget{Path: target.Path, Bytes: target.Bytes, SHA256: target.SHA256}}); err != nil { - return err + if canonicalNamespace { + retiredState, err := retireManagedState(store, state.SessionID) + if err != nil { + return err + } + retiredSnapshot, err := retireCanonicalNativeSnapshot(store, nativeRoot, state.SessionID, state.NativeSnapshot.Path, target.Path, retiredState) + if err != nil { + _ = restoreManagedState(store, state.SessionID, retiredState) + return err + } + mountedTarget, err := canonicalMountRoute(home, mount, current.RolloutPath) + if err == nil { + _, err = waitForTargetMatch(command.Context(), mountedTarget, target, mountWait) + } + if err != nil { + _ = restoreCanonicalNativeSnapshot(state.NativeSnapshot.Path, retiredSnapshot) + _ = restoreManagedState(store, state.SessionID, retiredState) + return fmt.Errorf("verify canonical native rollback: %w", err) + } + result.RetiredState = retiredState + } else { + if _, err := codex.RouteSession(command.Context(), codex.RouteOptions{CodexHome: home, SessionID: state.SessionID, ExpectedPath: current.RolloutPath, Target: codex.RouteTarget{Path: target.Path, Bytes: target.Bytes, SHA256: target.SHA256}}); err != nil { + return err + } + retiredState, err := retireManagedState(store, state.SessionID) + if err != nil { + return err + } + result.RetiredState = retiredState } result.Target = target result.Routed = true @@ -491,6 +755,10 @@ func newFSRollbackCommand() *cobra.Command { } command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + command.Flags().BoolVar(&canonicalNamespace, "canonical-namespace", false, "Restore current bytes to canonical native backing without changing SQLite routing") + command.Flags().StringVar(&nativeRoot, "native-root", "", "Canonical native rollback root; defaults to /fold-native") + command.Flags().DurationVar(&mountWait, "mount-wait", 15*time.Second, "Maximum wait for native passthrough after state retirement") command.Flags().StringVar(&targetPath, "to", "", "Native rollback target; defaults to the managed session directory") command.Flags().BoolVar(&apply, "apply", false, "Materialize current bytes and update the Codex route") command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") @@ -711,6 +979,26 @@ func evaluateCompatibility(ctx context.Context, store string, flags compatibilit return result, nil } +func validateCompatibilityCanary(home string, defaultHome string, store string, canonical bool, flags compatibilityFlags) error { + home = filepath.Clean(home) + defaultHome = filepath.Clean(defaultHome) + store = filepath.Clean(store) + if !canonical { + return errors.New("compatibility canary requires canonical namespace mode") + } + if home == defaultHome { + return errors.New("compatibility canary is forbidden for the real Codex home") + } + relative, err := filepath.Rel(home, store) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) || filepath.IsAbs(relative) { + return errors.New("compatibility canary store must be inside the isolated Codex home") + } + if flags.cliPath != "none" || flags.desktopPath != "none" { + return errors.New("compatibility canary requires --cli none and --desktop-app none") + } + return nil +} + func openFoldView(home string, store string, sessionID string) (codex.Session, fold.Manifest, *pack.Resolver, *vfs.View, error) { sessions, err := codex.LoadSessions(home) if err != nil { @@ -926,6 +1214,30 @@ func waitForTarget(ctx context.Context, target string, timeout time.Duration) (v } } +func waitForTargetMatch(ctx context.Context, target string, expected vfs.NativeFile, timeout time.Duration) (vfs.NativeFile, error) { + if timeout <= 0 { + timeout = 5 * time.Second + } + deadline := time.Now().Add(timeout) + for { + file, err := hashPath(target) + if err == nil && file.Bytes == expected.Bytes && file.SHA256 == expected.SHA256 { + return file, nil + } + if err != nil && !errors.Is(err, os.ErrNotExist) { + return vfs.NativeFile{}, err + } + if time.Now().After(deadline) { + return vfs.NativeFile{}, errors.New("timed out waiting for matching mounted session") + } + select { + case <-ctx.Done(): + return vfs.NativeFile{}, ctx.Err() + case <-time.After(25 * time.Millisecond): + } + } +} + func hashPath(path string) (vfs.NativeFile, error) { file, err := os.Open(path) if err != nil { diff --git a/internal/cli/fs_compatibility_import.go b/internal/cli/fs_compatibility_import.go new file mode 100644 index 0000000..c92dff8 --- /dev/null +++ b/internal/cli/fs_compatibility_import.go @@ -0,0 +1,72 @@ +package cli + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + + "github.com/jstar0/codexfold/internal/codex" + "github.com/jstar0/codexfold/internal/compat" + "github.com/spf13/cobra" +) + +type FSCompatibilityImportResult struct { + Contract compat.Contract `json:"contract"` + Path string `json:"path,omitempty"` + DryRun bool `json:"dry_run"` +} + +func newFSCompatibilityImportCommand() *cobra.Command { + var codexHome, storeDir, tracePath, clientKind, clientVersion, platform string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "compatibility-import", + Short: "Import a sanitized exact-version contract from a real filesystem trace", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + if !filepath.IsAbs(tracePath) || (clientKind != "cli" && clientKind != "desktop") || clientVersion == "" || platform == "" { + return errors.New("absolute trace path, cli or desktop client kind, version, and platform are required") + } + trace, err := os.Open(tracePath) + if err != nil { + return err + } + contract, parseErr := compat.ParseFSUsage(trace, compat.ContractOptions{Platform: platform, ClientKind: clientKind, ClientVersion: clientVersion}) + closeErr := trace.Close() + if parseErr != nil { + return parseErr + } + if closeErr != nil { + return closeErr + } + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + result := FSCompatibilityImportResult{Contract: contract, DryRun: !apply} + if apply { + result.Path, err = compat.Save(filepath.Join(resolveFoldStore(home, storeDir), "compatibility"), contract) + if err != nil { + return err + } + result.DryRun = false + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "kind=%s version=%s operations=%d dry_run=%t path=%s\n", contract.ClientKind, contract.ClientVersion, len(contract.Operations), result.DryRun, result.Path) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().StringVar(&tracePath, "trace", "", "Absolute path to a real fs_usage-compatible trace") + command.Flags().StringVar(&clientKind, "client-kind", "", "Client kind: cli or desktop") + command.Flags().StringVar(&clientVersion, "client-version", "", "Exact client version represented by the trace") + command.Flags().StringVar(&platform, "platform", runtime.GOOS, "Trace platform") + command.Flags().BoolVar(&apply, "apply", false, "Persist the sanitized contract") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} diff --git a/internal/cli/fs_namespace.go b/internal/cli/fs_namespace.go new file mode 100644 index 0000000..6bf160b --- /dev/null +++ b/internal/cli/fs_namespace.go @@ -0,0 +1,187 @@ +package cli + +import ( + "errors" + "fmt" + "path/filepath" + + "github.com/jstar0/codexfold/internal/codex" + "github.com/jstar0/codexfold/internal/sessionns" + "github.com/jstar0/codexfold/internal/vfs" + "github.com/spf13/cobra" +) + +type FSNamespaceResult struct { + sessionns.Result + DryRun bool `json:"dry_run"` +} + +func newFSNamespaceCommand() *cobra.Command { + command := &cobra.Command{Use: "namespace", Short: "Manage the canonical Codex session directory namespace"} + command.AddCommand(newFSNamespaceStatusCommand()) + command.AddCommand(newFSNamespaceActivateCommand()) + command.AddCommand(newFSNamespaceDeactivateCommand()) + command.AddCommand(newFSNamespaceRecoverCommand()) + return command +} + +func newFSNamespaceStatusCommand() *cobra.Command { + var codexHome, mountPoint, nativeRoot string + var jsonOutput bool + command := &cobra.Command{ + Use: "status", + Short: "Inspect the canonical namespace without changing it", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + options, err := namespaceOptions(codexHome, mountPoint, nativeRoot) + if err != nil { + return err + } + result, err := sessionns.Inspect(options) + if err != nil { + return err + } + return writeNamespaceResult(command, FSNamespaceResult{Result: result, DryRun: true}, jsonOutput) + }, + } + addNamespaceFlags(command, &codexHome, &mountPoint, &nativeRoot) + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSNamespaceActivateCommand() *cobra.Command { + var codexHome, mountPoint, nativeRoot string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "activate", + Short: "Atomically route Codex session directories through the canonical mount", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + options, err := namespaceOptions(codexHome, mountPoint, nativeRoot) + if err != nil { + return err + } + if !apply { + result, err := sessionns.Inspect(options) + if err != nil { + return err + } + return writeNamespaceResult(command, FSNamespaceResult{Result: result, DryRun: true}, jsonOutput) + } + if err := mountHealthProbe(options.Mount); err != nil { + return fmt.Errorf("canonical filesystem mount is not healthy: %w", err) + } + result, err := sessionns.Activate(options) + if err != nil { + return err + } + return writeNamespaceResult(command, FSNamespaceResult{Result: result}, jsonOutput) + }, + } + addNamespaceFlags(command, &codexHome, &mountPoint, &nativeRoot) + command.Flags().BoolVar(&apply, "apply", false, "Move native directories and install canonical links") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSNamespaceDeactivateCommand() *cobra.Command { + var codexHome, storeDir, mountPoint, nativeRoot string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "deactivate", + Short: "Restore ordinary Codex session directories from the retained native tree", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + options, err := namespaceOptions(codexHome, mountPoint, nativeRoot) + if err != nil { + return err + } + if !apply { + result, err := sessionns.Inspect(options) + if err != nil { + return err + } + return writeNamespaceResult(command, FSNamespaceResult{Result: result, DryRun: true}, jsonOutput) + } + states, err := vfs.DiscoverSessionStates(resolveFoldStore(options.Home, storeDir)) + if err != nil { + return err + } + if len(states) != 0 { + return errors.New("rollback all managed sessions before deactivating the namespace") + } + if err := mountHealthProbe(options.Mount); err == nil { + return errors.New("stop the filesystem service before deactivating the namespace") + } + result, err := sessionns.Deactivate(options) + if err != nil { + return err + } + return writeNamespaceResult(command, FSNamespaceResult{Result: result}, jsonOutput) + }, + } + addNamespaceFlags(command, &codexHome, &mountPoint, &nativeRoot) + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().BoolVar(&apply, "apply", false, "Remove canonical links and restore native directories") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSNamespaceRecoverCommand() *cobra.Command { + var codexHome, mountPoint, nativeRoot string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "recover", + Short: "Recover an interrupted namespace activation or deactivation", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + options, err := namespaceOptions(codexHome, mountPoint, nativeRoot) + if err != nil { + return err + } + var result sessionns.Result + if apply { + result, err = sessionns.Recover(options) + } else { + result, err = sessionns.Inspect(options) + } + if err != nil { + return err + } + return writeNamespaceResult(command, FSNamespaceResult{Result: result, DryRun: !apply}, jsonOutput) + }, + } + addNamespaceFlags(command, &codexHome, &mountPoint, &nativeRoot) + command.Flags().BoolVar(&apply, "apply", false, "Apply journal recovery") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func namespaceOptions(codexHome string, mountPoint string, nativeRoot string) (sessionns.Options, error) { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return sessionns.Options{}, err + } + mount := defaultMountPoint(home, mountPoint) + if nativeRoot == "" { + nativeRoot = filepath.Join(home, "fold-native") + } + if !filepath.IsAbs(nativeRoot) { + return sessionns.Options{}, errors.New("native root must be absolute") + } + return sessionns.Options{Home: home, Mount: mount, NativeRoot: filepath.Clean(nativeRoot), MountProbe: mountHealthProbe}, nil +} + +func addNamespaceFlags(command *cobra.Command, codexHome *string, mountPoint *string, nativeRoot *string) { + command.Flags().StringVar(codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(mountPoint, "mount", "", "Canonical mount path; defaults to /fold-fs") + command.Flags().StringVar(nativeRoot, "native-root", "", "Retained native tree; defaults to /fold-native") +} + +func writeNamespaceResult(command *cobra.Command, result FSNamespaceResult, jsonOutput bool) error { + if jsonOutput { + return writeJSON(command, result) + } + _, err := fmt.Fprintf(command.OutOrStdout(), "active=%t recovered=%t dry_run=%t home=%s mount=%s native_root=%s\n", result.Active, result.Recovered, result.DryRun, result.Home, result.Mount, result.NativeRoot) + return err +} diff --git a/internal/cli/fs_reconcile.go b/internal/cli/fs_reconcile.go new file mode 100644 index 0000000..4adda24 --- /dev/null +++ b/internal/cli/fs_reconcile.go @@ -0,0 +1,110 @@ +package cli + +import ( + "errors" + "fmt" + "path/filepath" + + "github.com/jstar0/codexfold/internal/reconcile" + "github.com/spf13/cobra" +) + +type FSReconcileRolloutResult struct { + reconcile.Result + DryRun bool `json:"dry_run"` +} + +func newFSRepairRolloutCommand() *cobra.Command { + var outputPath, orphanPath string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "repair-rollout ", + Short: "Recover deterministically interleaved JSONL writes into a separate rollout", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + if !apply { + return errors.New("repair-rollout requires --apply and writes only to a separate --output") + } + if !filepath.IsAbs(args[0]) || !filepath.IsAbs(outputPath) { + return errors.New("source and --output paths must be absolute") + } + result, err := reconcile.RepairWithOptions(args[0], outputPath, reconcile.RepairOptions{AllowOrphans: orphanPath != "", OrphanPath: orphanPath}) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), + "physical=%d invalid=%d reconstructed=%d orphans=%d output=%d regressions=%d max_buffer=%d path=%s sha256=%s\n", + result.PhysicalLines, + result.InvalidPhysicalLines, + result.ReconstructedRecords, + result.OrphanLines, + result.OutputRecords, + result.TimestampRegressions, + result.MaximumBufferedBytes, + result.OutputPath, + result.OutputSHA256, + ) + return err + }, + } + command.Flags().StringVar(&outputPath, "output", "", "Absolute output path for the repaired rollout") + command.Flags().StringVar(&orphanPath, "orphans", "", "Optional absolute path for unrecoverable raw fragments; enables salvage mode") + command.Flags().BoolVar(&apply, "apply", false, "Write a separately verified repaired rollout") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSReconcileRolloutCommand() *cobra.Command { + var outputPath string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "reconcile-rollout ", + Short: "Reconcile two monotonic rollout branches without replacing either source", + Args: cobra.ExactArgs(2), + RunE: func(command *cobra.Command, args []string) error { + if !filepath.IsAbs(args[0]) || !filepath.IsAbs(args[1]) { + return errors.New("base and branch paths must be absolute") + } + var result reconcile.Result + var err error + if apply { + if !filepath.IsAbs(outputPath) { + return errors.New("--output must be absolute with --apply") + } + result, err = reconcile.Merge(args[0], args[1], outputPath) + } else { + result, err = reconcile.Analyze(args[0], args[1]) + } + if err != nil { + return err + } + wrapped := FSReconcileRolloutResult{Result: result, DryRun: !apply} + if jsonOutput { + return writeJSON(command, wrapped) + } + _, err = fmt.Fprintf(command.OutOrStdout(), + "base=%d branch=%d shared=%d base_only=%d added=%d output=%d regressions=%d/%d/%d dry_run=%t path=%s sha256=%s\n", + result.Base.Records, + result.Branch.Records, + result.SharedRecords, + result.BaseOnlyRecords, + result.AddedFromBranch, + result.OutputRecords, + result.Base.TimestampRegressions, + result.Branch.TimestampRegressions, + result.OutputRegressions, + wrapped.DryRun, + result.OutputPath, + result.OutputSHA256, + ) + return err + }, + } + command.Flags().StringVar(&outputPath, "output", "", "Absolute output path for the reconciled rollout") + command.Flags().BoolVar(&apply, "apply", false, "Write a separately verified reconciled rollout") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} diff --git a/internal/cli/fs_service.go b/internal/cli/fs_service.go index 8d1a3a4..e6e8a61 100644 --- a/internal/cli/fs_service.go +++ b/internal/cli/fs_service.go @@ -2,14 +2,17 @@ package cli import ( "context" - "crypto/sha256" - "encoding/hex" + "encoding/json" "errors" "fmt" "io" "os" "path/filepath" "runtime" + "strings" + "sync" + "syscall" + "time" "github.com/jstar0/codexfold/internal/codex" "github.com/jstar0/codexfold/internal/mountfs" @@ -20,6 +23,42 @@ import ( const serviceLabel = "com.codexfold.fs" +type operationTrace struct { + mu sync.Mutex + file *os.File +} + +func newOperationRecorder(path string) (func(string), io.Closer, error) { + if !filepath.IsAbs(path) { + return nil, nil, errors.New("operation trace path must be absolute") + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, nil, err + } + file, err := os.OpenFile(filepath.Clean(path), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return nil, nil, err + } + trace := &operationTrace{file: file} + return trace.record, trace, nil +} + +func (t *operationTrace) record(operation string) { + t.mu.Lock() + defer t.mu.Unlock() + _, _ = fmt.Fprintf(t.file, "%d %s\n", time.Now().UnixNano(), operation) +} + +func (t *operationTrace) Close() error { + t.mu.Lock() + defer t.mu.Unlock() + if err := t.file.Sync(); err != nil { + _ = t.file.Close() + return err + } + return t.file.Close() +} + type FSServiceActionResult struct { Action string `json:"action"` Path string `json:"path,omitempty"` @@ -44,8 +83,8 @@ func newFSServiceCommand() *cobra.Command { } func newFSServiceInstallCommand() *cobra.Command { - var codexHome, storeDir, mountPoint, binaryPath, plistPath, logDir string - var apply, jsonOutput bool + var codexHome, storeDir, mountPoint, binaryPath, plistPath, logDir, nativeRoot, operationTracePath string + var apply, canonicalNamespace, jsonOutput bool command := &cobra.Command{ Use: "install", Short: "Render and optionally bootstrap a per-user launchd service", @@ -55,7 +94,20 @@ func newFSServiceInstallCommand() *cobra.Command { if err != nil { return err } - definition, err := service.RenderLaunchd(service.Options{Label: serviceLabel, BinaryPath: binary, CodexHome: home, StoreDir: store, MountPoint: mount, StdoutPath: filepath.Join(logs, "stdout.log"), StderrPath: filepath.Join(logs, "stderr.log")}) + if canonicalNamespace { + if nativeRoot == "" { + nativeRoot = filepath.Join(home, "fold-native") + } + if !filepath.IsAbs(nativeRoot) { + return errors.New("canonical service native root must be absolute") + } + nativeRoot = filepath.Clean(nativeRoot) + } + definition, err := service.RenderLaunchd(service.Options{ + Label: serviceLabel, BinaryPath: binary, CodexHome: home, StoreDir: store, MountPoint: mount, + StdoutPath: filepath.Join(logs, "stdout.log"), StderrPath: filepath.Join(logs, "stderr.log"), + CanonicalNamespace: canonicalNamespace, NativeRoot: nativeRoot, OperationTrace: operationTracePath, + }) if err != nil { return err } @@ -80,6 +132,10 @@ func newFSServiceInstallCommand() *cobra.Command { if err := manager.Kickstart(command.Context(), serviceLabel); err != nil { return err } + if _, err := manager.WaitHealthy(command.Context(), serviceLabel, mount, 15*time.Second); err != nil { + _ = manager.Bootout(command.Context(), plist) + return err + } } if jsonOutput { return writeJSON(command, result) @@ -89,13 +145,16 @@ func newFSServiceInstallCommand() *cobra.Command { }, } addServicePathFlags(command, &codexHome, &storeDir, &mountPoint, &binaryPath, &plistPath, &logDir) + command.Flags().BoolVar(&canonicalNamespace, "canonical-namespace", false, "Start the service with the canonical Codex session namespace") + command.Flags().StringVar(&nativeRoot, "native-root", "", "Canonical native backing root; defaults to /fold-native") + command.Flags().StringVar(&operationTracePath, "operation-trace", "", "Absolute path for sanitized FUSE operation names") command.Flags().BoolVar(&apply, "apply", false, "Write, bootstrap, and start the per-user service") command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") return command } func newFSServiceStartCommand() *cobra.Command { - return newFSServiceLifecycleCommand("start", func(ctx context.Context, manager service.Manager, plist string) error { + return newFSServiceLifecycleCommand("start", true, func(ctx context.Context, manager service.Manager, plist string) error { _ = manager.Bootout(ctx, plist) if err := manager.Bootstrap(ctx, plist); err != nil { return err @@ -105,13 +164,13 @@ func newFSServiceStartCommand() *cobra.Command { } func newFSServiceStopCommand() *cobra.Command { - return newFSServiceLifecycleCommand("stop", func(ctx context.Context, manager service.Manager, plist string) error { + return newFSServiceLifecycleCommand("stop", false, func(ctx context.Context, manager service.Manager, plist string) error { return manager.Bootout(ctx, plist) }) } -func newFSServiceLifecycleCommand(action string, run func(context.Context, service.Manager, string) error) *cobra.Command { - var plistPath string +func newFSServiceLifecycleCommand(action string, waitForMount bool, run func(context.Context, service.Manager, string) error) *cobra.Command { + var plistPath, codexHome, mountPoint string var apply, jsonOutput bool command := &cobra.Command{ Use: action, @@ -127,9 +186,20 @@ func newFSServiceLifecycleCommand(action string, run func(context.Context, servi if runtime.GOOS != "darwin" { return errors.New("launchd service lifecycle is available only on macOS") } - if err := run(command.Context(), service.Manager{}, plist); err != nil { + manager := service.Manager{} + if err := run(command.Context(), manager, plist); err != nil { return err } + if waitForMount { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + if _, err := manager.WaitHealthy(command.Context(), serviceLabel, defaultMountPoint(home, mountPoint), 15*time.Second); err != nil { + _ = manager.Bootout(command.Context(), plist) + return err + } + } } if jsonOutput { return writeJSON(command, result) @@ -139,6 +209,10 @@ func newFSServiceLifecycleCommand(action string, run func(context.Context, servi }, } command.Flags().StringVar(&plistPath, "plist", "", "LaunchAgent plist path") + if waitForMount { + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + } command.Flags().BoolVar(&apply, "apply", false, "Execute the launchctl action") command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") return command @@ -238,25 +312,12 @@ func managedRoutesMatchCurrentBytes(ctx context.Context, home string, store stri if !ok { return false, fmt.Errorf("Codex route missing for managed session %s", state.SessionID) } - if isGeneratedNativeFallbackPath(current.RolloutPath, store, state.SessionID) { - if _, err := hashPath(current.RolloutPath); err != nil { - return false, err - } - continue + if !isGeneratedNativeFallbackPath(current.RolloutPath, store, state.SessionID) { + return false, nil } - managed, resolver, err := openManagedSession(ctx, store, state) - if err != nil { + if _, err := hashPath(current.RolloutPath); err != nil { return false, err } - visible, err := hashManagedSession(ctx, managed) - _ = resolver.Close() - if err != nil { - return false, err - } - route, err := hashPath(current.RolloutPath) - if err != nil || route.Bytes != visible.Bytes || route.SHA256 != visible.SHA256 { - return false, nil - } } return true, nil } @@ -290,7 +351,14 @@ func quarantineManagedRoutes(ctx context.Context, home string, store string) (in if err != nil { return count, err } - targetPath := filepath.Join(store, "fs", "sessions", state.SessionID, "quarantine-current.jsonl") + targetDirectory := filepath.Join(store, "fs", "fallbacks", state.SessionID) + if err := os.MkdirAll(targetDirectory, 0o700); err != nil { + return count, err + } + if err := os.Chmod(targetDirectory, 0o700); err != nil { + return count, err + } + targetPath := filepath.Join(targetDirectory, "quarantine-current.jsonl") target, err := managed.MaterializeCurrent(ctx, targetPath, true) _ = resolver.Close() if err != nil { @@ -302,6 +370,9 @@ func quarantineManagedRoutes(ctx context.Context, home string, store string) (in if _, err := codex.RouteSession(ctx, codex.RouteOptions{CodexHome: home, SessionID: state.SessionID, ExpectedPath: current.RolloutPath, Target: codex.RouteTarget{Path: target.Path, Bytes: target.Bytes, SHA256: target.SHA256}}); err != nil { return count, err } + if _, err := retireManagedState(store, state.SessionID); err != nil { + return count, err + } count++ } return count, nil @@ -311,7 +382,10 @@ func isGeneratedNativeFallbackPath(path string, store string, sessionID string) if path == "" || store == "" || sessionID == "" { return false } - if filepath.Clean(filepath.Dir(path)) != filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) { + directory := filepath.Clean(filepath.Dir(path)) + legacyDirectory := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) + fallbackDirectory := filepath.Join(filepath.Clean(store), "fs", "fallbacks", sessionID) + if directory != legacyDirectory && directory != fallbackDirectory { return false } switch filepath.Base(path) { @@ -322,36 +396,365 @@ func isGeneratedNativeFallbackPath(path string, store string, sessionID string) } } -func hashManagedSession(ctx context.Context, session *vfs.Session) (vfs.NativeFile, error) { - reader, err := session.OpenReader() +func retireManagedState(store string, sessionID string) (string, error) { + if store == "" || sessionID == "" { + return "", errors.New("store and session ID are required") + } + source := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) + retiredRoot := filepath.Join(filepath.Clean(store), "fs", "retired") + if err := os.MkdirAll(retiredRoot, 0o700); err != nil { + return "", err + } + target := filepath.Join(retiredRoot, fmt.Sprintf("%s-%d", sessionID, time.Now().UnixNano())) + if err := os.Rename(source, target); err != nil { + return "", err + } + return target, nil +} + +func restoreManagedState(store string, sessionID string, retiredPath string) error { + if store == "" || sessionID == "" || retiredPath == "" { + return errors.New("store, session ID, and retired state path are required") + } + target := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) + return os.Rename(filepath.Clean(retiredPath), target) +} + +func retainCanonicalSnapshot(store string, sessionID string, source vfs.NativeFile) (vfs.NativeFile, error) { + if store == "" || !validSessionID(sessionID) || source.Path == "" { + return vfs.NativeFile{}, errors.New("store, session ID, and source snapshot are required") + } + sourcePath := filepath.Clean(source.Path) + verified, err := hashPath(sourcePath) + if err != nil { + return vfs.NativeFile{}, fmt.Errorf("verify canonical native snapshot: %w", err) + } + if verified.Bytes != source.Bytes || verified.SHA256 != source.SHA256 { + return vfs.NativeFile{}, errors.New("canonical native snapshot changed during migration") + } + retainedDir := filepath.Join(filepath.Clean(store), "fs", "snapshots", sessionID) + retainedPath := filepath.Join(retainedDir, "native.jsonl") + if err := os.MkdirAll(retainedDir, 0o700); err != nil { + return vfs.NativeFile{}, err + } + if _, err := os.Lstat(retainedPath); err == nil { + return vfs.NativeFile{}, errors.New("retained canonical snapshot already exists") + } else if !errors.Is(err, os.ErrNotExist) { + return vfs.NativeFile{}, err + } + if err := os.Link(sourcePath, retainedPath); err != nil { + if !errors.Is(err, syscall.EXDEV) { + return vfs.NativeFile{}, fmt.Errorf("stage canonical native snapshot: %w", err) + } + if err := copyCanonicalSnapshot(sourcePath, retainedPath); err != nil { + return vfs.NativeFile{}, fmt.Errorf("copy canonical native snapshot: %w", err) + } + } + retained, err := hashPath(retainedPath) + if err == nil && (retained.Bytes != source.Bytes || retained.SHA256 != source.SHA256) { + err = errors.New("retained canonical snapshot does not match source") + } if err != nil { + _ = os.Remove(retainedPath) return vfs.NativeFile{}, err } - defer reader.Close() - hasher := sha256.New() - buffer := make([]byte, 1<<20) - var offset int64 - for offset < reader.Size() { - need := len(buffer) - if remaining := reader.Size() - offset; int64(need) > remaining { - need = int(remaining) + retained.Path = retainedPath + return retained, nil +} + +func finalizeCanonicalSnapshotSource(sourcePath string, retained vfs.NativeFile) error { + sourcePath = filepath.Clean(sourcePath) + retained.Path = filepath.Clean(retained.Path) + source, err := hashPath(sourcePath) + if err != nil { + return fmt.Errorf("verify canonical source before cutover: %w", err) + } + hidden, err := hashPath(retained.Path) + if err != nil { + return fmt.Errorf("verify retained canonical snapshot before cutover: %w", err) + } + if source.Bytes != retained.Bytes || source.SHA256 != retained.SHA256 || hidden.Bytes != retained.Bytes || hidden.SHA256 != retained.SHA256 { + return errors.New("canonical source changed before cutover") + } + if err := os.Remove(sourcePath); err != nil { + return fmt.Errorf("hide canonical source after mount acknowledgement: %w", err) + } + return nil +} + +func copyCanonicalSnapshot(sourcePath string, retainedPath string) error { + source, err := os.Open(sourcePath) + if err != nil { + return err + } + defer source.Close() + target, err := os.OpenFile(retainedPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + if _, err := io.Copy(target, source); err != nil { + _ = target.Close() + _ = os.Remove(retainedPath) + return err + } + if err := target.Sync(); err != nil { + _ = target.Close() + _ = os.Remove(retainedPath) + return err + } + return target.Close() +} + +func restoreCanonicalSnapshotSource(originalPath string, retainedPath string) error { + originalPath = filepath.Clean(originalPath) + retainedPath = filepath.Clean(retainedPath) + if originalPath == "" || retainedPath == "" { + return errors.New("original and retained snapshot paths are required") + } + if _, err := os.Lstat(originalPath); err == nil { + original, originalErr := hashPath(originalPath) + retained, retainedErr := hashPath(retainedPath) + if originalErr != nil || retainedErr != nil || original.Bytes != retained.Bytes || original.SHA256 != retained.SHA256 { + return errors.New("cannot discard retained snapshot while canonical source differs") + } + if err := os.Remove(retainedPath); err != nil { + return err + } + _ = os.Remove(filepath.Dir(retainedPath)) + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if err := os.MkdirAll(filepath.Dir(originalPath), 0o700); err != nil { + return err + } + if err := os.Rename(retainedPath, originalPath); err != nil { + return fmt.Errorf("restore canonical native snapshot: %w", err) + } + return nil +} + +type mountAcknowledgement struct { + Generation uint64 `json:"generation"` + Route string `json:"route"` +} + +func writeMountAcknowledgement(store string, sessionID string, generation uint64, route string) error { + if store == "" || !validSessionID(sessionID) || generation == 0 || route == "" { + return errors.New("complete mount acknowledgement metadata is required") + } + directory := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) + data, err := json.Marshal(mountAcknowledgement{Generation: generation, Route: route}) + if err != nil { + return err + } + temporary, err := os.CreateTemp(directory, ".mounted-*.tmp") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return err + } + if _, err := temporary.Write(append(data, '\n')); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + return os.Rename(temporaryPath, filepath.Join(directory, "mounted.json")) +} + +func waitForMountAcknowledgement(ctx context.Context, store string, sessionID string, generation uint64, route string, timeout time.Duration) error { + if timeout <= 0 { + timeout = 15 * time.Second + } + path := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID, "mounted.json") + deadline := time.Now().Add(timeout) + for { + data, err := os.ReadFile(path) + if err == nil { + var acknowledgement mountAcknowledgement + if json.Unmarshal(data, &acknowledgement) == nil && acknowledgement.Generation == generation && acknowledgement.Route == route { + return nil + } + } else if !errors.Is(err, os.ErrNotExist) { + return err } - n, readErr := reader.ReadAt(ctx, buffer[:need], offset) - if n > 0 { - _, _ = hasher.Write(buffer[:n]) - offset += int64(n) + if time.Now().After(deadline) { + return errors.New("timed out waiting for the filesystem daemon") } - if readErr != nil && !errors.Is(readErr, io.EOF) { - return vfs.NativeFile{}, readErr + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(25 * time.Millisecond): } - if n == 0 { - break + } +} + +func retireCanonicalNativeSnapshot(store string, nativeRoot string, sessionID string, snapshotPath string, currentPath string, retiredState string) (string, error) { + snapshotPath = filepath.Clean(snapshotPath) + if snapshotPath == filepath.Clean(currentPath) { + return "", nil + } + var relative string + legacyRelative, legacyErr := relativeWithin(filepath.Clean(nativeRoot), snapshotPath) + hiddenRoot := filepath.Join(filepath.Clean(store), "fs", "snapshots", sessionID) + _, hiddenErr := relativeWithin(hiddenRoot, snapshotPath) + switch { + case legacyErr == nil: + relative = legacyRelative + case hiddenErr == nil && filepath.Base(snapshotPath) == "native.jsonl": + relative = filepath.Join("store-snapshot", "native.jsonl") + default: + return "", errors.New("canonical native snapshot is outside the retained snapshot roots") + } + target := filepath.Join(filepath.Clean(retiredState), "retained-native", relative) + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + return "", err + } + if err := os.Rename(snapshotPath, target); err != nil { + return "", err + } + oldSidecar := filepath.Join(filepath.Dir(snapshotPath), "._"+filepath.Base(snapshotPath)) + newSidecar := filepath.Join(filepath.Dir(target), "._"+filepath.Base(target)) + if _, err := os.Lstat(oldSidecar); err == nil { + if err := os.Rename(oldSidecar, newSidecar); err != nil { + _ = os.Rename(target, snapshotPath) + return "", err } + } else if !errors.Is(err, os.ErrNotExist) { + _ = os.Rename(target, snapshotPath) + return "", err + } + if hiddenErr == nil { + _ = os.Remove(hiddenRoot) + } + return target, nil +} + +func validSessionID(sessionID string) bool { + return sessionID != "" && sessionID != "." && sessionID != ".." && !strings.ContainsAny(sessionID, "/\\\x00") +} + +func relativeWithin(root string, target string) (string, error) { + root = filepath.Clean(root) + target = filepath.Clean(target) + relative, err := filepath.Rel(root, target) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", errors.New("path is outside root") + } + return relative, nil +} + +func restoreCanonicalNativeSnapshot(snapshotPath string, retiredSnapshot string) error { + if retiredSnapshot == "" { + return nil + } + if err := os.MkdirAll(filepath.Dir(snapshotPath), 0o700); err != nil { + return err + } + if err := os.Rename(retiredSnapshot, snapshotPath); err != nil { + return err + } + retiredSidecar := filepath.Join(filepath.Dir(retiredSnapshot), "._"+filepath.Base(retiredSnapshot)) + originalSidecar := filepath.Join(filepath.Dir(snapshotPath), "._"+filepath.Base(snapshotPath)) + if _, err := os.Lstat(retiredSidecar); err == nil { + return os.Rename(retiredSidecar, originalSidecar) + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + return nil +} + +func canonicalMountRoute(home string, mount string, route string) (string, error) { + relative, err := canonicalRelativeRoute(home, route) + if err != nil { + return "", err + } + return filepath.Join(filepath.Clean(mount), relative), nil +} + +func canonicalNativeRoute(home string, nativeRoot string, route string) (string, error) { + relative, err := canonicalRelativeRoute(home, route) + if err != nil { + return "", err + } + if !filepath.IsAbs(nativeRoot) { + return "", errors.New("canonical native root must be absolute") + } + return filepath.Join(filepath.Clean(nativeRoot), relative), nil +} + +func canonicalNamespaceRoute(home string, mount string, route string) (string, error) { + relative, homeErr := canonicalRelativeRoute(home, route) + if homeErr != nil { + var mountErr error + relative, mountErr = canonicalRelativeRoute(mount, route) + if mountErr != nil { + return "", homeErr + } + } + return "/" + filepath.ToSlash(relative), nil +} + +func canonicalSessionRoutes(home string, mount string, store string, states []vfs.SessionState, sessions []codex.Session) (map[string]string, error) { + managed := make(map[string]struct{}, len(states)) + for _, state := range states { + managed[state.SessionID] = struct{}{} + } + routes := make(map[string]string, len(states)) + for _, session := range sessions { + if _, exists := managed[session.ID]; !exists { + continue + } + if isGeneratedNativeFallbackPath(session.RolloutPath, store, session.ID) { + continue + } + route, err := canonicalNamespaceRoute(home, mount, session.RolloutPath) + if err != nil { + return nil, err + } + routes[session.ID] = route + } + return routes, nil +} + +func discoverCanonicalRoutes(home string, mount string, store string, states []vfs.SessionState, load func(string) ([]codex.Session, error)) (map[string]string, error) { + if len(states) == 0 { + return map[string]string{}, nil + } + sessions, err := load(home) + if err != nil { + return nil, err + } + return canonicalSessionRoutes(home, mount, store, states, sessions) +} + +func canonicalRelativeRoute(home string, route string) (string, error) { + if !filepath.IsAbs(home) || !filepath.IsAbs(route) { + return "", errors.New("canonical Codex and route paths must be absolute") + } + home = filepath.Clean(home) + relative, err := filepath.Rel(home, filepath.Clean(route)) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", errors.New("Codex route is outside its home directory") + } + first, remainder := relative, "" + if separator := strings.IndexByte(relative, byte(filepath.Separator)); separator >= 0 { + first, remainder = relative[:separator], relative[separator+1:] } - if offset != reader.Size() { - return vfs.NativeFile{}, errors.New("managed session ended before its declared size") + if (first != "sessions" && first != "archived_sessions") || remainder == "" || !strings.HasSuffix(remainder, ".jsonl") { + return "", errors.New("Codex route is not inside sessions or archived_sessions") } - return vfs.NativeFile{Bytes: offset, SHA256: hex.EncodeToString(hasher.Sum(nil))}, nil + return relative, nil } func addServicePathFlags(command *cobra.Command, codexHome, storeDir, mountPoint, binaryPath, plistPath, logDir *string) { diff --git a/internal/cli/fs_test.go b/internal/cli/fs_test.go index f3fa2dc..d583f69 100644 --- a/internal/cli/fs_test.go +++ b/internal/cli/fs_test.go @@ -5,10 +5,12 @@ import ( "context" "database/sql" "encoding/json" + "errors" "os" "path/filepath" "runtime" "testing" + "time" "github.com/jstar0/codexfold/internal/codex" "github.com/jstar0/codexfold/internal/compat" @@ -23,8 +25,10 @@ func TestRootExposesPackAndFilesystemCommands(t *testing.T) { root := NewRootCommand() for _, commandPath := range [][]string{ {"pack", "build"}, {"pack", "doctor"}, - {"fs", "status"}, {"fs", "doctor"}, {"fs", "compatibility"}, {"fs", "benchmark"}, + {"fs", "status"}, {"fs", "doctor"}, {"fs", "compatibility"}, {"fs", "compatibility-import"}, {"fs", "benchmark"}, {"fs", "serve"}, {"fs", "migrate"}, {"fs", "rollback"}, {"fs", "compact"}, {"fs", "recover"}, + {"fs", "namespace", "status"}, {"fs", "namespace", "activate"}, + {"fs", "namespace", "deactivate"}, {"fs", "namespace", "recover"}, {"fs", "service", "install"}, {"fs", "service", "start"}, {"fs", "service", "stop"}, {"fs", "service", "status"}, {"fs", "service", "update-preflight"}, } { @@ -34,6 +38,268 @@ func TestRootExposesPackAndFilesystemCommands(t *testing.T) { } } +func TestFSCompatibilityImportPersistsOnlySanitizedContract(t *testing.T) { + home := t.TempDir() + store := filepath.Join(home, "fold-store") + trace := filepath.Join(home, "private-trace.log") + traceText := "12:00:00 open /Users/private/.codex/secret.jsonl codex.1\n12:00:01 read /Users/private/.codex/secret.jsonl codex.1\n" + if err := os.WriteFile(trace, []byte(traceText), 0o600); err != nil { + t.Fatal(err) + } + executeFS(t, []string{ + "fs", "compatibility-import", "--apply", "--codex-home", home, "--store", store, + "--trace", trace, "--client-kind", "cli", "--client-version", "1.2.3", + }) + contracts, err := compat.LoadAll(filepath.Join(store, "compatibility")) + if err != nil || len(contracts) != 1 || contracts[0].ClientVersion != "1.2.3" { + t.Fatalf("contracts = %#v err=%v", contracts, err) + } + data, err := os.ReadFile(filepath.Join(store, "compatibility", runtime.GOOS, "cli", "1.2.3.json")) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(data, []byte("/Users/private")) || bytes.Contains(data, []byte("secret.jsonl")) { + t.Fatalf("sanitized contract leaked trace content: %s", data) + } +} + +func TestOperationRecorderWritesOnlyTimeAndOperation(t *testing.T) { + tracePath := filepath.Join(t.TempDir(), "operations.log") + record, closer, err := newOperationRecorder(tracePath) + if err != nil { + t.Fatal(err) + } + record("open") + record("read") + if err := closer.Close(); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(tracePath) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(data, []byte("/")) || !bytes.Contains(data, []byte(" open\n")) || !bytes.Contains(data, []byte(" read\n")) { + t.Fatalf("operation trace = %q", data) + } +} + +func TestFSNamespaceActivateAndDeactivateCommandsPreserveNativeFiles(t *testing.T) { + root := t.TempDir() + home := filepath.Join(root, "home") + mount := filepath.Join(root, "mount") + nativeRoot := filepath.Join(home, "fold-native") + configPath := filepath.Join(home, "config.toml") + authPath := filepath.Join(home, "auth.json") + configBefore := []byte("model_provider = \"third-party\"\n") + authBefore := []byte("{\"access_token\":\"test\"}\n") + if err := os.MkdirAll(home, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(configPath, configBefore, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(authPath, authBefore, 0o600); err != nil { + t.Fatal(err) + } + for _, path := range []string{ + filepath.Join(home, "sessions", "active.jsonl"), + filepath.Join(home, "archived_sessions", "archived.jsonl"), + } { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(filepath.Base(path)), 0o600); err != nil { + t.Fatal(err) + } + } + for _, directory := range []string{"sessions", "archived_sessions"} { + if err := os.MkdirAll(filepath.Join(mount, directory), 0o700); err != nil { + t.Fatal(err) + } + } + database, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := database.Exec(`create table threads (id text primary key, rollout_path text not null)`); err != nil { + _ = database.Close() + t.Fatal(err) + } + if err := database.Close(); err != nil { + t.Fatal(err) + } + previousProbe := mountHealthProbe + t.Cleanup(func() { mountHealthProbe = previousProbe }) + mountHealthProbe = func(string) error { return nil } + executeFS(t, []string{ + "fs", "namespace", "activate", "--apply", + "--codex-home", home, "--mount", mount, "--native-root", nativeRoot, + }) + for _, directory := range []string{"sessions", "archived_sessions"} { + if target, err := os.Readlink(filepath.Join(home, directory)); err != nil || filepath.Clean(target) != filepath.Join(mount, directory) { + t.Fatalf("namespace link %s = %q err=%v", directory, target, err) + } + } + mountHealthProbe = func(string) error { return errors.New("not mounted") } + executeFS(t, []string{ + "fs", "namespace", "deactivate", "--apply", + "--codex-home", home, "--mount", mount, "--native-root", nativeRoot, + }) + for _, path := range []string{ + filepath.Join(home, "sessions", "active.jsonl"), + filepath.Join(home, "archived_sessions", "archived.jsonl"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("restored file %s: %v", path, err) + } + } + if got, err := os.ReadFile(configPath); err != nil || !bytes.Equal(got, configBefore) { + t.Fatalf("config.toml changed during namespace lifecycle: %q err=%v", got, err) + } + if got, err := os.ReadFile(authPath); err != nil || !bytes.Equal(got, authBefore) { + t.Fatalf("auth.json changed during namespace lifecycle: %q err=%v", got, err) + } +} + +func TestFSNamespaceDeactivateRejectsManagedSessions(t *testing.T) { + root := t.TempDir() + home := filepath.Join(root, "home") + mount := filepath.Join(root, "mount") + nativeRoot := filepath.Join(home, "fold-native") + store := filepath.Join(home, "fold-store") + for _, directory := range []string{ + filepath.Join(home, "sessions"), filepath.Join(home, "archived_sessions"), + filepath.Join(mount, "sessions"), filepath.Join(mount, "archived_sessions"), + filepath.Join(nativeRoot, "sessions"), filepath.Join(nativeRoot, "archived_sessions"), + } { + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + } + stateDirectory := filepath.Join(store, "fs", "sessions", "managed") + if err := os.MkdirAll(stateDirectory, 0o700); err != nil { + t.Fatal(err) + } + state := vfs.SessionState{ + Version: 1, SessionID: "managed", Generation: 1, + ManifestPath: filepath.Join(store, "manifests", "managed.json"), + BaseSHA256: "0000000000000000000000000000000000000000000000000000000000000000", + DeltaPath: filepath.Join(stateDirectory, "delta.jsonl"), + } + data, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(stateDirectory, "state.json"), data, 0o600); err != nil { + t.Fatal(err) + } + previousProbe := mountHealthProbe + t.Cleanup(func() { mountHealthProbe = previousProbe }) + mountHealthProbe = func(string) error { return errors.New("not mounted") } + command := NewRootCommand() + command.SetOut(&bytes.Buffer{}) + command.SetErr(&bytes.Buffer{}) + command.SetArgs([]string{ + "fs", "namespace", "deactivate", "--apply", + "--codex-home", home, "--store", store, "--mount", mount, "--native-root", nativeRoot, + }) + err = command.Execute() + if err == nil || err.Error() != "rollback all managed sessions before deactivating the namespace" { + t.Fatalf("deactivate error = %v", err) + } +} + +func TestCanonicalMountRouteMirrorsCodexSessionNamespace(t *testing.T) { + home := filepath.Join(string(filepath.Separator), "tmp", "codex-home") + mount := filepath.Join(string(filepath.Separator), "tmp", "codex-fold") + route := filepath.Join(home, "archived_sessions", "rollout-session.jsonl") + got, err := canonicalMountRoute(home, mount, route) + if err != nil || got != filepath.Join(mount, "archived_sessions", "rollout-session.jsonl") { + t.Fatalf("canonicalMountRoute = %q err=%v", got, err) + } + if _, err := canonicalMountRoute(home, mount, filepath.Join(home, "other", "rollout.jsonl")); err == nil { + t.Fatal("non-canonical Codex route should be rejected") + } +} + +func TestCanonicalSessionRoutesIgnoreUnmanagedSessionsOutsideCodexHome(t *testing.T) { + home := filepath.Join(string(filepath.Separator), "tmp", "codex-home") + mount := filepath.Join(home, "fold-fs") + states := []vfs.SessionState{{SessionID: "managed"}} + sessions := []codex.Session{ + {ID: "managed", RolloutPath: filepath.Join(home, "sessions", "2026", "07", "12", "rollout-managed.jsonl")}, + {ID: "unmanaged", RolloutPath: filepath.Join(string(filepath.Separator), "tmp", "native-fallback.jsonl")}, + } + routes, err := canonicalSessionRoutes(home, mount, filepath.Join(home, "fold-store"), states, sessions) + if err != nil { + t.Fatal(err) + } + if len(routes) != 1 || routes["managed"] != "/sessions/2026/07/12/rollout-managed.jsonl" { + t.Fatalf("canonical routes = %#v", routes) + } +} + +func TestCanonicalSessionRoutesSkipManagedNativeFallback(t *testing.T) { + home := t.TempDir() + mount := filepath.Join(home, "fold-fs") + store := filepath.Join(home, "fold-store") + state := vfs.SessionState{SessionID: "session"} + fallback := filepath.Join(store, "fs", "sessions", "session", "quarantine-current.jsonl") + routes, err := canonicalSessionRoutes(home, mount, store, []vfs.SessionState{state}, []codex.Session{{ID: "session", RolloutPath: fallback}}) + if err != nil { + t.Fatal(err) + } + if len(routes) != 0 { + t.Fatalf("native fallback leaked into canonical routes: %#v", routes) + } +} + +func TestCanonicalSessionRoutesAcceptDesktopMountAlias(t *testing.T) { + home := filepath.Join(string(filepath.Separator), "tmp", "codex-home") + mount := filepath.Join(home, "fold-fs") + states := []vfs.SessionState{{SessionID: "managed"}} + sessions := []codex.Session{{ + ID: "managed", + RolloutPath: filepath.Join(mount, "sessions", "2026", "07", "13", "rollout-managed.jsonl"), + }} + routes, err := canonicalSessionRoutes(home, mount, filepath.Join(home, "fold-store"), states, sessions) + if err != nil { + t.Fatal(err) + } + if len(routes) != 1 || routes["managed"] != "/sessions/2026/07/13/rollout-managed.jsonl" { + t.Fatalf("canonical routes from mount alias = %#v", routes) + } +} + +func TestDiscoverCanonicalRoutesSkipsCodexDatabaseWhenNoSessionsAreManaged(t *testing.T) { + called := false + routes, err := discoverCanonicalRoutes("/tmp/codex-home", "/tmp/codex-home/fold-fs", "/tmp/store", nil, func(string) ([]codex.Session, error) { + called = true + return nil, errors.New("database should not be opened") + }) + if err != nil || called || len(routes) != 0 { + t.Fatalf("empty canonical routes = %#v called=%t err=%v", routes, called, err) + } +} + +func TestCanonicalFSServeRequiresAbsoluteNativeRoot(t *testing.T) { + home := t.TempDir() + for _, nativeRoot := range []string{"", "relative-native-root"} { + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "serve", + "--canonical-namespace", + "--native-root", nativeRoot, + "--codex-home", home, + }) + if err := root.Execute(); err == nil { + t.Fatalf("native root %q should be rejected", nativeRoot) + } + } +} + func TestFSServiceInstallIsDryRunByDefaultAndApplyRequiresFuseBuild(t *testing.T) { home, storeDir, _ := fsFixture(t, true) plistPath := filepath.Join(home, "LaunchAgents", "com.codexfold.fs.plist") @@ -48,19 +314,15 @@ func TestFSServiceInstallIsDryRunByDefaultAndApplyRequiresFuseBuild(t *testing.T if _, err := os.Stat(plistPath); !os.IsNotExist(err) { t.Fatalf("dry-run wrote plist: %v", err) } + if mountfs.Available() { + return + } root = NewRootCommand() root.SetOut(&bytes.Buffer{}) root.SetErr(&bytes.Buffer{}) root.SetArgs([]string{"fs", "service", "install", "--codex-home", home, "--store", storeDir, "--plist", plistPath, "--apply"}) err := root.Execute() - if mountfs.Available() { - if err != nil { - t.Fatalf("FUSE build should install the service definition: %v", err) - } - if _, statErr := os.Stat(plistPath); statErr != nil { - t.Fatalf("service definition was not written: %v", statErr) - } - } else if err == nil { + if err == nil { t.Fatal("default build should reject service installation without a FUSE host") } } @@ -125,6 +387,9 @@ func TestFSUpdatePreflightQuarantineRoutesLatestVisibleBytesNative(t *testing.T) if !bytes.Equal(quarantineBytes, want) { t.Fatalf("quarantine route is stale: got=%q want=%q", quarantineBytes, want) } + if _, err := managedState(storeDir, "session"); err == nil { + t.Fatal("quarantine left the session managed") + } } func TestPackBuildAndDoctorCommandsUseFoldStore(t *testing.T) { @@ -218,6 +483,35 @@ func TestFSMigrateApplyRejectsPlainDirectoryThatOnlyLooksLikeMount(t *testing.T) } } +func TestCompatibilityCanaryRequiresIsolatedCanonicalHomeAndSkippedClients(t *testing.T) { + defaultHome := filepath.Join(t.TempDir(), ".codex") + isolatedHome := filepath.Join(t.TempDir(), "isolated") + isolatedStore := filepath.Join(isolatedHome, "fold-store") + skipped := compatibilityFlags{cliPath: "none", desktopPath: "none"} + if err := validateCompatibilityCanary(isolatedHome, defaultHome, isolatedStore, true, skipped); err != nil { + t.Fatalf("isolated canonical canary was rejected: %v", err) + } + for _, test := range []struct { + name string + home string + store string + canonical bool + flags compatibilityFlags + }{ + {name: "real home", home: defaultHome, store: filepath.Join(defaultHome, "fold-store"), canonical: true, flags: skipped}, + {name: "external store", home: isolatedHome, store: filepath.Join(t.TempDir(), "store"), canonical: true, flags: skipped}, + {name: "flat mount", home: isolatedHome, store: isolatedStore, canonical: false, flags: skipped}, + {name: "live cli", home: isolatedHome, store: isolatedStore, canonical: true, flags: compatibilityFlags{cliPath: "codex", desktopPath: "none"}}, + {name: "live desktop", home: isolatedHome, store: isolatedStore, canonical: true, flags: compatibilityFlags{cliPath: "none", desktopPath: "/Applications/ChatGPT.app"}}, + } { + t.Run(test.name, func(t *testing.T) { + if err := validateCompatibilityCanary(test.home, defaultHome, test.store, test.canonical, test.flags); err == nil { + t.Fatal("unsafe compatibility canary configuration was accepted") + } + }) + } +} + func TestFSStatusDoesNotClaimTransparentReadiness(t *testing.T) { root := NewRootCommand() var output bytes.Buffer @@ -286,6 +580,90 @@ func TestFSMigrateApplyInitializesManagedStateAndRoutesVerifiedTarget(t *testing } } +func TestFSMigrateCanonicalKeepsCodexRouteAndHidesRetainedSnapshot(t *testing.T) { + allowFixtureMount(t) + home := t.TempDir() + storeDir := filepath.Join(home, "fold-store") + route := filepath.Join(home, "archived_sessions", "rollout-session.jsonl") + if err := os.MkdirAll(filepath.Dir(route), 0o700); err != nil { + t.Fatal(err) + } + source := []byte("{\"type\":\"session_meta\"}\n{\"canonical\":true}\n") + if err := os.WriteFile(route, source, 0o600); err != nil { + t.Fatal(err) + } + writeStateFixture(t, home, route) + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`update threads set archived = 1, id = 'session' where id = 'fixture'`); err != nil { + _ = db.Close() + t.Fatal(err) + } + _ = db.Close() + if _, err := fold.Fold(context.Background(), codex.Session{ID: "session", RolloutPath: route, Archived: true}, fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { + t.Fatal(err) + } + if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { + t.Fatal(err) + } + nativeRoot := filepath.Join(home, "fold-native") + nativePath := filepath.Join(nativeRoot, "archived_sessions", filepath.Base(route)) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(route, nativePath); err != nil { + t.Fatal(err) + } + mount := filepath.Join(home, "fold-fs") + target := filepath.Join(mount, "archived_sessions", filepath.Base(route)) + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target, source, 0o600); err != nil { + t.Fatal(err) + } + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + acknowledged := make(chan error, 1) + go func() { + statePath := filepath.Join(storeDir, "fs", "sessions", "session", "state.json") + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + state, err := vfs.LoadSessionState(statePath) + if err == nil { + acknowledged <- writeMountAcknowledgement(storeDir, "session", state.Generation, "/archived_sessions/"+filepath.Base(route)) + return + } + time.Sleep(10 * time.Millisecond) + } + acknowledged <- errors.New("managed state was not created") + }() + executeFS(t, []string{ + "fs", "migrate", "session", "--apply", "--canonical-namespace", + "--codex-home", home, "--store", storeDir, "--mount", mount, "--native-root", nativeRoot, + "--cli", cliPath, "--desktop-app", "none", + }) + if err := <-acknowledged; err != nil { + t.Fatal(err) + } + sessions, err := codex.LoadSessions(home) + if err != nil || len(sessions) != 1 || filepath.Clean(sessions[0].RolloutPath) != filepath.Clean(route) { + t.Fatalf("canonical migration changed Codex route: sessions=%#v err=%v", sessions, err) + } + states, err := vfs.DiscoverSessionStates(storeDir) + retainedPath := filepath.Join(storeDir, "fs", "snapshots", "session", "native.jsonl") + if err != nil || len(states) != 1 || filepath.Clean(states[0].NativeSnapshot.Path) != filepath.Clean(retainedPath) { + t.Fatalf("canonical native snapshot = %#v err=%v", states, err) + } + if _, err := os.Stat(nativePath); !os.IsNotExist(err) { + t.Fatalf("canonical source remained visible after migration: %v", err) + } + if got, err := os.ReadFile(retainedPath); err != nil || !bytes.Equal(got, source) { + t.Fatalf("hidden retained snapshot = %q err=%v", got, err) + } +} + func TestFSRollbackUsesLatestVisibleBytesAfterVirtualAppend(t *testing.T) { allowFixtureMount(t) home, storeDir, nativePath := fsFixture(t, true) @@ -334,6 +712,254 @@ func TestFSRollbackUsesLatestVisibleBytesAfterVirtualAppend(t *testing.T) { if !bytes.Equal(fallback, want) { t.Fatalf("rollback used stale bytes: got=%q want=%q", fallback, want) } + if _, err := managedState(storeDir, "session"); err == nil { + t.Fatal("rollback left the session managed") + } +} + +func TestFSRollbackCanonicalRetiresManagedStateAndKeepsRoute(t *testing.T) { + allowFixtureMount(t) + home, storeDir, originalPath := fsFixture(t, true) + original, err := os.ReadFile(originalPath) + if err != nil { + t.Fatal(err) + } + filename := "rollout-session.jsonl" + snapshotRoute := filepath.Join(home, "archived_sessions", filename) + route := filepath.Join(home, "sessions", "2026", "07", "12", filename) + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`update threads set rollout_path = ? where id = 'session'`, route); err != nil { + _ = db.Close() + t.Fatal(err) + } + _ = db.Close() + nativeRoot := filepath.Join(home, "fold-native") + nativePath := filepath.Join(nativeRoot, "archived_sessions", filename) + targetNativePath := filepath.Join(nativeRoot, "sessions", "2026", "07", "12", filename) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(originalPath, nativePath); err != nil { + t.Fatal(err) + } + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + native, err := hashPath(nativePath) + if err != nil { + t.Fatal(err) + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: native, + }) + if err != nil { + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + tail := []byte("{\"canonical_rollback\":true}\n") + if _, err := writer.Append(context.Background(), tail); err != nil { + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + t.Fatal(err) + } + _ = writer.Close() + _ = resolver.Close() + mount := filepath.Join(home, "fold-fs") + mountedTarget := filepath.Join(mount, "sessions", "2026", "07", "12", filename) + if err := os.MkdirAll(filepath.Dir(mountedTarget), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(mountedTarget, original, 0o600); err != nil { + t.Fatal(err) + } + stateDirectory := filepath.Join(storeDir, "fs", "sessions", "session") + copyDone := make(chan error, 1) + go func() { + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(stateDirectory); os.IsNotExist(err) { + data, readErr := os.ReadFile(targetNativePath) + if readErr == nil { + readErr = os.WriteFile(mountedTarget, data, 0o600) + } + copyDone <- readErr + return + } + time.Sleep(10 * time.Millisecond) + } + copyDone <- errors.New("managed state was not retired") + }() + executeFS(t, []string{ + "fs", "rollback", "session", "--apply", "--canonical-namespace", + "--codex-home", home, "--store", storeDir, "--mount", mount, "--native-root", nativeRoot, + }) + if err := <-copyDone; err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), original...), tail...) + if got, err := os.ReadFile(targetNativePath); err != nil || !bytes.Equal(got, want) { + t.Fatalf("canonical rollback bytes = %q err=%v", got, err) + } + if _, err := os.Stat(nativePath); !os.IsNotExist(err) { + t.Fatalf("retained snapshot remained visible at %s: %v", snapshotRoute, err) + } + sessions, err := codex.LoadSessions(home) + if err != nil || len(sessions) != 1 || filepath.Clean(sessions[0].RolloutPath) != filepath.Clean(route) { + t.Fatalf("canonical rollback changed route: sessions=%#v err=%v", sessions, err) + } + if _, err := os.Stat(stateDirectory); !os.IsNotExist(err) { + t.Fatalf("managed state remained after canonical rollback: %v", err) + } + retired, err := filepath.Glob(filepath.Join(storeDir, "fs", "retired", "session-*")) + if err != nil || len(retired) != 1 { + t.Fatalf("retired state = %#v err=%v", retired, err) + } + retained, err := filepath.Glob(filepath.Join(retired[0], "retained-native", "archived_sessions", filename)) + if err != nil || len(retained) != 1 { + t.Fatalf("retired native snapshot = %#v err=%v", retained, err) + } +} + +func TestFSRollbackCanonicalRetiresHiddenSnapshot(t *testing.T) { + allowFixtureMount(t) + home, storeDir, originalPath := fsFixture(t, true) + original, err := os.ReadFile(originalPath) + if err != nil { + t.Fatal(err) + } + filename := "rollout-session.jsonl" + route := filepath.Join(home, "sessions", "2026", "07", "12", filename) + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`update threads set rollout_path = ? where id = 'session'`, route); err != nil { + _ = db.Close() + t.Fatal(err) + } + _ = db.Close() + nativeRoot := filepath.Join(home, "fold-native") + nativePath := filepath.Join(nativeRoot, "archived_sessions", filename) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(originalPath, nativePath); err != nil { + t.Fatal(err) + } + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + native, err := hashPath(nativePath) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + hiddenPath := filepath.Join(storeDir, "fs", "snapshots", "session", "native.jsonl") + if err := os.MkdirAll(filepath.Dir(hiddenPath), 0o700); err != nil { + _ = resolver.Close() + t.Fatal(err) + } + if err := os.Rename(nativePath, hiddenPath); err != nil { + _ = resolver.Close() + t.Fatal(err) + } + native.Path = hiddenPath + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: native, + }) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + tail := []byte("{\"hidden_snapshot_rollback\":true}\n") + if _, err := writer.Append(context.Background(), tail); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + _ = writer.Close() + _ = resolver.Close() + mount := filepath.Join(home, "fold-fs") + mountedTarget := filepath.Join(mount, "sessions", "2026", "07", "12", filename) + targetNativePath := filepath.Join(nativeRoot, "sessions", "2026", "07", "12", filename) + if err := os.MkdirAll(filepath.Dir(mountedTarget), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(mountedTarget, original, 0o600); err != nil { + t.Fatal(err) + } + stateDirectory := filepath.Join(storeDir, "fs", "sessions", "session") + copyDone := make(chan error, 1) + go func() { + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(stateDirectory); os.IsNotExist(err) { + data, readErr := os.ReadFile(targetNativePath) + if readErr == nil { + readErr = os.WriteFile(mountedTarget, data, 0o600) + } + copyDone <- readErr + return + } + time.Sleep(10 * time.Millisecond) + } + copyDone <- errors.New("managed state was not retired") + }() + // The mounted target is only used as the FUSE visibility probe. The + // canonical rollback writes the latest bytes to the retained native route. + executeFS(t, []string{ + "fs", "rollback", "session", "--apply", "--canonical-namespace", + "--codex-home", home, "--store", storeDir, "--mount", mount, "--native-root", nativeRoot, + }) + if err := <-copyDone; err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), original...), tail...) + retired, err := filepath.Glob(filepath.Join(storeDir, "fs", "retired", "session-*", "retained-native", "store-snapshot", "native.jsonl")) + if err != nil || len(retired) != 1 { + t.Fatalf("hidden snapshot retirement = %#v err=%v", retired, err) + } + if got, err := os.ReadFile(retired[0]); err != nil || !bytes.Equal(got, original) { + t.Fatalf("retired hidden snapshot bytes = %q err=%v", got, err) + } + if got, err := os.ReadFile(targetNativePath); err != nil || !bytes.Equal(got, want) { + t.Fatalf("canonical rollback bytes = %q err=%v", got, err) + } + if _, err := os.Stat(hiddenPath); !os.IsNotExist(err) { + t.Fatalf("hidden snapshot remained after retirement: %v", err) + } + if _, err := os.Stat(nativePath); !os.IsNotExist(err) { + t.Fatalf("legacy native snapshot unexpectedly restored: %v", err) + } } func TestFSUpdatePreflightPreservesNewerNativeFallbackAfterRollback(t *testing.T) { diff --git a/internal/compat/compat_test.go b/internal/compat/compat_test.go index 43a0ba4..71c8433 100644 --- a/internal/compat/compat_test.go +++ b/internal/compat/compat_test.go @@ -46,6 +46,17 @@ func TestParseFSUsageProducesSanitizedOperationContract(t *testing.T) { } } +func TestParseFSUsageRecognizesSanitizedFuseAdapterOperations(t *testing.T) { + trace := "1 getattr\n2 readdir\n3 open\n4 read\n5 release\n6 rename\n7 fsync\n" + contract, err := ParseFSUsage(strings.NewReader(trace), ContractOptions{Platform: "darwin", ClientKind: "cli", ClientVersion: "1.2.3"}) + if err != nil { + t.Fatal(err) + } + if len(contract.Operations) != 7 { + t.Fatalf("adapter operations = %#v", contract.Operations) + } +} + func TestEvaluateQuarantinesUnknownClientVersion(t *testing.T) { contracts := []Contract{{Version: ContractVersion, Platform: "darwin", ClientKind: "cli", ClientVersion: "1.0.0", TraceSHA256: strings.Repeat("a", 64)}} approved := Evaluate([]ClientVersion{{Platform: "darwin", Kind: "cli", Version: "1.0.0"}}, contracts) diff --git a/internal/compat/fsusage.go b/internal/compat/fsusage.go index ea03d9f..4c0cec2 100644 --- a/internal/compat/fsusage.go +++ b/internal/compat/fsusage.go @@ -17,7 +17,7 @@ type ContractOptions struct { ClientVersion string } -var operationPattern = regexp.MustCompile(`(?i)\b(open|openat|close|read|pread|readv|write|pwrite|writev|fsync|fdatasync|stat|stat64|lstat|lstat64|fstat|fstat64|mmap|truncate|ftruncate|rename|renameat|unlink|unlinkat|flock|fcntl|clonefile|getattrlist)\b`) +var operationPattern = regexp.MustCompile(`(?i)\b(open|openat|close|release|read|pread|readv|write|pwrite|writev|flush|fsync|fdatasync|stat|stat64|lstat|lstat64|fstat|fstat64|statfs|getattr|readdir|access|chmod|chown|utimens|mmap|truncate|ftruncate|create|mknod|mkdir|rmdir|link|symlink|readlink|rename|renameat|unlink|unlinkat|flock|fcntl|clonefile|getattrlist|setxattr|getxattr|listxattr|removexattr)\b`) var signaturePattern = regexp.MustCompile(`\([A-Z_]{4,32}\)|<[A-Z0-9_=+-]+>`) func ParseFSUsage(reader io.Reader, options ContractOptions) (Contract, error) { diff --git a/internal/mountfs/filesystem.go b/internal/mountfs/filesystem.go index 7aa3e8f..5793feb 100644 --- a/internal/mountfs/filesystem.go +++ b/internal/mountfs/filesystem.go @@ -2,10 +2,13 @@ package mountfs import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "io" "os" "path" + "path/filepath" "sort" "strings" "sync" @@ -24,24 +27,54 @@ type Attr struct { type fileHandle struct { mu sync.Mutex session *vfs.Session + native *os.File read *vfs.ReadHandle write *vfs.WriteHandle append bool } type Filesystem struct { - mu sync.RWMutex - loadMu sync.Mutex - sessions map[string]*vfs.Session - handles map[uint64]*fileHandle - next uint64 - loader func(string) (*vfs.Session, error) + mu sync.RWMutex + loadMu sync.Mutex + sessions map[string]*vfs.Session + paths map[string]string + retained map[string]string + directories map[string]struct{} + handles map[uint64]*fileHandle + next uint64 + loader func(string) (*vfs.Session, error) + canonical bool + nativeRoot string } func New() *Filesystem { return &Filesystem{sessions: make(map[string]*vfs.Session), handles: make(map[uint64]*fileHandle), next: 1} } +func NewCanonical() *Filesystem { + return &Filesystem{ + sessions: make(map[string]*vfs.Session), paths: make(map[string]string), + retained: make(map[string]string), + directories: map[string]struct{}{`/`: {}, `/sessions`: {}, `/archived_sessions`: {}}, + handles: make(map[uint64]*fileHandle), next: 1, canonical: true, + } +} + +func (f *Filesystem) SetNativeRoot(root string) { + if root != "" { + root = filepath.Clean(root) + } + f.mu.Lock() + f.nativeRoot = root + for retained := range f.retained { + delete(f.retained, retained) + } + for sessionID, session := range f.sessions { + f.registerRetainedPathLocked(sessionID, session) + } + f.mu.Unlock() +} + func (f *Filesystem) AddSession(sessionID string, session *vfs.Session) error { if sessionID == "" || strings.ContainsAny(sessionID, "/\\\x00") || session == nil { return errors.New("safe session ID and session are required") @@ -66,6 +99,104 @@ func (f *Filesystem) UpsertSession(sessionID string, session *vfs.Session) error return nil } +func (f *Filesystem) AddSessionAt(sessionID string, name string, session *vfs.Session) error { + cleaned := cleanPath(name) + if !f.canonical || !safeSessionID(sessionID) || session == nil || !canonicalSessionPath(cleaned) { + return errors.New("canonical filesystem, safe session ID, path, and session are required") + } + f.mu.Lock() + defer f.mu.Unlock() + if _, exists := f.sessions[sessionID]; exists { + return errors.New("session is already mounted") + } + if _, exists := f.paths[cleaned]; exists { + return errors.New("session path is already mounted") + } + f.ensureDirectoryChainLocked(path.Dir(cleaned)) + f.sessions[sessionID] = session + f.paths[cleaned] = sessionID + f.registerRetainedPathLocked(sessionID, session) + return nil +} + +func (f *Filesystem) UpsertSessionAt(sessionID string, name string, session *vfs.Session) error { + cleaned := cleanPath(name) + if !f.canonical || !safeSessionID(sessionID) || session == nil || !canonicalSessionPath(cleaned) { + return errors.New("canonical filesystem, safe session ID, path, and session are required") + } + f.mu.Lock() + defer f.mu.Unlock() + f.ensureDirectoryChainLocked(path.Dir(cleaned)) + var previousPath string + for route, currentID := range f.paths { + if currentID == sessionID { + previousPath = route + delete(f.paths, route) + } + } + if err := moveManagedMetadata(f.nativeRoot, previousPath, cleaned); err != nil { + if previousPath != "" { + f.paths[previousPath] = sessionID + } + return err + } + f.sessions[sessionID] = session + f.paths[cleaned] = sessionID + f.registerRetainedPathLocked(sessionID, session) + return nil +} + +func (f *Filesystem) MoveSessionAt(sessionID string, name string) error { + cleaned := cleanPath(name) + if !f.canonical || !safeSessionID(sessionID) || !canonicalSessionPath(cleaned) { + return errors.New("canonical filesystem, safe session ID, and path are required") + } + f.mu.Lock() + defer f.mu.Unlock() + if _, exists := f.sessions[sessionID]; !exists { + return os.ErrNotExist + } + f.ensureDirectoryChainLocked(path.Dir(cleaned)) + var previousPath string + for route, currentID := range f.paths { + if currentID == sessionID { + previousPath = route + delete(f.paths, route) + } + } + if err := moveManagedMetadata(f.nativeRoot, previousPath, cleaned); err != nil { + if previousPath != "" { + f.paths[previousPath] = sessionID + } + return err + } + f.paths[cleaned] = sessionID + return nil +} + +func (f *Filesystem) RemoveSession(sessionID string) error { + if !safeSessionID(sessionID) { + return errors.New("safe session ID is required") + } + f.mu.Lock() + defer f.mu.Unlock() + if _, exists := f.sessions[sessionID]; !exists { + return os.ErrNotExist + } + delete(f.sessions, sessionID) + for route, currentID := range f.paths { + if currentID == sessionID { + delete(f.paths, route) + } + } + for retained, currentID := range f.retained { + if currentID == sessionID { + delete(f.retained, retained) + } + } + return nil +} + func (f *Filesystem) SetSessionLoader(loader func(string) (*vfs.Session, error)) { f.mu.Lock() f.loader = loader @@ -73,10 +204,66 @@ func (f *Filesystem) SetSessionLoader(loader func(string) (*vfs.Session, error)) } func (f *Filesystem) ReadDir(name string) ([]string, syscall.Errno) { - if cleanPath(name) != "/" { + cleaned := cleanPath(name) + if !f.canonical && cleaned != "/" { return nil, syscall.ENOTDIR } f.mu.RLock() + if f.canonical { + if !canonicalNamespacePath(cleaned) { + f.mu.RUnlock() + return nil, syscall.ENOTDIR + } + _, virtualDirectory := f.directories[cleaned] + nativeRoot := f.nativeRoot + if !virtualDirectory && nativeRoot == "" { + f.mu.RUnlock() + return nil, syscall.ENOTDIR + } + if !virtualDirectory && nativeRoot != "" { + info, err := os.Stat(nativePathFromRoot(nativeRoot, cleaned)) + if err != nil || !info.IsDir() { + f.mu.RUnlock() + return nil, syscall.ENOTDIR + } + } + entrySet := make(map[string]struct{}) + hiddenEntries := make(map[string]struct{}) + for retained := range f.retained { + if path.Dir(retained) == cleaned { + hiddenEntries[path.Base(retained)] = struct{}{} + } + } + for directory := range f.directories { + if directory != cleaned && path.Dir(directory) == cleaned { + entrySet[path.Base(directory)] = struct{}{} + } + } + for route := range f.paths { + if path.Dir(route) == cleaned { + entrySet[path.Base(route)] = struct{}{} + } + } + f.mu.RUnlock() + if nativeRoot != "" && cleaned != "/" { + if nativeEntries, err := os.ReadDir(nativePathFromRoot(nativeRoot, cleaned)); err == nil { + for _, entry := range nativeEntries { + if _, hidden := hiddenEntries[entry.Name()]; hidden { + continue + } + entrySet[entry.Name()] = struct{}{} + } + } else if !errors.Is(err, os.ErrNotExist) { + return nil, errnoFor(err) + } + } + entries := make([]string, 0, len(entrySet)) + for entry := range entrySet { + entries = append(entries, entry) + } + sort.Strings(entries) + return entries, 0 + } entries := make([]string, 0, len(f.sessions)) for sessionID := range f.sessions { entries = append(entries, sessionID+".jsonl") @@ -87,13 +274,41 @@ func (f *Filesystem) ReadDir(name string) ([]string, syscall.Errno) { } func (f *Filesystem) Getattr(name string) (Attr, syscall.Errno) { - if cleanPath(name) == "/" { + cleaned := cleanPath(name) + if cleaned == "/" { return Attr{Mode: syscall.S_IFDIR | 0o700}, 0 } - session, errno := f.sessionForPath(name) + if f.canonical { + f.mu.RLock() + _, directory := f.directories[cleaned] + f.mu.RUnlock() + if directory { + return Attr{Mode: syscall.S_IFDIR | 0o700}, 0 + } + if session, errno := f.sessionForPath(cleaned); errno == 0 { + return sessionAttr(session) + } + if nativePath, ok := f.nativePath(cleaned); ok { + info, err := os.Stat(nativePath) + if err == nil { + if info.IsDir() { + return Attr{Mode: syscall.S_IFDIR | 0o700, ModTime: info.ModTime()}, 0 + } + return Attr{Mode: syscall.S_IFREG | 0o600, Size: info.Size(), ModTime: info.ModTime()}, 0 + } + if !errors.Is(err, os.ErrNotExist) { + return Attr{}, errnoFor(err) + } + } + } + session, errno := f.sessionForPath(cleaned) if errno != 0 { return Attr{}, errno } + return sessionAttr(session) +} + +func sessionAttr(session *vfs.Session) (Attr, syscall.Errno) { info, err := session.VisibleInfo() if err != nil { return Attr{}, errnoFor(err) @@ -104,7 +319,23 @@ func (f *Filesystem) Getattr(name string) (Attr, syscall.Errno) { func (f *Filesystem) Open(name string, flags int) (uint64, syscall.Errno) { session, errno := f.sessionForPath(name) if errno != 0 { - return 0, errno + if !f.canonical { + return 0, errno + } + nativePath, ok := f.nativePath(cleanPath(name)) + if !ok { + return 0, errno + } + native, err := os.OpenFile(nativePath, flags, 0o600) + if err != nil { + return 0, errnoFor(err) + } + f.mu.Lock() + handleID := f.next + f.next++ + f.handles[handleID] = &fileHandle{native: native, append: flags&os.O_APPEND != 0} + f.mu.Unlock() + return handleID, 0 } handle := &fileHandle{session: session, append: flags&os.O_APPEND != 0} access := flags & (os.O_WRONLY | os.O_RDWR) @@ -144,11 +375,21 @@ func (f *Filesystem) Open(name string, flags int) (uint64, syscall.Errno) { func (f *Filesystem) Read(handleID uint64, destination []byte, offset int64) (int, syscall.Errno) { handle, errno := f.handle(handleID) - if errno != 0 || handle.read == nil { + if errno != 0 { return 0, syscall.EBADF } handle.mu.Lock() defer handle.mu.Unlock() + if handle.native != nil { + n, err := handle.native.ReadAt(destination, offset) + if err != nil && !errors.Is(err, io.EOF) { + return n, errnoFor(err) + } + return n, 0 + } + if handle.read == nil { + return 0, syscall.EBADF + } n, err := handle.read.ReadAt(context.Background(), destination, offset) if err != nil && !errors.Is(err, io.EOF) { return n, errnoFor(err) @@ -158,11 +399,24 @@ func (f *Filesystem) Read(handleID uint64, destination []byte, offset int64) (in func (f *Filesystem) Write(handleID uint64, data []byte, offset int64) (int, syscall.Errno) { handle, errno := f.handle(handleID) - if errno != 0 || handle.write == nil { + if errno != 0 { return 0, syscall.EBADF } handle.mu.Lock() defer handle.mu.Unlock() + if handle.native != nil { + var n int + var err error + if handle.append { + n, err = handle.native.Write(data) + } else { + n, err = handle.native.WriteAt(data, offset) + } + return n, errnoFor(err) + } + if handle.write == nil { + return 0, syscall.EBADF + } var n int var err error if handle.append { @@ -191,11 +445,17 @@ func (f *Filesystem) Write(handleID uint64, data []byte, offset int64) (int, sys func (f *Filesystem) Truncate(handleID uint64, size int64) syscall.Errno { handle, errno := f.handle(handleID) - if errno != 0 || handle.write == nil { + if errno != 0 { return syscall.EBADF } handle.mu.Lock() defer handle.mu.Unlock() + if handle.native != nil { + return errnoFor(handle.native.Truncate(size)) + } + if handle.write == nil { + return syscall.EBADF + } if err := handle.write.Truncate(context.Background(), size); err != nil { return errnoFor(err) } @@ -208,6 +468,11 @@ func (f *Filesystem) Truncate(handleID uint64, size int64) syscall.Errno { func (f *Filesystem) TruncatePath(name string, size int64) syscall.Errno { session, errno := f.sessionForPath(name) if errno != 0 { + if f.canonical { + if nativePath, ok := f.nativePath(cleanPath(name)); ok { + return errnoFor(os.Truncate(nativePath, size)) + } + } return errno } if handle := f.lockActiveWriter(session); handle != nil { @@ -251,6 +516,9 @@ func (f *Filesystem) Fsync(handleID uint64) syscall.Errno { } handle.mu.Lock() defer handle.mu.Unlock() + if handle.native != nil { + return errnoFor(handle.native.Sync()) + } if handle.write == nil { return 0 } @@ -274,6 +542,9 @@ func (f *Filesystem) Release(handleID uint64) syscall.Errno { } handle.mu.Lock() defer handle.mu.Unlock() + if handle.native != nil { + return errnoFor(handle.native.Close()) + } var result syscall.Errno if handle.read != nil { if err := handle.read.Close(); err != nil { @@ -303,11 +574,114 @@ func refreshReader(handle *fileHandle) syscall.Errno { return 0 } -func (f *Filesystem) Rename(string, string) syscall.Errno { return syscall.EPERM } -func (f *Filesystem) Unlink(string) syscall.Errno { return syscall.EPERM } +func (f *Filesystem) Mkdir(name string, _ uint32) syscall.Errno { + cleaned := cleanPath(name) + if !f.canonical || cleaned == "" || cleaned == "/" || !canonicalNamespacePath(cleaned) { + return syscall.EPERM + } + f.mu.Lock() + if _, exists := f.directories[cleaned]; exists { + f.mu.Unlock() + return syscall.EEXIST + } + if _, exists := f.paths[cleaned]; exists { + f.mu.Unlock() + return syscall.EEXIST + } + root := f.nativeRoot + _, virtualParent := f.directories[path.Dir(cleaned)] + f.mu.Unlock() + if !virtualParent && root == "" { + return syscall.ENOENT + } + if root != "" { + if err := os.Mkdir(nativePathFromRoot(root, cleaned), 0o700); err != nil { + return errnoFor(err) + } + } + f.mu.Lock() + defer f.mu.Unlock() + f.directories[cleaned] = struct{}{} + return 0 +} + +func (f *Filesystem) Rename(oldName string, newName string) syscall.Errno { + if !f.canonical { + return syscall.EPERM + } + oldPath, newPath := cleanPath(oldName), cleanPath(newName) + if !canonicalSessionPath(oldPath) || !canonicalSessionPath(newPath) { + return syscall.EPERM + } + f.mu.Lock() + sessionID, exists := f.paths[oldPath] + if !exists { + if _, hidden := f.retained[oldPath]; hidden { + f.mu.Unlock() + return syscall.ENOENT + } + root := f.nativeRoot + f.mu.Unlock() + if root == "" { + return syscall.ENOENT + } + oldNative := nativePathFromRoot(root, oldPath) + newNative := nativePathFromRoot(root, newPath) + if err := os.Rename(oldNative, newNative); err != nil { + return errnoFor(err) + } + return 0 + } + defer f.mu.Unlock() + if _, exists := f.directories[path.Dir(newPath)]; !exists { + root := f.nativeRoot + info, err := os.Stat(nativePathFromRoot(root, path.Dir(newPath))) + if root == "" || err != nil || !info.IsDir() { + return syscall.ENOENT + } + } + if _, exists := f.paths[newPath]; exists { + return syscall.EEXIST + } + if err := moveManagedXattrCarrier(f.nativeRoot, oldPath, newPath); err != nil { + return errnoFor(err) + } + delete(f.paths, oldPath) + f.paths[newPath] = sessionID + return 0 +} + +func (f *Filesystem) Unlink(name string) syscall.Errno { + if !f.canonical { + return syscall.EPERM + } + cleaned := cleanPath(name) + f.mu.RLock() + _, managed := f.paths[cleaned] + f.mu.RUnlock() + if managed { + return syscall.EPERM + } + nativePath, ok := f.nativePath(cleaned) + if !ok { + return syscall.ENOENT + } + err := os.Remove(nativePath) + return errnoFor(err) +} func (f *Filesystem) sessionForPath(name string) (*vfs.Session, syscall.Errno) { cleaned := cleanPath(name) + if f.canonical { + f.mu.RLock() + sessionID := f.paths[cleaned] + session := f.sessions[sessionID] + f.mu.RUnlock() + if session == nil { + return nil, syscall.ENOENT + } + return session, 0 + } if cleaned == "/" || strings.Count(cleaned, "/") != 1 || !strings.HasSuffix(cleaned, ".jsonl") { return nil, syscall.ENOENT } @@ -350,6 +724,70 @@ func (f *Filesystem) sessionForPath(name string) (*vfs.Session, syscall.Errno) { return session, 0 } +func safeSessionID(sessionID string) bool { + return sessionID != "" && !strings.ContainsAny(sessionID, "/\\\x00") +} + +func canonicalSessionPath(name string) bool { + if name == "" || !strings.HasSuffix(name, ".jsonl") { + return false + } + return strings.HasPrefix(name, "/sessions/") || strings.HasPrefix(name, "/archived_sessions/") +} + +func canonicalNamespacePath(name string) bool { + return name == "/" || name == "/sessions" || name == "/archived_sessions" || + strings.HasPrefix(name, "/sessions/") || strings.HasPrefix(name, "/archived_sessions/") +} + +func moveAppleDoubleSidecar(root string, oldPath string, newPath string) error { + if root == "" || oldPath == "" || newPath == "" || !canonicalSessionPath(oldPath) || !canonicalSessionPath(newPath) { + return nil + } + oldSidecar := filepath.Join(nativePathFromRoot(root, path.Dir(oldPath)), "._"+path.Base(oldPath)) + newSidecar := filepath.Join(nativePathFromRoot(root, path.Dir(newPath)), "._"+path.Base(newPath)) + if _, err := os.Lstat(oldSidecar); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return err + } + return os.Rename(oldSidecar, newSidecar) +} + +func managedXattrCarrier(root string, name string) string { + digest := sha256.Sum256([]byte(cleanPath(name))) + return filepath.Join(root, ".codexfold-xattrs", hex.EncodeToString(digest[:])) +} + +func moveManagedMetadata(root string, oldPath string, newPath string) error { + if err := moveAppleDoubleSidecar(root, oldPath, newPath); err != nil { + return err + } + return moveManagedXattrCarrier(root, oldPath, newPath) +} + +func moveManagedXattrCarrier(root string, oldPath string, newPath string) error { + if root == "" || oldPath == "" || newPath == "" || !canonicalSessionPath(oldPath) || !canonicalSessionPath(newPath) { + return nil + } + oldCarrier := managedXattrCarrier(root, oldPath) + newCarrier := managedXattrCarrier(root, newPath) + if _, err := os.Lstat(oldCarrier); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return err + } + return os.Rename(oldCarrier, newCarrier) +} + +func (f *Filesystem) ensureDirectoryChainLocked(directory string) { + for directory != "." && directory != "/" && directory != "" { + f.directories[directory] = struct{}{} + directory = path.Dir(directory) + } + f.directories["/"] = struct{}{} +} + func (f *Filesystem) handle(handleID uint64) (*fileHandle, syscall.Errno) { f.mu.RLock() handle := f.handles[handleID] @@ -360,6 +798,41 @@ func (f *Filesystem) handle(handleID uint64) (*fileHandle, syscall.Errno) { return handle, 0 } +func (f *Filesystem) nativePath(name string) (string, bool) { + f.mu.RLock() + root := f.nativeRoot + _, retained := f.retained[name] + f.mu.RUnlock() + if !f.canonical || root == "" || retained || name == "" || name == "/" || !canonicalNamespacePath(name) { + return "", false + } + return nativePathFromRoot(root, name), true +} + +func (f *Filesystem) registerRetainedPathLocked(sessionID string, session *vfs.Session) { + for retained, currentID := range f.retained { + if currentID == sessionID { + delete(f.retained, retained) + } + } + if f.nativeRoot == "" || session == nil { + return + } + snapshot := filepath.Clean(session.State().NativeSnapshot.Path) + relative, err := filepath.Rel(f.nativeRoot, snapshot) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return + } + retained := cleanPath(filepath.ToSlash(relative)) + if canonicalSessionPath(retained) { + f.retained[retained] = sessionID + } +} + +func nativePathFromRoot(root string, name string) string { + return filepath.Join(root, filepath.FromSlash(strings.TrimPrefix(name, "/"))) +} + func cleanPath(name string) string { if name == "" || strings.ContainsRune(name, '\x00') || strings.Contains(name, "..") { return "" diff --git a/internal/mountfs/filesystem_test.go b/internal/mountfs/filesystem_test.go index 28835a6..b384cec 100644 --- a/internal/mountfs/filesystem_test.go +++ b/internal/mountfs/filesystem_test.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "strings" "syscall" "testing" @@ -174,6 +175,374 @@ func TestFilesystemRejectsUnsafeAndManagementMutations(t *testing.T) { } } +func TestCanonicalFilesystemMovesManagedSessionBetweenArchiveAndActivePaths(t *testing.T) { + source := []byte("canonical-session\n") + session := mountSessionFixture(t, "session", source) + filesystem := NewCanonical() + filename := "rollout-2026-07-12T14-28-28-session.jsonl" + archivedPath := "/archived_sessions/" + filename + if err := filesystem.AddSessionAt("session", archivedPath, session); err != nil { + t.Fatal(err) + } + for _, directory := range []string{"/sessions/2026", "/sessions/2026/07", "/sessions/2026/07/12"} { + if errno := filesystem.Mkdir(directory, 0o700); errno != 0 { + t.Fatalf("Mkdir %s errno=%v", directory, errno) + } + } + activePath := "/sessions/2026/07/12/" + filename + if errno := filesystem.Rename(archivedPath, activePath); errno != 0 { + t.Fatalf("Rename errno=%v", errno) + } + if _, errno := filesystem.Getattr(archivedPath); errno != syscall.ENOENT { + t.Fatalf("archived path errno=%v, want ENOENT", errno) + } + attribute, errno := filesystem.Getattr(activePath) + if errno != 0 || attribute.Mode&syscall.S_IFREG == 0 || attribute.Size != int64(len(source)) { + t.Fatalf("active Getattr = %#v errno=%v", attribute, errno) + } + entries, errno := filesystem.ReadDir("/sessions/2026/07/12") + if errno != 0 || len(entries) != 1 || entries[0] != filename { + t.Fatalf("active ReadDir = %#v errno=%v", entries, errno) + } + handle, errno := filesystem.Open(activePath, os.O_RDONLY) + if errno != 0 { + t.Fatalf("Open active errno=%v", errno) + } + defer filesystem.Release(handle) + got := make([]byte, len(source)) + if n, errno := filesystem.Read(handle, got, 0); errno != 0 || n != len(source) || !bytes.Equal(got, source) { + t.Fatalf("Read active = %d errno=%v bytes=%q", n, errno, got) + } +} + +func TestCanonicalFilesystemManagedSessionMasksRetainedSnapshotAtCurrentRoute(t *testing.T) { + root := t.TempDir() + filename := "rollout-2026-07-12T14-28-28-session.jsonl" + archivedPath := "/archived_sessions/" + filename + nativePath := filepath.Join(root, "archived_sessions", filename) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "sessions"), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("native-base\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + session := mountSessionWithNativeSnapshot(t, "session", base, nativePath) + writer, err := session.OpenWriter() + if err != nil { + t.Fatal(err) + } + tail := []byte("managed-tail\n") + if _, err := writer.Append(context.Background(), tail); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + if err := filesystem.AddSessionAt("session", archivedPath, session); err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), base...), tail...) + attribute, errno := filesystem.Getattr(archivedPath) + if errno != 0 || attribute.Size != int64(len(want)) { + t.Fatalf("managed Getattr = %#v errno=%v, want size %d", attribute, errno, len(want)) + } + handle, errno := filesystem.Open(archivedPath, os.O_RDONLY) + if errno != 0 { + t.Fatalf("managed Open errno=%v", errno) + } + defer filesystem.Release(handle) + got := make([]byte, len(want)) + if n, errno := filesystem.Read(handle, got, 0); errno != 0 || n != len(want) || !bytes.Equal(got, want) { + t.Fatalf("managed Read = %d errno=%v bytes=%q want=%q", n, errno, got, want) + } +} + +func TestCanonicalFilesystemHidesRetainedSnapshotAfterManagedRouteMoves(t *testing.T) { + root := t.TempDir() + filename := "rollout-2026-07-12T14-28-28-session.jsonl" + archivedPath := "/archived_sessions/" + filename + activePath := "/sessions/2026/07/12/" + filename + nativePath := filepath.Join(root, "archived_sessions", filename) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "sessions"), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("retained-base\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + session := mountSessionWithNativeSnapshot(t, "session", base, nativePath) + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + if err := filesystem.AddSessionAt("session", archivedPath, session); err != nil { + t.Fatal(err) + } + for _, directory := range []string{"/sessions/2026", "/sessions/2026/07", "/sessions/2026/07/12"} { + if errno := filesystem.Mkdir(directory, 0o700); errno != 0 { + t.Fatalf("Mkdir %s errno=%v", directory, errno) + } + } + if errno := filesystem.Rename(archivedPath, activePath); errno != 0 { + t.Fatalf("Rename errno=%v", errno) + } + if _, errno := filesystem.Getattr(archivedPath); errno != syscall.ENOENT { + t.Fatalf("retained archived Getattr errno=%v, want ENOENT", errno) + } + if _, errno := filesystem.Open(archivedPath, os.O_RDONLY); errno != syscall.ENOENT { + t.Fatalf("retained archived Open errno=%v, want ENOENT", errno) + } + entries, errno := filesystem.ReadDir("/archived_sessions") + if errno != 0 { + t.Fatalf("archived ReadDir errno=%v", errno) + } + for _, entry := range entries { + if entry == filename { + t.Fatalf("retained snapshot leaked into archived directory: %#v", entries) + } + } +} + +func TestCanonicalFilesystemMovesManagedSessionIntoExistingNativeDirectoryAfterRestart(t *testing.T) { + root := t.TempDir() + activeDirectory := filepath.Join(root, "sessions", "2026", "07", "12") + if err := os.MkdirAll(activeDirectory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + + session := mountSessionFixture(t, "session", []byte("restart-route\n")) + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + filename := "rollout-2026-07-12T14-28-28-session.jsonl" + archivedPath := "/archived_sessions/" + filename + activePath := "/sessions/2026/07/12/" + filename + if err := filesystem.AddSessionAt("session", archivedPath, session); err != nil { + t.Fatal(err) + } + if errno := filesystem.Rename(archivedPath, activePath); errno != 0 { + t.Fatalf("Rename into existing native directory errno=%v", errno) + } + if _, errno := filesystem.Getattr(archivedPath); errno != syscall.ENOENT { + t.Fatalf("archived path errno=%v, want ENOENT", errno) + } + if _, errno := filesystem.Getattr(activePath); errno != 0 { + t.Fatalf("active path errno=%v", errno) + } + if len(filesystem.paths) != 1 || filesystem.paths[activePath] != "session" { + t.Fatalf("managed routes after rename = %#v", filesystem.paths) + } + for _, nativePath := range []string{ + nativePathFromRoot(root, archivedPath), + nativePathFromRoot(root, activePath), + } { + if _, err := os.Stat(nativePath); !os.IsNotExist(err) { + t.Fatalf("managed session was duplicated into native root: path=%s err=%v", nativePath, err) + } + } +} + +func TestCanonicalFilesystemKeepsEmptyNativeRootDisabled(t *testing.T) { + filesystem := NewCanonical() + filesystem.SetNativeRoot("") + if filesystem.nativeRoot != "" { + t.Fatalf("empty native root became %q", filesystem.nativeRoot) + } + if _, ok := filesystem.nativePath("/sessions/2026/07/12/rollout.jsonl"); ok { + t.Fatal("empty native root should not resolve a backing path") + } +} + +func TestCanonicalFilesystemHidesNativeFilesOutsideSessionNamespace(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "sessions"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "outside.txt"), []byte("outside"), 0o600); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + entries, errno := filesystem.ReadDir("/") + if errno != 0 || len(entries) != 2 || entries[0] != "archived_sessions" || entries[1] != "sessions" { + t.Fatalf("canonical root entries = %#v errno=%v", entries, errno) + } + if _, errno := filesystem.Getattr("/outside.txt"); errno != syscall.ENOENT { + t.Fatalf("outside Getattr errno=%v, want ENOENT", errno) + } + if _, errno := filesystem.Open("/outside.txt", os.O_RDONLY); errno != syscall.ENOENT { + t.Fatalf("outside Open errno=%v, want ENOENT", errno) + } + if errno := filesystem.Mkdir("/outside", 0o700); errno != syscall.EPERM { + t.Fatalf("outside Mkdir errno=%v, want EPERM", errno) + } +} + +func TestCanonicalFilesystemMovesAppleDoubleSidecarWithManagedRoute(t *testing.T) { + root := t.TempDir() + oldDirectory := filepath.Join(root, "archived_sessions") + newDirectory := filepath.Join(root, "sessions", "2026", "07", "12") + if err := os.MkdirAll(oldDirectory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(newDirectory, 0o700); err != nil { + t.Fatal(err) + } + filename := "rollout-session.jsonl" + oldPath := "/archived_sessions/" + filename + newPath := "/sessions/2026/07/12/" + filename + oldSidecar := filepath.Join(oldDirectory, "._"+filename) + if err := os.WriteFile(oldSidecar, []byte("appledouble-metadata"), 0o600); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + if err := filesystem.AddSessionAt("session", oldPath, mountSessionFixture(t, "session", []byte("session\n"))); err != nil { + t.Fatal(err) + } + if errno := filesystem.MoveSessionAt("session", newPath); errno != nil { + t.Fatalf("MoveSessionAt errno=%v", errno) + } + if _, err := os.Stat(oldSidecar); !os.IsNotExist(err) { + t.Fatalf("old AppleDouble sidecar remained: %v", err) + } + newSidecar := filepath.Join(newDirectory, "._"+filename) + if got, err := os.ReadFile(newSidecar); err != nil || string(got) != "appledouble-metadata" { + t.Fatalf("new AppleDouble sidecar = %q err=%v", got, err) + } + entries, errno := filesystem.ReadDir(filepath.Dir(newPath)) + if errno != 0 || len(entries) != 2 || entries[0] != "._"+filename || entries[1] != filename { + t.Fatalf("session directory entries = %#v errno=%v", entries, errno) + } +} + +func TestCanonicalFilesystemUpsertMovesExistingSessionRoute(t *testing.T) { + session := mountSessionFixture(t, "session", []byte("route-update\n")) + filesystem := NewCanonical() + oldPath := "/archived_sessions/rollout-session.jsonl" + newPath := "/sessions/2026/07/12/rollout-session.jsonl" + if err := filesystem.AddSessionAt("session", oldPath, session); err != nil { + t.Fatal(err) + } + if err := filesystem.UpsertSessionAt("session", newPath, session); err != nil { + t.Fatal(err) + } + if _, errno := filesystem.Getattr(oldPath); errno != syscall.ENOENT { + t.Fatalf("old route errno=%v, want ENOENT", errno) + } + if _, errno := filesystem.Getattr(newPath); errno != 0 { + t.Fatalf("new route errno=%v", errno) + } +} + +func TestCanonicalFilesystemRemoveSessionRevealsNativeFile(t *testing.T) { + root := t.TempDir() + route := "/archived_sessions/rollout-session.jsonl" + nativePath := nativePathFromRoot(root, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(nativePath, []byte("native\n"), 0o600); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + if err := filesystem.AddSessionAt("session", route, mountSessionFixture(t, "managed", []byte("managed\n"))); err != nil { + t.Fatal(err) + } + if err := filesystem.RemoveSession("session"); err != nil { + t.Fatal(err) + } + handle, errno := filesystem.Open(route, os.O_RDONLY) + if errno != 0 { + t.Fatalf("open native after removal errno=%v", errno) + } + defer filesystem.Release(handle) + got := make([]byte, len("native\n")) + if n, errno := filesystem.Read(handle, got, 0); errno != 0 || n != len(got) || string(got) != "native\n" { + t.Fatalf("native after removal = %q n=%d errno=%v", got, n, errno) + } +} + +func TestCanonicalFilesystemPassesThroughNativeSessionFiles(t *testing.T) { + root := t.TempDir() + nativeDirectory := filepath.Join(root, "sessions", "2026", "07", "12") + if err := os.MkdirAll(nativeDirectory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + nativePath := filepath.Join(nativeDirectory, "native.jsonl") + source := []byte("native-session\n") + if err := os.WriteFile(nativePath, source, 0o600); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + entries, errno := filesystem.ReadDir("/sessions/2026/07/12") + if errno != 0 || len(entries) != 1 || entries[0] != "native.jsonl" { + t.Fatalf("native ReadDir = %#v errno=%v", entries, errno) + } + handle, errno := filesystem.Open("/sessions/2026/07/12/native.jsonl", os.O_RDONLY) + if errno != 0 { + t.Fatalf("native Open errno=%v", errno) + } + got := make([]byte, len(source)) + if n, errno := filesystem.Read(handle, got, 0); errno != 0 || n != len(source) || !bytes.Equal(got, source) { + t.Fatalf("native Read = %d errno=%v bytes=%q", n, errno, got) + } + if errno := filesystem.Release(handle); errno != 0 { + t.Fatalf("native Release errno=%v", errno) + } + + createdPath := "/sessions/2026/07/12/created.jsonl" + created, errno := filesystem.Open(createdPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL) + if errno != 0 { + t.Fatalf("native create Open errno=%v", errno) + } + createdBytes := []byte("created-session\n") + if n, errno := filesystem.Write(created, createdBytes, 0); errno != 0 || n != len(createdBytes) { + t.Fatalf("native create Write = %d errno=%v", n, errno) + } + if errno := filesystem.Release(created); errno != 0 { + t.Fatalf("native create Release errno=%v", errno) + } + if got, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(strings.TrimPrefix(createdPath, "/")))); err != nil || !bytes.Equal(got, createdBytes) { + t.Fatalf("native created bytes = %q err=%v", got, err) + } + + renamedPath := "/archived_sessions/created.jsonl" + if errno := filesystem.Rename(createdPath, renamedPath); errno != 0 { + t.Fatalf("native Rename errno=%v", errno) + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(strings.TrimPrefix(createdPath, "/")))); !os.IsNotExist(err) { + t.Fatalf("native source remained after rename: %v", err) + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(strings.TrimPrefix(renamedPath, "/")))); err != nil { + t.Fatalf("native destination missing after rename: %v", err) + } + if errno := filesystem.Unlink(renamedPath); errno != 0 { + t.Fatalf("native Unlink errno=%v", errno) + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(strings.TrimPrefix(renamedPath, "/")))); !os.IsNotExist(err) { + t.Fatalf("native destination remained after unlink: %v", err) + } +} + func TestFilesystemUpsertChangesNewOpensWithoutInvalidatingExistingHandles(t *testing.T) { first := mountSessionFixture(t, "first-session", []byte("first")) second := mountSessionFixture(t, "second-session", []byte("second")) @@ -266,3 +635,16 @@ func mountSessionFixture(t *testing.T, sessionID string, source []byte) *vfs.Ses } return session } + +func mountSessionWithNativeSnapshot(t *testing.T, sessionID string, source []byte, nativePath string) *vfs.Session { + t.Helper() + root := t.TempDir() + digest := sha256.Sum256(source) + hexDigest := hex.EncodeToString(digest[:]) + manifest := fold.Manifest{Version: fold.ManifestVersion, Kind: fold.ManifestKind, Session: fold.ManifestSession{ID: sessionID, RolloutPath: nativePath}, Source: fold.ManifestSource{Bytes: int64(len(source)), SHA256: hexDigest}, Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: hexDigest, RawBytes: int64(len(source))}}}} + session, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, Reader: mountReader{hexDigest: source}, NativeSnapshot: vfs.NativeFile{Path: nativePath, Bytes: int64(len(source)), SHA256: hexDigest}}) + if err != nil { + t.Fatal(err) + } + return session +} diff --git a/internal/mountfs/fuse_integration_test.go b/internal/mountfs/fuse_integration_test.go index 921a21d..8fbb689 100644 --- a/internal/mountfs/fuse_integration_test.go +++ b/internal/mountfs/fuse_integration_test.go @@ -11,6 +11,7 @@ import ( "io" "os" "path/filepath" + "strings" "sync" "sync/atomic" "testing" @@ -19,6 +20,7 @@ import ( "github.com/jstar0/codexfold/internal/fold" "github.com/jstar0/codexfold/internal/service" "github.com/jstar0/codexfold/internal/vfs" + "golang.org/x/sys/unix" ) func TestRealFuseMountNativeFileOperations(t *testing.T) { @@ -55,6 +57,10 @@ func TestRealFuseMountNativeFileOperations(t *testing.T) { t.Fatal(err) } stopMount := startRealMount(t, mountPoint, filesystem) + identity, err := os.ReadFile(filepath.Join(mountPoint, ".codexfold-health")) + if err != nil || len(identity) < 16 { + t.Fatalf("mount identity file is unavailable: size=%d err=%v", len(identity), err) + } target := filepath.Join(mountPoint, "fixture.jsonl") entries, err := os.ReadDir(mountPoint) if err != nil || len(entries) != 1 || entries[0].Name() != "fixture.jsonl" { @@ -146,12 +152,107 @@ func TestRealFuseMountNativeFileOperations(t *testing.T) { waitForRealUnmount(t, mountPoint) } +func TestRealFuseMountCanonicalManagedRename(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + source := []byte("canonical-session\n") + managed := mountSessionFixture(t, "fixture", source) + nativeRoot := filepath.Join(root, "native") + nativeActiveDirectory := filepath.Join(nativeRoot, "sessions", "2026", "07", "12") + if err := os.MkdirAll(nativeActiveDirectory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(nativeRoot, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + filename := "rollout-2026-07-12T14-28-28-fixture.jsonl" + archivedPath := "/archived_sessions/" + filename + if err := filesystem.AddSessionAt("fixture", archivedPath, managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + var recordedMu sync.Mutex + var recorded []string + stopMount := startRealMountWithOptions(t, HostOptions{ + MountPoint: mountPoint, Filesystem: filesystem, Foreground: true, + OperationRecorder: func(operation string) { + recordedMu.Lock() + recorded = append(recorded, operation) + recordedMu.Unlock() + }, + }) + archivedTarget := filepath.Join(mountPoint, "archived_sessions", filename) + activeTarget := filepath.Join(mountPoint, "sessions", "2026", "07", "12", filename) + attributeName := "com.codexfold.test" + attributeValue := []byte("persistent-metadata") + if err := unix.Setxattr(archivedTarget, attributeName, attributeValue, 0); err != nil { + t.Fatalf("set managed xattr: %v", err) + } + archivedSidecar := filepath.Join(nativeRoot, "archived_sessions", "._"+filename) + activeSidecar := filepath.Join(nativeActiveDirectory, "._"+filename) + sidecar, err := os.ReadFile(archivedSidecar) + if err != nil || !bytes.Contains(sidecar, []byte(attributeName)) || !bytes.Contains(sidecar, attributeValue) { + t.Fatalf("AppleDouble sidecar did not preserve xattr: bytes=%d err=%v", len(sidecar), err) + } + if err := os.Rename(archivedTarget, activeTarget); err != nil { + t.Fatalf("rename canonical managed session: %v", err) + } + if _, err := os.Stat(archivedTarget); !os.IsNotExist(err) { + t.Fatalf("archived path remained after rename: %v", err) + } + got, err := os.ReadFile(activeTarget) + if err != nil || !bytes.Equal(got, source) { + t.Fatalf("active managed bytes differ: got=%q err=%v", got, err) + } + if _, err := os.Stat(archivedSidecar); !os.IsNotExist(err) { + t.Fatalf("archived AppleDouble sidecar remained after rename: %v", err) + } + movedSidecar, err := os.ReadFile(activeSidecar) + if err != nil || !bytes.Contains(movedSidecar, []byte(attributeName)) || !bytes.Contains(movedSidecar, attributeValue) { + t.Fatalf("active AppleDouble sidecar lost xattr: bytes=%d err=%v", len(movedSidecar), err) + } + if err := os.Rename(activeTarget, archivedTarget); err != nil { + t.Fatalf("rename canonical managed session back: %v", err) + } + if _, err := os.Stat(activeTarget); !os.IsNotExist(err) { + t.Fatalf("active path remained after reverse rename: %v", err) + } + if _, err := os.Stat(activeSidecar); !os.IsNotExist(err) { + t.Fatalf("active AppleDouble sidecar remained after reverse rename: %v", err) + } + restoredSidecar, err := os.ReadFile(archivedSidecar) + if err != nil || !bytes.Contains(restoredSidecar, []byte(attributeName)) || !bytes.Contains(restoredSidecar, attributeValue) { + t.Fatalf("restored AppleDouble sidecar lost xattr: bytes=%d err=%v", len(restoredSidecar), err) + } + recordedMu.Lock() + joined := strings.Join(recorded, ",") + recordedMu.Unlock() + for _, operation := range []string{"getattr", "rename", "open", "read", "release"} { + if !strings.Contains(joined, operation) { + t.Fatalf("operation trace missing %q: %s", operation, joined) + } + } + stopMount() + waitForRealUnmount(t, mountPoint) +} + func startRealMount(t *testing.T, mountPoint string, filesystem *Filesystem) func() { + return startRealMountWithOptions(t, HostOptions{MountPoint: mountPoint, Filesystem: filesystem, Foreground: true}) +} + +func startRealMountWithOptions(t *testing.T, options HostOptions) func() { t.Helper() ctx, cancel := context.WithCancel(context.Background()) mountDone := make(chan error, 1) go func() { - mountDone <- Mount(ctx, HostOptions{MountPoint: mountPoint, Filesystem: filesystem, Foreground: true}) + mountDone <- Mount(ctx, options) }() var stopOnce sync.Once stopMount := func() { @@ -168,16 +269,19 @@ func startRealMount(t *testing.T, mountPoint string, filesystem *Filesystem) fun }) } t.Cleanup(stopMount) - waitForRealMount(t, mountPoint, mountDone) + waitForRealMount(t, options.MountPoint, mountDone) return stopMount } func waitForRealMount(t *testing.T, mountPoint string, mountDone <-chan error) { t.Helper() deadline := time.Now().Add(20 * time.Second) + var lastProbeErr error for time.Now().Before(deadline) { if err := service.ProbeMount(mountPoint); err == nil { return + } else { + lastProbeErr = err } select { case err := <-mountDone: @@ -185,7 +289,7 @@ func waitForRealMount(t *testing.T, mountPoint string, mountDone <-chan error) { case <-time.After(100 * time.Millisecond): } } - t.Fatal("FUSE mount did not become healthy") + t.Fatalf("FUSE mount did not become healthy: %v", lastProbeErr) } func waitForRealUnmount(t *testing.T, mountPoint string) { diff --git a/internal/mountfs/host.go b/internal/mountfs/host.go index 2faaf71..1594874 100644 --- a/internal/mountfs/host.go +++ b/internal/mountfs/host.go @@ -3,19 +3,52 @@ package mountfs import ( "context" "errors" + "fmt" + "os" ) var ErrPrerequisite = errors.New("FUSE host prerequisite is unavailable in this build") type HostOptions struct { - MountPoint string - Filesystem *Filesystem - Foreground bool + MountPoint string + Filesystem *Filesystem + Foreground bool + OperationRecorder func(string) } func Mount(ctx context.Context, options HostOptions) error { if options.MountPoint == "" || options.Filesystem == nil { return errors.New("mount point and filesystem are required") } + if !Available() { + return ErrPrerequisite + } + if err := prepareMountPoint(options.MountPoint); err != nil { + return err + } return mountHost(ctx, options) } + +func prepareMountPoint(path string) error { + info, err := os.Lstat(path) + if err != nil { + return fmt.Errorf("inspect mount backing directory: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 { + return errors.New("mount backing path must not be a symlink") + } + if !info.IsDir() { + return errors.New("mount backing path is not a directory") + } + entries, err := os.ReadDir(path) + if err != nil { + return fmt.Errorf("inspect mount backing contents: %w", err) + } + if len(entries) != 0 { + return errors.New("mount backing directory must be empty") + } + if err := os.Chmod(path, 0o500); err != nil { + return fmt.Errorf("seal mount backing directory: %w", err) + } + return nil +} diff --git a/internal/mountfs/host_cgofuse.go b/internal/mountfs/host_cgofuse.go index 209cee4..b4ba78d 100644 --- a/internal/mountfs/host_cgofuse.go +++ b/internal/mountfs/host_cgofuse.go @@ -3,24 +3,43 @@ package mountfs import ( + "bytes" "context" "errors" "fmt" "os" + "path/filepath" "runtime" + "strings" "syscall" + "github.com/jstar0/codexfold/internal/mountid" "github.com/winfsp/cgofuse/fuse" + "golang.org/x/sys/unix" ) type fuseFilesystem struct { fuse.FileSystemBase - core *Filesystem + core *Filesystem + recorder func(string) + mountIdentity []byte } +const healthHandle = ^uint64(0) - 1 + func Available() bool { return true } func (f *fuseFilesystem) Getattr(name string, stat *fuse.Stat_t, _ uint64) int { + f.record("getattr") + if cleanPath(name) == "/"+mountid.Path { + stat.Mode = syscall.S_IFREG | 0o400 + stat.Size = int64(len(f.mountIdentity)) + stat.Nlink = 1 + stat.Blksize = 4096 + stat.Blocks = (stat.Size + 511) / 512 + stat.Uid, stat.Gid, _ = fuse.Getcontext() + return 0 + } attribute, errno := f.core.Getattr(name) if errno != 0 { return -int(errno) @@ -37,7 +56,44 @@ func (f *fuseFilesystem) Getattr(name string, stat *fuse.Stat_t, _ uint64) int { return 0 } +func (f *fuseFilesystem) Statfs(name string, stat *fuse.Statfs_t) int { + f.core.mu.RLock() + root := f.core.nativeRoot + f.core.mu.RUnlock() + if root == "" { + result := -int(syscall.ENOENT) + f.recordResult("statfs", name, result) + return result + } + var source unix.Statfs_t + result := unixResult(unix.Statfs(root, &source)) + if result == 0 { + stat.Bsize = uint64(source.Bsize) + if source.Iosize > 0 { + stat.Frsize = uint64(source.Iosize) + } else { + stat.Frsize = uint64(source.Bsize) + } + stat.Blocks = source.Blocks + stat.Bfree = source.Bfree + stat.Bavail = source.Bavail + stat.Files = source.Files + stat.Ffree = source.Ffree + stat.Favail = source.Ffree + stat.Namemax = 255 + } + f.recordResult("statfs", name, result) + return result +} + +func (f *fuseFilesystem) Mknod(name string, _ uint32, _ uint64) int { + result := -int(syscall.ENOSYS) + f.recordResult("mknod", name, result) + return result +} + func (f *fuseFilesystem) Opendir(name string) (int, uint64) { + f.record("opendir") if _, errno := f.core.ReadDir(name); errno != 0 { return -int(errno), ^uint64(0) } @@ -45,6 +101,7 @@ func (f *fuseFilesystem) Opendir(name string) (int, uint64) { } func (f *fuseFilesystem) Readdir(name string, fill func(string, *fuse.Stat_t, int64) bool, _ int64, _ uint64) int { + f.record("readdir") entries, errno := f.core.ReadDir(name) if errno != 0 { return -int(errno) @@ -60,14 +117,41 @@ func (f *fuseFilesystem) Readdir(name string, fill func(string, *fuse.Stat_t, in } func (f *fuseFilesystem) Open(name string, flags int) (int, uint64) { + if cleanPath(name) == "/"+mountid.Path { + if flags&fuse.O_ACCMODE != fuse.O_RDONLY { + return -int(syscall.EPERM), ^uint64(0) + } + return 0, healthHandle + } handle, errno := f.core.Open(name, translateOpenFlags(flags)) if errno != 0 { - return -int(errno), ^uint64(0) + result := -int(errno) + f.recordResult("open", name, result) + return result, ^uint64(0) } + f.recordResult("open", name, 0) + return 0, handle +} + +func (f *fuseFilesystem) Create(name string, flags int, _ uint32) (int, uint64) { + handle, errno := f.core.Open(name, translateOpenFlags(flags)|os.O_CREATE) + if errno != 0 { + result := -int(errno) + f.recordResult("create", name, result) + return result, ^uint64(0) + } + f.recordResult("create", name, 0) return 0, handle } func (f *fuseFilesystem) Read(_ string, destination []byte, offset int64, handle uint64) int { + f.record("read") + if handle == healthHandle { + if offset < 0 || offset >= int64(len(f.mountIdentity)) { + return 0 + } + return copy(destination, f.mountIdentity[offset:]) + } n, errno := f.core.Read(handle, destination, offset) if errno != 0 { return -int(errno) @@ -76,6 +160,7 @@ func (f *fuseFilesystem) Read(_ string, destination []byte, offset int64, handle } func (f *fuseFilesystem) Write(_ string, data []byte, offset int64, handle uint64) int { + f.record("write") n, errno := f.core.Write(handle, data, offset) if errno != 0 { return -int(errno) @@ -84,6 +169,7 @@ func (f *fuseFilesystem) Write(_ string, data []byte, offset int64, handle uint6 } func (f *fuseFilesystem) Truncate(name string, size int64, handle uint64) int { + f.record("truncate") var errno syscall.Errno if handle == 0 || handle == ^uint64(0) { errno = f.core.TruncatePath(name, size) @@ -94,28 +180,255 @@ func (f *fuseFilesystem) Truncate(name string, size int64, handle uint64) int { } func (f *fuseFilesystem) Flush(_ string, handle uint64) int { + f.record("flush") + if handle == healthHandle { + return 0 + } return -int(f.core.Flush(handle)) } func (f *fuseFilesystem) Fsync(_ string, _ bool, handle uint64) int { + f.record("fsync") + if handle == healthHandle { + return 0 + } return -int(f.core.Fsync(handle)) } func (f *fuseFilesystem) Release(_ string, handle uint64) int { + f.record("release") + if handle == healthHandle { + return 0 + } return -int(f.core.Release(handle)) } +func (f *fuseFilesystem) Mkdir(name string, mode uint32) int { + result := -int(f.core.Mkdir(name, mode)) + f.recordResult("mkdir", name, result) + return result +} + +func (f *fuseFilesystem) Rmdir(name string) int { + result := -int(syscall.ENOSYS) + f.recordResult("rmdir", name, result) + return result +} + +func (f *fuseFilesystem) Link(oldName string, _ string) int { + result := -int(syscall.ENOSYS) + f.recordResult("link", oldName, result) + return result +} + +func (f *fuseFilesystem) Symlink(_ string, newName string) int { + result := -int(syscall.ENOSYS) + f.recordResult("symlink", newName, result) + return result +} + +func (f *fuseFilesystem) Readlink(name string) (int, string) { + result := -int(syscall.ENOSYS) + f.recordResult("readlink", name, result) + return result, "" +} + func (f *fuseFilesystem) Rename(oldName string, newName string) int { - return -int(f.core.Rename(oldName, newName)) + result := -int(f.core.Rename(oldName, newName)) + f.recordResult("rename", oldName, result) + return result } -func (f *fuseFilesystem) Unlink(name string) int { return -int(f.core.Unlink(name)) } +func (f *fuseFilesystem) Unlink(name string) int { + result := -int(f.core.Unlink(name)) + f.recordResult("unlink", name, result) + return result +} func (f *fuseFilesystem) Access(name string, _ uint32) int { + f.record("access") + if cleanPath(name) == "/"+mountid.Path { + return 0 + } _, errno := f.core.Getattr(name) return -int(errno) } +func (f *fuseFilesystem) Chmod(name string, mode uint32) int { + path, managed, errc := f.metadataPath(name) + if errc != 0 { + f.recordResult("chmod", name, errc) + return errc + } + result := 0 + if !managed { + result = unixResult(os.Chmod(path, os.FileMode(mode)&os.ModePerm)) + } + f.recordResult("chmod", name, result) + return result +} + +func (f *fuseFilesystem) Chown(name string, uid uint32, gid uint32) int { + path, managed, errc := f.metadataPath(name) + if errc != 0 { + f.recordResult("chown", name, errc) + return errc + } + result := 0 + if !managed { + result = unixResult(os.Chown(path, int(uid), int(gid))) + } + f.recordResult("chown", name, result) + return result +} + +func (f *fuseFilesystem) Utimens(name string, times []fuse.Timespec) int { + path, managed, errc := f.metadataPath(name) + if errc != 0 { + f.recordResult("utimens", name, errc) + return errc + } + result := 0 + if !managed { + if len(times) != 2 { + result = -int(syscall.EINVAL) + } else { + unixTimes := []unix.Timespec{{Sec: times[0].Sec, Nsec: times[0].Nsec}, {Sec: times[1].Sec, Nsec: times[1].Nsec}} + result = unixResult(unix.UtimesNanoAt(unix.AT_FDCWD, path, unixTimes, 0)) + } + } + f.recordResult("utimens", name, result) + return result +} + +func (f *fuseFilesystem) Setxattr(name string, attribute string, value []byte, flags int) int { + f.record("setxattr") + path, errc := f.xattrPath(name, true) + if errc != 0 { + return errc + } + return unixResult(unix.Setxattr(path, attribute, value, flags)) +} + +func (f *fuseFilesystem) Getxattr(name string, attribute string) (int, []byte) { + f.record("getxattr") + path, errc := f.xattrPath(name, false) + if errc != 0 { + return errc, nil + } + size, err := unix.Getxattr(path, attribute, nil) + if err != nil { + return unixResult(err), nil + } + value := make([]byte, size) + n, err := unix.Getxattr(path, attribute, value) + if err != nil { + return unixResult(err), nil + } + return 0, value[:n] +} + +func (f *fuseFilesystem) Listxattr(name string, fill func(string) bool) int { + f.record("listxattr") + path, errc := f.xattrPath(name, false) + if errc != 0 { + return errc + } + size, err := unix.Listxattr(path, nil) + if err != nil { + return unixResult(err) + } + buffer := make([]byte, size) + n, err := unix.Listxattr(path, buffer) + if err != nil { + return unixResult(err) + } + for _, attribute := range bytes.Split(bytes.TrimRight(buffer[:n], "\x00"), []byte{0}) { + if len(attribute) != 0 && !fill(string(attribute)) { + break + } + } + return 0 +} + +func (f *fuseFilesystem) Removexattr(name string, attribute string) int { + f.record("removexattr") + path, errc := f.xattrPath(name, false) + if errc != 0 { + return errc + } + return unixResult(unix.Removexattr(path, attribute)) +} + +func (f *fuseFilesystem) xattrPath(name string, create bool) (string, int) { + cleaned := cleanPath(name) + if _, errno := f.core.sessionForPath(cleaned); errno == 0 { + f.core.mu.RLock() + root := f.core.nativeRoot + f.core.mu.RUnlock() + if root == "" { + return "", -int(syscall.ENOTSUP) + } + carrier := managedXattrCarrier(root, cleaned) + if create { + if err := os.MkdirAll(filepath.Dir(carrier), 0o700); err != nil { + return "", unixResult(err) + } + file, err := os.OpenFile(carrier, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return "", unixResult(err) + } + if err := file.Close(); err != nil { + return "", unixResult(err) + } + } + return carrier, 0 + } + if native, ok := f.core.nativePath(cleaned); ok { + return native, 0 + } + return "", -int(syscall.ENOENT) +} + +func (f *fuseFilesystem) metadataPath(name string) (string, bool, int) { + cleaned := cleanPath(name) + if _, errno := f.core.sessionForPath(cleaned); errno == 0 { + return "", true, 0 + } + if native, ok := f.core.nativePath(cleaned); ok { + return native, false, 0 + } + return "", false, -int(syscall.ENOENT) +} + +func unixResult(err error) int { + if err == nil { + return 0 + } + var errno syscall.Errno + if errors.As(err, &errno) { + return -int(errno) + } + return -int(syscall.EIO) +} + +func (f *fuseFilesystem) record(operation string) { + if f.recorder != nil { + f.recorder(operation) + } +} + +func (f *fuseFilesystem) recordResult(operation string, name string, result int) { + kind := "other" + base := filepath.Base(name) + if strings.HasPrefix(base, "._") { + kind = "appledouble" + } else if strings.HasSuffix(base, ".jsonl") { + kind = "session" + } + f.record(fmt.Sprintf("%s kind=%s result=%d", operation, kind, result)) +} + func translateOpenFlags(flags int) int { translated := os.O_RDONLY switch flags & fuse.O_ACCMODE { @@ -130,6 +443,12 @@ func translateOpenFlags(flags int) int { if flags&fuse.O_TRUNC != 0 { translated |= os.O_TRUNC } + if flags&fuse.O_CREAT != 0 { + translated |= os.O_CREATE + } + if flags&fuse.O_EXCL != 0 { + translated |= os.O_EXCL + } return translated } @@ -139,7 +458,11 @@ func mountHost(ctx context.Context, options HostOptions) (result error) { result = fmt.Errorf("%w: %v", ErrPrerequisite, recovered) } }() - filesystem := &fuseFilesystem{core: options.Filesystem} + identity, err := mountid.New() + if err != nil { + return fmt.Errorf("generate mount identity: %w", err) + } + filesystem := &fuseFilesystem{core: options.Filesystem, recorder: options.OperationRecorder, mountIdentity: []byte(identity)} host := fuse.NewFileSystemHost(filesystem) arguments := []string{"-o", "fsname=codexfold", "-o", "default_permissions", "-o", "attr_timeout=0", "-o", "entry_timeout=0", "-o", "negative_timeout=0"} if options.Foreground { diff --git a/internal/mountfs/host_safety_test.go b/internal/mountfs/host_safety_test.go new file mode 100644 index 0000000..feae965 --- /dev/null +++ b/internal/mountfs/host_safety_test.go @@ -0,0 +1,58 @@ +package mountfs + +import ( + "os" + "path/filepath" + "testing" +) + +func TestPrepareMountPointRejectsOrdinaryFiles(t *testing.T) { + mountPoint := filepath.Join(t.TempDir(), "mount") + if err := os.MkdirAll(filepath.Join(mountPoint, "sessions"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(mountPoint, "sessions", "stale.jsonl"), []byte("stale\n"), 0o600); err != nil { + t.Fatal(err) + } + + if err := prepareMountPoint(mountPoint); err == nil { + t.Fatal("non-empty ordinary directory must not be accepted as a mount backing directory") + } +} + +func TestPrepareMountPointSealsEmptyBackingDirectory(t *testing.T) { + mountPoint := filepath.Join(t.TempDir(), "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + + if err := prepareMountPoint(mountPoint); err != nil { + t.Fatal(err) + } + info, err := os.Stat(mountPoint) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm()&0o200 != 0 { + t.Fatalf("unmounted backing directory remained writable: mode=%#o", info.Mode().Perm()) + } + if err := os.Mkdir(filepath.Join(mountPoint, "sessions"), 0o700); err == nil { + t.Fatal("sealed unmounted backing directory accepted a namespace write") + } +} + +func TestPrepareMountPointRejectsSymlink(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "target") + if err := os.MkdirAll(target, 0o700); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.Symlink(target, mountPoint); err != nil { + t.Fatal(err) + } + + if err := prepareMountPoint(mountPoint); err == nil { + t.Fatal("mount backing path must not be a symlink") + } +} diff --git a/internal/mountid/identity.go b/internal/mountid/identity.go new file mode 100644 index 0000000..34a3279 --- /dev/null +++ b/internal/mountid/identity.go @@ -0,0 +1,34 @@ +package mountid + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "strings" +) + +const ( + Path = ".codexfold-health" + prefix = "codexfold-v1:" +) + +func New() (string, error) { + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + return "", err + } + return prefix + hex.EncodeToString(random[:]), nil +} + +func Validate(value []byte) error { + text := string(value) + if !strings.HasPrefix(text, prefix) { + return errors.New("mount identity prefix is invalid") + } + digest := strings.TrimPrefix(text, prefix) + decoded, err := hex.DecodeString(digest) + if err != nil || len(decoded) != 16 { + return errors.New("mount identity payload is invalid") + } + return nil +} diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go new file mode 100644 index 0000000..b4eb192 --- /dev/null +++ b/internal/reconcile/reconcile.go @@ -0,0 +1,442 @@ +package reconcile + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "time" +) + +type SourceSummary struct { + Path string `json:"path"` + Bytes int64 `json:"bytes"` + Records int64 `json:"records"` + SHA256 string `json:"sha256"` + FirstTimestamp string `json:"first_timestamp"` + LastTimestamp string `json:"last_timestamp"` + TimestampRegressions int64 `json:"timestamp_regressions"` +} + +type Result struct { + Base SourceSummary `json:"base"` + Branch SourceSummary `json:"branch"` + SharedRecords int64 `json:"shared_records"` + BaseOnlyRecords int64 `json:"base_only_records"` + AddedFromBranch int64 `json:"added_from_branch"` + OutputRecords int64 `json:"output_records"` + OutputBytes int64 `json:"output_bytes,omitempty"` + OutputSHA256 string `json:"output_sha256,omitempty"` + OutputPath string `json:"output_path,omitempty"` + OutputRegressions int64 `json:"output_timestamp_regressions,omitempty"` +} + +type recordKey struct { + digest [sha256.Size]byte + length int64 +} + +type recordRef struct { + source int + sequence int64 + offset int64 + length int64 + timestamp time.Time + key recordKey +} + +type scannedSource struct { + summary SourceSummary + records []recordRef + counts map[recordKey]int64 +} + +func Analyze(basePath, branchPath string) (Result, error) { + base, err := scanPath(basePath, 0) + if err != nil { + return Result{}, fmt.Errorf("scan base: %w", err) + } + branch, err := scanPath(branchPath, 1) + if err != nil { + return Result{}, fmt.Errorf("scan branch: %w", err) + } + result, _ := reconcileRecords(base, branch) + return result, nil +} + +func Merge(basePath, branchPath, outputPath string) (Result, error) { + if outputPath == "" { + return Result{}, errors.New("output path is required") + } + baseAbs, err := filepath.Abs(basePath) + if err != nil { + return Result{}, err + } + branchAbs, err := filepath.Abs(branchPath) + if err != nil { + return Result{}, err + } + outputAbs, err := filepath.Abs(outputPath) + if err != nil { + return Result{}, err + } + if outputAbs == baseAbs || outputAbs == branchAbs { + return Result{}, errors.New("output must not replace either source") + } + if _, err := os.Lstat(outputAbs); err == nil { + return Result{}, fmt.Errorf("output already exists: %s", outputAbs) + } else if !errors.Is(err, os.ErrNotExist) { + return Result{}, err + } + + base, err := scanPath(baseAbs, 0) + if err != nil { + return Result{}, fmt.Errorf("scan base: %w", err) + } + branch, err := scanPath(branchAbs, 1) + if err != nil { + return Result{}, fmt.Errorf("scan branch: %w", err) + } + result, records := reconcileRecords(base, branch) + sort.SliceStable(records, func(i, j int) bool { + if records[i].timestamp.Equal(records[j].timestamp) { + if records[i].source == records[j].source { + return records[i].sequence < records[j].sequence + } + return records[i].source < records[j].source + } + return records[i].timestamp.Before(records[j].timestamp) + }) + + baseFile, err := os.Open(baseAbs) + if err != nil { + return Result{}, err + } + defer baseFile.Close() + branchFile, err := os.Open(branchAbs) + if err != nil { + return Result{}, err + } + defer branchFile.Close() + + if err := os.MkdirAll(filepath.Dir(outputAbs), 0o700); err != nil { + return Result{}, err + } + temp, err := os.CreateTemp(filepath.Dir(outputAbs), ".codexfold-reconcile-*.tmp") + if err != nil { + return Result{}, err + } + tempPath := temp.Name() + committed := false + defer func() { + if !committed { + _ = temp.Close() + _ = os.Remove(tempPath) + } + }() + if err := temp.Chmod(0o600); err != nil { + return Result{}, err + } + + outputHasher := sha256.New() + writer := io.MultiWriter(temp, outputHasher) + for _, record := range records { + source := baseFile + if record.source == 1 { + source = branchFile + } + if _, err := io.Copy(writer, io.NewSectionReader(source, record.offset, record.length)); err != nil { + return Result{}, fmt.Errorf("write merged record: %w", err) + } + } + if err := temp.Sync(); err != nil { + return Result{}, err + } + if err := temp.Close(); err != nil { + return Result{}, err + } + if err := verifyUnchanged(baseAbs, base.summary); err != nil { + return Result{}, fmt.Errorf("base changed during merge: %w", err) + } + if err := verifyUnchanged(branchAbs, branch.summary); err != nil { + return Result{}, fmt.Errorf("branch changed during merge: %w", err) + } + if err := os.Rename(tempPath, outputAbs); err != nil { + return Result{}, err + } + committed = true + if err := syncDir(filepath.Dir(outputAbs)); err != nil { + return Result{}, err + } + + output, err := scanPath(outputAbs, 2) + if err != nil { + return Result{}, fmt.Errorf("verify output: %w", err) + } + result.OutputPath = outputAbs + result.OutputBytes = output.summary.Bytes + result.OutputSHA256 = hex.EncodeToString(outputHasher.Sum(nil)) + result.OutputRegressions = output.summary.TimestampRegressions + if output.summary.Records != result.OutputRecords || output.summary.SHA256 != result.OutputSHA256 || output.summary.TimestampRegressions != 0 { + return Result{}, errors.New("merged output verification failed") + } + return result, nil +} + +func reconcileRecords(base, branch scannedSource) (Result, []recordRef) { + remaining := make(map[recordKey]int64, len(base.counts)) + for key, count := range base.counts { + remaining[key] = count + } + records := make([]recordRef, 0, len(base.records)+len(branch.records)) + records = append(records, base.records...) + var shared int64 + var added int64 + for _, record := range branch.records { + if remaining[record.key] > 0 { + remaining[record.key]-- + shared++ + continue + } + records = append(records, record) + added++ + } + return Result{ + Base: base.summary, + Branch: branch.summary, + SharedRecords: shared, + BaseOnlyRecords: base.summary.Records - shared, + AddedFromBranch: added, + OutputRecords: base.summary.Records + added, + }, records +} + +func scanPath(path string, source int) (scannedSource, error) { + file, err := os.Open(path) + if err != nil { + return scannedSource{}, err + } + defer file.Close() + before, err := file.Stat() + if err != nil { + return scannedSource{}, err + } + + result := scannedSource{ + summary: SourceSummary{Path: path}, + counts: make(map[recordKey]int64), + } + reader := bufio.NewReaderSize(file, 1024*1024) + fileHasher := sha256.New() + var offset int64 + var previous time.Time + for sequence := int64(0); ; sequence++ { + start := offset + recordHasher := sha256.New() + timestampExtractor := newTimestampExtractor() + hasData := false + reachedEOF := false + for { + fragment, readErr := reader.ReadSlice('\n') + if len(fragment) > 0 { + hasData = true + offset += int64(len(fragment)) + _, _ = fileHasher.Write(fragment) + _, _ = recordHasher.Write(fragment) + if err := timestampExtractor.Write(fragment); err != nil { + return scannedSource{}, err + } + } + switch { + case readErr == nil: + goto recordComplete + case errors.Is(readErr, bufio.ErrBufferFull): + continue + case errors.Is(readErr, io.EOF): + reachedEOF = true + goto recordComplete + default: + return scannedSource{}, readErr + } + } + + recordComplete: + if !hasData { + break + } + timestamp, err := timestampExtractor.Timestamp() + if err != nil { + return scannedSource{}, fmt.Errorf("record %d at byte %d: %w", sequence+1, start, err) + } + if !previous.IsZero() && timestamp.Before(previous) { + result.summary.TimestampRegressions++ + } + if result.summary.Records == 0 { + result.summary.FirstTimestamp = timestamp.Format(time.RFC3339Nano) + } + previous = timestamp + result.summary.LastTimestamp = timestamp.Format(time.RFC3339Nano) + var digest [sha256.Size]byte + copy(digest[:], recordHasher.Sum(nil)) + key := recordKey{digest: digest, length: offset - start} + result.records = append(result.records, recordRef{ + source: source, + sequence: sequence, + offset: start, + length: offset - start, + timestamp: timestamp, + key: key, + }) + result.counts[key]++ + result.summary.Records++ + if reachedEOF { + break + } + } + after, err := file.Stat() + if err != nil { + return scannedSource{}, err + } + if before.Size() != after.Size() || !before.ModTime().Equal(after.ModTime()) { + return scannedSource{}, errors.New("source changed while scanning") + } + result.summary.Bytes = offset + result.summary.SHA256 = hex.EncodeToString(fileHasher.Sum(nil)) + return result, nil +} + +type timestampExtractor struct { + depth int + inString bool + escaped bool + capture bool + token []byte + topKey string + expectation extractorExpectation + timestamp string +} + +type extractorExpectation uint8 + +const ( + expectKey extractorExpectation = iota + expectColon + expectValue +) + +func newTimestampExtractor() *timestampExtractor { + return ×tampExtractor{expectation: expectKey} +} + +func (e *timestampExtractor) Write(data []byte) error { + for _, char := range data { + if e.inString { + if e.escaped { + e.escaped = false + if e.capture { + e.token = append(e.token, char) + } + continue + } + if char == '\\' { + e.escaped = true + continue + } + if char == '"' { + e.inString = false + if e.capture { + switch e.expectation { + case expectKey: + e.topKey = string(e.token) + e.expectation = expectColon + case expectValue: + if e.topKey == "timestamp" { + e.timestamp = string(e.token) + } + } + } + e.token = e.token[:0] + e.capture = false + continue + } + if e.capture { + e.token = append(e.token, char) + } + continue + } + + switch char { + case '"': + e.inString = true + if e.depth == 1 && (e.expectation == expectKey || (e.expectation == expectValue && e.topKey == "timestamp")) { + e.capture = true + e.token = e.token[:0] + } + case '{', '[': + e.depth++ + case '}', ']': + if e.depth > 0 { + e.depth-- + } + case ':': + if e.depth == 1 && e.expectation == expectColon { + e.expectation = expectValue + } + case ',': + if e.depth == 1 { + e.expectation = expectKey + e.topKey = "" + } + } + } + return nil +} + +func (e *timestampExtractor) Timestamp() (time.Time, error) { + if e.timestamp == "" { + return time.Time{}, errors.New("top-level timestamp not found in record") + } + timestamp, err := time.Parse(time.RFC3339Nano, e.timestamp) + if err != nil { + return time.Time{}, fmt.Errorf("invalid timestamp %q: %w", e.timestamp, err) + } + return timestamp, nil +} + +func verifyUnchanged(path string, expected SourceSummary) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + info, err := file.Stat() + if err != nil { + return err + } + if info.Size() != expected.Bytes { + return fmt.Errorf("size is %d, expected %d", info.Size(), expected.Bytes) + } + hasher := sha256.New() + if _, err := io.Copy(hasher, file); err != nil { + return err + } + actual := hex.EncodeToString(hasher.Sum(nil)) + if actual != expected.SHA256 { + return fmt.Errorf("sha256 is %s, expected %s", actual, expected.SHA256) + } + return nil +} + +func syncDir(path string) error { + dir, err := os.Open(path) + if err != nil { + return err + } + defer dir.Close() + return dir.Sync() +} diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go new file mode 100644 index 0000000..c8e351c --- /dev/null +++ b/internal/reconcile/reconcile_test.go @@ -0,0 +1,133 @@ +package reconcile + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestMergeInsertsBranchOnlyRecordsByTimestamp(t *testing.T) { + dir := t.TempDir() + base := writeRollout(t, dir, "base.jsonl", []string{ + record("2026-07-13T01:00:00Z", "a"), + record("2026-07-13T01:02:00Z", "c"), + }) + branch := writeRollout(t, dir, "branch.jsonl", []string{ + record("2026-07-13T01:00:00Z", "a"), + record("2026-07-13T01:01:00Z", "b"), + record("2026-07-13T01:02:00Z", "c"), + }) + output := filepath.Join(dir, "merged.jsonl") + + result, err := Merge(base, branch, output) + if err != nil { + t.Fatal(err) + } + if result.SharedRecords != 2 || result.AddedFromBranch != 1 || result.OutputRecords != 3 { + t.Fatalf("unexpected result: %#v", result) + } + data, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + want := strings.Join([]string{ + record("2026-07-13T01:00:00Z", "a"), + record("2026-07-13T01:01:00Z", "b"), + record("2026-07-13T01:02:00Z", "c"), + }, "\n") + "\n" + if string(data) != want { + t.Fatalf("merged bytes:\n%s\nwant:\n%s", data, want) + } +} + +func TestMergePreservesExcessDuplicateOccurrence(t *testing.T) { + dir := t.TempDir() + line := record("2026-07-13T01:00:00Z", "same") + base := writeRollout(t, dir, "base.jsonl", []string{line}) + branch := writeRollout(t, dir, "branch.jsonl", []string{line, line}) + output := filepath.Join(dir, "merged.jsonl") + + result, err := Merge(base, branch, output) + if err != nil { + t.Fatal(err) + } + if result.SharedRecords != 1 || result.AddedFromBranch != 1 || result.OutputRecords != 2 { + t.Fatalf("unexpected result: %#v", result) + } + data, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if string(data) != line+"\n"+line+"\n" { + t.Fatalf("duplicate occurrence was not preserved: %q", data) + } +} + +func TestMergeSortsTimestampRegression(t *testing.T) { + dir := t.TempDir() + base := writeRollout(t, dir, "base.jsonl", []string{ + record("2026-07-13T01:01:00Z", "later"), + record("2026-07-13T01:00:00Z", "earlier"), + }) + branch := writeRollout(t, dir, "branch.jsonl", []string{ + record("2026-07-13T01:00:30Z", "branch"), + }) + + report, err := Analyze(base, branch) + if err != nil { + t.Fatal(err) + } + if report.Base.TimestampRegressions != 1 { + t.Fatalf("regressions = %d, want 1", report.Base.TimestampRegressions) + } + output := filepath.Join(dir, "merged.jsonl") + if _, err := Merge(base, branch, output); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"value":"earlier"`) || !strings.HasPrefix(string(data), record("2026-07-13T01:00:00Z", "earlier")) { + t.Fatalf("merge did not sort records: %q", data) + } +} + +func TestAnalyzeRejectsMissingTimestamp(t *testing.T) { + dir := t.TempDir() + base := writeRollout(t, dir, "base.jsonl", []string{`{"type":"event_msg"}`}) + branch := writeRollout(t, dir, "branch.jsonl", []string{record("2026-07-13T01:00:00Z", "ok")}) + + if _, err := Analyze(base, branch); err == nil { + t.Fatal("analyze accepted a record without a timestamp") + } +} + +func TestAnalyzeFindsTopLevelTimestampAfterLargePayload(t *testing.T) { + dir := t.TempDir() + line := `{"payload":{"text":"` + strings.Repeat("x", 16*1024) + `","timestamp":"2000-01-01T00:00:00Z"},"timestamp":"2026-07-13T01:00:00Z","type":"session_meta"}` + base := writeRollout(t, dir, "base.jsonl", []string{line}) + branch := writeRollout(t, dir, "branch.jsonl", []string{line}) + + report, err := Analyze(base, branch) + if err != nil { + t.Fatal(err) + } + if report.Base.FirstTimestamp != "2026-07-13T01:00:00Z" { + t.Fatalf("timestamp = %s", report.Base.FirstTimestamp) + } +} + +func record(timestamp, value string) string { + return `{"timestamp":"` + timestamp + `","type":"event_msg","payload":{"value":"` + value + `"}}` +} + +func writeRollout(t *testing.T, dir, name string, lines []string) string { + t.Helper() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil { + t.Fatal(err) + } + return path +} diff --git a/internal/reconcile/repair.go b/internal/reconcile/repair.go new file mode 100644 index 0000000..d1f0d05 --- /dev/null +++ b/internal/reconcile/repair.go @@ -0,0 +1,369 @@ +package reconcile + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "time" +) + +const maxRepairBufferedBytes = int64(64 * 1024 * 1024) + +var recordStartPattern = regexp.MustCompile(`\{"timestamp"\s*:`) + +type RepairResult struct { + SourcePath string `json:"source_path"` + SourceBytes int64 `json:"source_bytes"` + SourceSHA256 string `json:"source_sha256"` + PhysicalLines int64 `json:"physical_lines"` + InvalidPhysicalLines int64 `json:"invalid_physical_lines"` + ReconstructedRecords int64 `json:"reconstructed_records"` + OutputPath string `json:"output_path"` + OutputBytes int64 `json:"output_bytes"` + OutputRecords int64 `json:"output_records"` + OutputSHA256 string `json:"output_sha256"` + TimestampRegressions int64 `json:"timestamp_regressions"` + MaximumBufferedBytes int64 `json:"maximum_buffered_bytes"` + OrphanBytes int64 `json:"orphan_bytes,omitempty"` + OrphanLines int64 `json:"orphan_lines,omitempty"` +} + +type RepairOptions struct { + AllowOrphans bool + OrphanPath string +} + +type repairFrame struct { + partial []byte + pending [][]byte + startedLine int64 +} + +type repairWriter struct { + writer io.Writer + stack []repairFrame + bufferedBytes int64 + maximumBuffered int64 + outputRecords int64 + reconstructed int64 + previousTimestamp time.Time + timestampRegressions int64 + orphanWriter *bufio.Writer + allowOrphans bool + orphanBytes int64 + orphanLines int64 +} + +func Repair(sourcePath, outputPath string) (RepairResult, error) { + return RepairWithOptions(sourcePath, outputPath, RepairOptions{}) +} + +func RepairWithOptions(sourcePath, outputPath string, options RepairOptions) (RepairResult, error) { + if outputPath == "" { + return RepairResult{}, errors.New("output path is required") + } + sourceAbs, err := filepath.Abs(sourcePath) + if err != nil { + return RepairResult{}, err + } + outputAbs, err := filepath.Abs(outputPath) + if err != nil { + return RepairResult{}, err + } + if sourceAbs == outputAbs { + return RepairResult{}, errors.New("output must not replace the source") + } + if _, err := os.Lstat(outputAbs); err == nil { + return RepairResult{}, fmt.Errorf("output already exists: %s", outputAbs) + } else if !errors.Is(err, os.ErrNotExist) { + return RepairResult{}, err + } + + source, err := os.Open(sourceAbs) + if err != nil { + return RepairResult{}, err + } + defer source.Close() + before, err := source.Stat() + if err != nil { + return RepairResult{}, err + } + if err := os.MkdirAll(filepath.Dir(outputAbs), 0o700); err != nil { + return RepairResult{}, err + } + temp, err := os.CreateTemp(filepath.Dir(outputAbs), ".codexfold-repair-*.tmp") + if err != nil { + return RepairResult{}, err + } + tempPath := temp.Name() + committed := false + defer func() { + if !committed { + _ = temp.Close() + _ = os.Remove(tempPath) + } + }() + if err := temp.Chmod(0o600); err != nil { + return RepairResult{}, err + } + var orphanFile *os.File + var orphanWriter *bufio.Writer + if options.AllowOrphans { + if options.OrphanPath == "" { + return RepairResult{}, errors.New("orphan path is required when allow orphans is enabled") + } + orphanFile, err = os.OpenFile(options.OrphanPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return RepairResult{}, err + } + defer orphanFile.Close() + orphanWriter = bufio.NewWriterSize(orphanFile, 64*1024) + } + + result := RepairResult{SourcePath: sourceAbs, OutputPath: outputAbs} + sourceHasher := sha256.New() + outputHasher := sha256.New() + processor := repairWriter{writer: io.MultiWriter(temp, outputHasher), orphanWriter: orphanWriter, allowOrphans: options.AllowOrphans} + reader := bufio.NewReaderSize(source, 1024*1024) + for { + line, readErr := reader.ReadBytes('\n') + if len(line) > 0 { + result.PhysicalLines++ + result.SourceBytes += int64(len(line)) + _, _ = sourceHasher.Write(line) + line = bytes.TrimSuffix(line, []byte{'\n'}) + line = bytes.TrimSuffix(line, []byte{'\r'}) + if json.Valid(line) { + if err := processor.acceptValid(line); err != nil { + return RepairResult{}, fmt.Errorf("physical line %d: %w", result.PhysicalLines, err) + } + } else { + result.InvalidPhysicalLines++ + if err := processor.acceptFragment(line, result.PhysicalLines); err != nil { + return RepairResult{}, fmt.Errorf("physical line %d: %w", result.PhysicalLines, err) + } + } + } + if errors.Is(readErr, io.EOF) { + break + } + if readErr != nil { + return RepairResult{}, readErr + } + } + if len(processor.stack) != 0 && !options.AllowOrphans { + return RepairResult{}, fmt.Errorf("unresolved interrupted record started at physical line %d", processor.stack[0].startedLine) + } + if len(processor.stack) != 0 { + for _, frame := range processor.stack { + if err := processor.writeOrphan(frame.partial); err != nil { + return RepairResult{}, err + } + for _, pending := range frame.pending { + if err := processor.writeOrphan(pending); err != nil { + return RepairResult{}, err + } + } + } + processor.stack = nil + } + if processor.timestampRegressions != 0 && !options.AllowOrphans { + return RepairResult{}, fmt.Errorf("repaired record order still has %d timestamp regressions", processor.timestampRegressions) + } + after, err := source.Stat() + if err != nil { + return RepairResult{}, err + } + if before.Size() != after.Size() || !before.ModTime().Equal(after.ModTime()) { + return RepairResult{}, errors.New("source changed while repairing") + } + if err := temp.Sync(); err != nil { + return RepairResult{}, err + } + if err := temp.Close(); err != nil { + return RepairResult{}, err + } + if orphanWriter != nil { + if err := orphanWriter.Flush(); err != nil { + return RepairResult{}, err + } + if err := orphanFile.Sync(); err != nil { + return RepairResult{}, err + } + } + verified, err := scanPath(tempPath, 2) + if err != nil { + return RepairResult{}, fmt.Errorf("verify repaired output: %w", err) + } + if (!options.AllowOrphans && verified.summary.TimestampRegressions != 0) || verified.summary.Records != processor.outputRecords { + return RepairResult{}, errors.New("repaired output verification failed") + } + outputDigest := hex.EncodeToString(outputHasher.Sum(nil)) + if verified.summary.SHA256 != outputDigest { + return RepairResult{}, errors.New("repaired output digest verification failed") + } + if err := os.Rename(tempPath, outputAbs); err != nil { + return RepairResult{}, err + } + committed = true + if err := syncDir(filepath.Dir(outputAbs)); err != nil { + return RepairResult{}, err + } + + result.SourceSHA256 = hex.EncodeToString(sourceHasher.Sum(nil)) + result.ReconstructedRecords = processor.reconstructed + result.OutputBytes = verified.summary.Bytes + result.OutputRecords = verified.summary.Records + result.OutputSHA256 = outputDigest + result.TimestampRegressions = verified.summary.TimestampRegressions + result.MaximumBufferedBytes = processor.maximumBuffered + result.OrphanBytes = processor.orphanBytes + result.OrphanLines = processor.orphanLines + return result, nil +} + +func (w *repairWriter) acceptValid(record []byte) error { + if len(w.stack) == 0 { + return w.writeRecord(record) + } + copyOfRecord := append([]byte(nil), record...) + top := &w.stack[len(w.stack)-1] + top.pending = append(top.pending, copyOfRecord) + return w.addBuffered(int64(len(copyOfRecord))) +} + +func (w *repairWriter) acceptFragment(line []byte, physicalLine int64) error { + starts := recordStartPattern.FindAllIndex(line, -1) + if len(starts) == 0 { + if len(w.stack) == 0 { + return w.handleOrphan(line) + } + return w.appendFragment(line) + } + + cursor := 0 + for _, match := range starts { + start := match[0] + if start > cursor { + if len(w.stack) == 0 { + if len(bytes.TrimSpace(line[cursor:start])) != 0 { + if err := w.handleOrphan(line[cursor:start]); err != nil { + return err + } + } + } else if err := w.appendFragment(line[cursor:start]); err != nil { + return err + } + } + w.stack = append(w.stack, repairFrame{startedLine: physicalLine}) + cursor = start + } + return w.appendFragment(line[cursor:]) +} + +func (w *repairWriter) appendFragment(fragment []byte) error { + if len(w.stack) == 0 { + return w.handleOrphan(fragment) + } + top := &w.stack[len(w.stack)-1] + top.partial = append(top.partial, fragment...) + if err := w.addBuffered(int64(len(fragment))); err != nil { + return err + } + if json.Valid(top.partial) { + return w.finishTop() + } + return nil +} + +func (w *repairWriter) handleOrphan(fragment []byte) error { + if len(bytes.TrimSpace(fragment)) == 0 { + return nil + } + if !w.allowOrphans { + return errors.New("orphan JSON fragment has no active interrupted record") + } + return w.writeOrphan(fragment) +} + +func (w *repairWriter) writeOrphan(fragment []byte) error { + if w.orphanWriter == nil { + return errors.New("orphan writer is not configured") + } + if _, err := w.orphanWriter.Write(fragment); err != nil { + return err + } + if err := w.orphanWriter.WriteByte('\n'); err != nil { + return err + } + w.orphanBytes += int64(len(fragment)) + w.orphanLines++ + return nil +} + +func (w *repairWriter) finishTop() error { + index := len(w.stack) - 1 + frame := w.stack[index] + w.stack = w.stack[:index] + w.reconstructed++ + records := make([][]byte, 0, 1+len(frame.pending)) + records = append(records, frame.partial) + records = append(records, frame.pending...) + if len(w.stack) != 0 { + parent := &w.stack[len(w.stack)-1] + parent.pending = append(parent.pending, records...) + return nil + } + for _, record := range records { + if err := w.writeRecord(record); err != nil { + return err + } + w.bufferedBytes -= int64(len(record)) + } + return nil +} + +func (w *repairWriter) writeRecord(record []byte) error { + if !json.Valid(record) { + return errors.New("attempted to emit invalid JSON record") + } + extractor := newTimestampExtractor() + if err := extractor.Write(record); err != nil { + return err + } + timestamp, err := extractor.Timestamp() + if err != nil { + return err + } + if !w.previousTimestamp.IsZero() && timestamp.Before(w.previousTimestamp) { + w.timestampRegressions++ + } + w.previousTimestamp = timestamp + if _, err := w.writer.Write(record); err != nil { + return err + } + if _, err := w.writer.Write([]byte{'\n'}); err != nil { + return err + } + w.outputRecords++ + return nil +} + +func (w *repairWriter) addBuffered(bytes int64) error { + w.bufferedBytes += bytes + if w.bufferedBytes > w.maximumBuffered { + w.maximumBuffered = w.bufferedBytes + } + if w.bufferedBytes > maxRepairBufferedBytes { + return fmt.Errorf("interrupted record buffer exceeded %d bytes", maxRepairBufferedBytes) + } + return nil +} diff --git a/internal/reconcile/repair_test.go b/internal/reconcile/repair_test.go new file mode 100644 index 0000000..7ae7ae3 --- /dev/null +++ b/internal/reconcile/repair_test.go @@ -0,0 +1,98 @@ +package reconcile + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRepairRestoresInterruptedRecordBeforeInsertedRecord(t *testing.T) { + dir := t.TempDir() + outer := recordWithText("2026-07-13T01:00:00Z", "abcdef") + inner := recordWithText("2026-07-13T01:00:01Z", "inner") + prefix, suffix := splitAt(t, outer, "abc") + input := filepath.Join(dir, "broken.jsonl") + if err := os.WriteFile(input, []byte(prefix+inner+"\n"+suffix+"\n"), 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(dir, "repaired.jsonl") + + result, err := Repair(input, output) + if err != nil { + t.Fatal(err) + } + if result.InvalidPhysicalLines != 2 || result.ReconstructedRecords != 2 || result.OutputRecords != 2 { + t.Fatalf("unexpected result: %#v", result) + } + data, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if string(data) != outer+"\n"+inner+"\n" { + t.Fatalf("repaired bytes:\n%s", data) + } +} + +func TestRepairBuffersValidPhysicalRecordsWhileOuterRecordIsOpen(t *testing.T) { + dir := t.TempDir() + outer := recordWithText("2026-07-13T01:00:00Z", "abcdef") + inner := recordWithText("2026-07-13T01:00:01Z", "inner") + prefix, suffix := splitAt(t, outer, "abc") + input := filepath.Join(dir, "broken.jsonl") + if err := os.WriteFile(input, []byte(prefix+"\n"+inner+"\n"+suffix+"\n"), 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(dir, "repaired.jsonl") + + if _, err := Repair(input, output); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if string(data) != outer+"\n"+inner+"\n" { + t.Fatalf("repaired bytes:\n%s", data) + } +} + +func TestRepairRestoresNestedInterruptions(t *testing.T) { + dir := t.TempDir() + outer := recordWithText("2026-07-13T01:00:00Z", "abcdef") + middle := recordWithText("2026-07-13T01:00:01Z", "ghijkl") + inner := recordWithText("2026-07-13T01:00:02Z", "inner") + outerPrefix, outerSuffix := splitAt(t, outer, "abc") + middlePrefix, middleSuffix := splitAt(t, middle, "ghi") + input := filepath.Join(dir, "broken.jsonl") + physical := outerPrefix + middlePrefix + inner + "\n" + middleSuffix + "\n" + outerSuffix + "\n" + if err := os.WriteFile(input, []byte(physical), 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(dir, "repaired.jsonl") + + if _, err := Repair(input, output); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if string(data) != outer+"\n"+middle+"\n"+inner+"\n" { + t.Fatalf("repaired bytes:\n%s", data) + } +} + +func recordWithText(timestamp, text string) string { + return `{"timestamp":"` + timestamp + `","type":"event_msg","payload":{"text":"` + text + `"}}` +} + +func splitAt(t *testing.T, value, marker string) (string, string) { + t.Helper() + index := strings.Index(value, marker) + if index < 0 { + t.Fatalf("marker %q not found", marker) + } + index += len(marker) + return value[:index], value[index:] +} diff --git a/internal/service/mount_probe_darwin.go b/internal/service/mount_probe_darwin.go index 3377503..7865037 100644 --- a/internal/service/mount_probe_darwin.go +++ b/internal/service/mount_probe_darwin.go @@ -4,9 +4,12 @@ package service import ( "errors" + "fmt" + "os" "path/filepath" "strings" + "github.com/jstar0/codexfold/internal/mountid" "golang.org/x/sys/unix" ) @@ -28,6 +31,16 @@ func defaultMountProbe(path string) error { if !macFUSE && !fuseT { return errors.New("mount root is not backed by a supported FUSE provider") } + value, err := os.ReadFile(filepath.Join(path, mountid.Path)) + if err != nil { + return fmt.Errorf("read CodexFold mount identity: %w", err) + } + if len(value) == 0 || len(value) > 256 { + return errors.New("CodexFold mount identity size is invalid") + } + if err := mountid.Validate(value); err != nil { + return err + } return nil } diff --git a/internal/service/process_lock.go b/internal/service/process_lock.go new file mode 100644 index 0000000..373e0d0 --- /dev/null +++ b/internal/service/process_lock.go @@ -0,0 +1,64 @@ +package service + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +type ProcessLock struct { + file *os.File +} + +func AcquireProcessLock(path string) (*ProcessLock, error) { + if !filepath.IsAbs(path) { + return nil, errors.New("absolute process lock path is required") + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, err + } + file, err := os.OpenFile(filepath.Clean(path), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, err + } + locked, err := tryLockProcessFile(file) + if err != nil { + _ = file.Close() + return nil, err + } + if !locked { + _ = file.Close() + return nil, errors.New("filesystem service process lock is already held") + } + if err := file.Truncate(0); err != nil { + _ = unlockProcessFile(file) + _ = file.Close() + return nil, err + } + if _, err := fmt.Fprintf(file, "%d\n", os.Getpid()); err != nil { + _ = unlockProcessFile(file) + _ = file.Close() + return nil, err + } + if err := file.Sync(); err != nil { + _ = unlockProcessFile(file) + _ = file.Close() + return nil, err + } + return &ProcessLock{file: file}, nil +} + +func (l *ProcessLock) Close() error { + if l == nil || l.file == nil { + return nil + } + file := l.file + l.file = nil + unlockErr := unlockProcessFile(file) + closeErr := file.Close() + if unlockErr != nil { + return unlockErr + } + return closeErr +} diff --git a/internal/service/process_lock_unix.go b/internal/service/process_lock_unix.go new file mode 100644 index 0000000..3eb3013 --- /dev/null +++ b/internal/service/process_lock_unix.go @@ -0,0 +1,22 @@ +//go:build !windows + +package service + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +func tryLockProcessFile(file *os.File) (bool, error) { + err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if errors.Is(err, unix.EWOULDBLOCK) { + return false, nil + } + return err == nil, err +} + +func unlockProcessFile(file *os.File) error { + return unix.Flock(int(file.Fd()), unix.LOCK_UN) +} diff --git a/internal/service/process_lock_windows.go b/internal/service/process_lock_windows.go new file mode 100644 index 0000000..7ed4855 --- /dev/null +++ b/internal/service/process_lock_windows.go @@ -0,0 +1,23 @@ +//go:build windows + +package service + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +func tryLockProcessFile(file *os.File) (bool, error) { + overlapped := new(windows.Overlapped) + err := windows.LockFileEx(windows.Handle(file.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, overlapped) + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return false, nil + } + return err == nil, err +} + +func unlockProcessFile(file *os.File) error { + return windows.UnlockFileEx(windows.Handle(file.Fd()), 0, 1, 0, new(windows.Overlapped)) +} diff --git a/internal/service/service.go b/internal/service/service.go index 0d2c3a4..693a1bb 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -10,19 +10,23 @@ import ( "os/exec" "path/filepath" "strings" + "time" "github.com/jstar0/codexfold/internal/compat" "github.com/jstar0/codexfold/internal/fsctl" ) type Options struct { - Label string - BinaryPath string - CodexHome string - StoreDir string - MountPoint string - StdoutPath string - StderrPath string + Label string + BinaryPath string + CodexHome string + StoreDir string + MountPoint string + StdoutPath string + StderrPath string + CanonicalNamespace bool + NativeRoot string + OperationTrace string } type InstallResult struct { @@ -78,6 +82,12 @@ func RenderLaunchd(options Options) ([]byte, error) { options.BinaryPath, "fs", "serve", "--apply", "--foreground=true", "--codex-home", options.CodexHome, "--store", options.StoreDir, "--mount", options.MountPoint, } + if options.CanonicalNamespace { + arguments = append(arguments, "--canonical-namespace", "--native-root", options.NativeRoot) + } + if options.OperationTrace != "" { + arguments = append(arguments, "--operation-trace", options.OperationTrace) + } var output bytes.Buffer output.WriteString("\n") output.WriteString("\n") @@ -157,16 +167,19 @@ func (m Manager) Kickstart(ctx context.Context, label string) error { if !safeLabel(label) { return errors.New("safe launchd label is required") } - _, err := m.runner().Run(ctx, "launchctl", "kickstart", "-k", m.domain()+"/"+label) + _, err := m.runner().Run(ctx, "launchctl", "kickstart", m.domain()+"/"+label) return err } func (m Manager) Status(ctx context.Context, label string, mountPoint string) Status { result := Status{} - if _, err := m.runner().Run(ctx, "launchctl", "print", m.domain()+"/"+label); err != nil { + output, err := m.runner().Run(ctx, "launchctl", "print", m.domain()+"/"+label) + if err != nil { result.DaemonError = err.Error() - } else { + } else if strings.Contains(string(output), "state = running") { result.DaemonRunning = true + } else { + result.DaemonError = "launchd job is loaded but not running" } probe := m.MountProbe if probe == nil { @@ -180,6 +193,30 @@ func (m Manager) Status(ctx context.Context, label string, mountPoint string) St return result } +func (m Manager) WaitHealthy(ctx context.Context, label string, mountPoint string, timeout time.Duration) (Status, error) { + if timeout <= 0 { + timeout = 15 * time.Second + } + deadline := time.NewTimer(timeout) + defer deadline.Stop() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + var last Status + for { + last = m.Status(ctx, label, mountPoint) + if last.DaemonRunning && last.MountHealthy { + return last, nil + } + select { + case <-ctx.Done(): + return last, ctx.Err() + case <-deadline.C: + return last, fmt.Errorf("filesystem service did not become healthy: daemon=%t mount=%t daemon_error=%q mount_error=%q", last.DaemonRunning, last.MountHealthy, last.DaemonError, last.MountError) + case <-ticker.C: + } + } +} + func ProbeMount(path string) error { return defaultMountProbe(path) } func EvaluateUpdate(input UpdateInput) UpdateDecision { @@ -225,6 +262,12 @@ func validateOptions(options Options) error { return fmt.Errorf("%s path must be absolute", name) } } + if options.CanonicalNamespace && !filepath.IsAbs(options.NativeRoot) { + return errors.New("canonical namespace requires an absolute native root") + } + if options.OperationTrace != "" && !filepath.IsAbs(options.OperationTrace) { + return errors.New("operation trace path must be absolute") + } return nil } diff --git a/internal/service/service_test.go b/internal/service/service_test.go index 00dc6b1..af21dd4 100644 --- a/internal/service/service_test.go +++ b/internal/service/service_test.go @@ -9,6 +9,7 @@ import ( "runtime" "strings" "testing" + "time" "github.com/jstar0/codexfold/internal/compat" "github.com/jstar0/codexfold/internal/fsctl" @@ -20,13 +21,19 @@ func TestRenderLaunchdUsesAbsoluteArgumentsAndContainsNoSessionContent(t *testin Label: "com.codexfold.fs", BinaryPath: filepath.Join(root, "bin", "codexfold"), CodexHome: filepath.Join(root, "codex"), StoreDir: filepath.Join(root, "store"), MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "logs", "stdout.log"), - StderrPath: filepath.Join(root, "logs", "stderr.log"), + StderrPath: filepath.Join(root, "logs", "stderr.log"), CanonicalNamespace: true, + NativeRoot: filepath.Join(root, "native"), OperationTrace: filepath.Join(root, "logs", "operations.log"), }) if err != nil { t.Fatalf("RenderLaunchd: %v", err) } text := string(definition) - for _, required := range []string{"fs", "serve", "--apply", filepath.Join(root, "store"), filepath.Join(root, "mount")} { + for _, required := range []string{ + "fs", "serve", "--apply", + "--canonical-namespace", "--native-root", + "--operation-trace", filepath.Join(root, "logs", "operations.log"), + filepath.Join(root, "store"), filepath.Join(root, "mount"), filepath.Join(root, "native"), + } { if !strings.Contains(text, required) { t.Fatalf("definition missing %q:\n%s", required, text) } @@ -50,7 +57,7 @@ func TestRenderLaunchdUsesAbsoluteArgumentsAndContainsNoSessionContent(t *testin func TestManagerUsesOnlyPerUserLaunchctlAndSeparatesDaemonFromMount(t *testing.T) { root := t.TempDir() - runner := &recordingRunner{outputs: map[string][]byte{"launchctl print gui/501/com.codexfold.fs": []byte("running")}} + runner := &recordingRunner{outputs: map[string][]byte{"launchctl print gui/501/com.codexfold.fs": []byte("state = running\npid = 123\n")}} manager := Manager{UID: 501, Runner: runner, MountProbe: func(string) error { return errors.New("mount unavailable") }} plist := filepath.Join(root, "com.codexfold.fs.plist") if err := os.WriteFile(plist, []byte("plist"), 0o600); err != nil { @@ -67,11 +74,41 @@ func TestManagerUsesOnlyPerUserLaunchctlAndSeparatesDaemonFromMount(t *testing.T t.Fatalf("status did not separate daemon and mount: %#v", status) } joined := strings.Join(runner.calls, "\n") - if strings.Contains(joined, "sudo") || !strings.Contains(joined, "launchctl bootstrap gui/501") || !strings.Contains(joined, "launchctl kickstart -k gui/501/com.codexfold.fs") { + if strings.Contains(joined, "sudo") || !strings.Contains(joined, "launchctl bootstrap gui/501") || !strings.Contains(joined, "launchctl kickstart gui/501/com.codexfold.fs") { t.Fatalf("unexpected lifecycle commands:\n%s", joined) } } +func TestStatusDoesNotTreatLoadedExitedJobAsRunning(t *testing.T) { + runner := &recordingRunner{outputs: map[string][]byte{ + "launchctl print gui/501/com.codexfold.fs": []byte("state = exited\nlast exit code = 1\n"), + }} + status := (Manager{UID: 501, Runner: runner, MountProbe: func(string) error { return errors.New("not mounted") }}).Status( + context.Background(), "com.codexfold.fs", filepath.Join(t.TempDir(), "mount"), + ) + if status.DaemonRunning || status.DaemonError == "" { + t.Fatalf("loaded exited job was reported as running: %#v", status) + } +} + +func TestWaitHealthyRequiresRunningDaemonAndLiveMount(t *testing.T) { + runner := &recordingRunner{outputs: map[string][]byte{ + "launchctl print gui/501/com.codexfold.fs": []byte("state = running\npid = 123\n"), + }} + probes := 0 + manager := Manager{UID: 501, Runner: runner, MountProbe: func(string) error { + probes++ + if probes < 3 { + return errors.New("mount starting") + } + return nil + }} + status, err := manager.WaitHealthy(context.Background(), "com.codexfold.fs", filepath.Join(t.TempDir(), "mount"), time.Second) + if err != nil || !status.DaemonRunning || !status.MountHealthy || probes != 3 { + t.Fatalf("WaitHealthy status=%#v probes=%d err=%v", status, probes, err) + } +} + func TestEvaluateUpdateQuarantinesUnknownVersionsAndRejectsPreviewAutomation(t *testing.T) { unknown := EvaluateUpdate(UpdateInput{Capability: fsctl.StorageEngine, DoctorHealthy: true, Compatibility: compat.Evaluation{Quarantine: true}, NativeFallbackReady: false}) if unknown.Allowed || !unknown.Quarantine || !unknown.RequiresNativeFallback { @@ -91,6 +128,29 @@ func TestEvaluateUpdateQuarantinesUnknownVersionsAndRejectsPreviewAutomation(t * } } +func TestProcessLockAllowsOnlyOneFilesystemHost(t *testing.T) { + path := filepath.Join(t.TempDir(), "service.lock") + first, err := AcquireProcessLock(path) + if err != nil { + t.Fatal(err) + } + defer first.Close() + + if _, err := AcquireProcessLock(path); err == nil { + t.Fatal("a second filesystem host acquired the same process lock") + } + if err := first.Close(); err != nil { + t.Fatal(err) + } + second, err := AcquireProcessLock(path) + if err != nil { + t.Fatalf("lock was not released after the first host exited: %v", err) + } + if err := second.Close(); err != nil { + t.Fatal(err) + } +} + type recordingRunner struct { calls []string outputs map[string][]byte diff --git a/internal/sessionns/activation.go b/internal/sessionns/activation.go new file mode 100644 index 0000000..a2bedb1 --- /dev/null +++ b/internal/sessionns/activation.go @@ -0,0 +1,331 @@ +package sessionns + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" +) + +const ( + actionActivate = "activate" + actionDeactivate = "deactivate" +) + +var sessionDirectories = []string{"sessions", "archived_sessions"} + +type Options struct { + Home string + Mount string + NativeRoot string + MountProbe func(string) error +} + +type Result struct { + Active bool `json:"active"` + Recovered bool `json:"recovered"` + Home string `json:"home"` + Mount string `json:"mount"` + NativeRoot string `json:"native_root"` + Journal string `json:"journal"` +} + +type journal struct { + Version int `json:"version"` + Action string `json:"action"` +} + +func Inspect(options Options) (Result, error) { + options, err := validate(options) + if err != nil { + return Result{}, err + } + result := resultFor(options) + activeLinks := 0 + nativeDirectories := 0 + nativeEntries := 0 + for _, name := range sessionDirectories { + homePath := filepath.Join(options.Home, name) + info, err := os.Lstat(homePath) + if err != nil { + return Result{}, fmt.Errorf("inspect %s: %w", homePath, err) + } + if info.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(homePath) + if err != nil || filepath.Clean(target) != filepath.Join(options.Mount, name) { + return Result{}, fmt.Errorf("unexpected namespace link %s", homePath) + } + activeLinks++ + } else if !info.IsDir() { + return Result{}, fmt.Errorf("namespace source is not a directory: %s", homePath) + } + nativePath := filepath.Join(options.NativeRoot, name) + if info, err := os.Stat(nativePath); err == nil && info.IsDir() { + nativeDirectories++ + entries, err := os.ReadDir(nativePath) + if err != nil { + return Result{}, err + } + nativeEntries += len(entries) + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return Result{}, err + } + } + if activeLinks == len(sessionDirectories) && nativeDirectories == len(sessionDirectories) { + result.Active = true + return result, nil + } + if activeLinks == 0 && nativeEntries == 0 { + return result, nil + } + return Result{}, errors.New("session namespace is partially activated") +} + +func Activate(options Options) (Result, error) { + options, err := validate(options) + if err != nil { + return Result{}, err + } + if _, err := os.Stat(journalPath(options)); err == nil { + if _, recoverErr := Recover(options); recoverErr != nil { + return Result{}, recoverErr + } + } else if !errors.Is(err, os.ErrNotExist) { + return Result{}, err + } + if options.MountProbe == nil { + return Result{}, errors.New("mount identity probe is required for namespace activation") + } + if err := options.MountProbe(options.Mount); err != nil { + return Result{}, fmt.Errorf("canonical mount identity is not healthy: %w", err) + } + status, err := Inspect(options) + if err == nil && status.Active { + if err := installRouteGuard(options); err != nil { + return Result{}, err + } + return status, nil + } + if err != nil { + return Result{}, err + } + for _, name := range sessionDirectories { + if info, err := os.Stat(filepath.Join(options.Mount, name)); err != nil || !info.IsDir() { + return Result{}, fmt.Errorf("canonical mount directory is unavailable: %s", filepath.Join(options.Mount, name)) + } + } + if err := os.MkdirAll(options.NativeRoot, 0o700); err != nil { + return Result{}, err + } + for _, name := range sessionDirectories { + if err := removeEmptyDirectory(filepath.Join(options.NativeRoot, name)); err != nil { + return Result{}, err + } + } + if err := writeJournal(options, journal{Version: 1, Action: actionActivate}); err != nil { + return Result{}, err + } + for _, name := range sessionDirectories { + homePath := filepath.Join(options.Home, name) + nativePath := filepath.Join(options.NativeRoot, name) + if err := os.Rename(homePath, nativePath); err != nil { + return rollbackAfterError(options, err) + } + if err := os.Symlink(filepath.Join(options.Mount, name), homePath); err != nil { + return rollbackAfterError(options, err) + } + } + if err := installRouteGuard(options); err != nil { + return rollbackAfterError(options, err) + } + if err := removeJournal(options); err != nil { + return Result{}, err + } + return Inspect(options) +} + +func Deactivate(options Options) (Result, error) { + options, err := validate(options) + if err != nil { + return Result{}, err + } + status, err := Recover(options) + if err != nil { + return Result{}, err + } + if !status.Active { + return status, nil + } + if err := writeJournal(options, journal{Version: 1, Action: actionDeactivate}); err != nil { + return Result{}, err + } + for _, name := range sessionDirectories { + homePath := filepath.Join(options.Home, name) + if err := os.Remove(homePath); err != nil { + return finishAfterError(options, err) + } + if err := os.Rename(filepath.Join(options.NativeRoot, name), homePath); err != nil { + return finishAfterError(options, err) + } + } + if err := removeRouteGuard(options); err != nil { + return finishAfterError(options, err) + } + if err := removeJournal(options); err != nil { + return Result{}, err + } + return Inspect(options) +} + +func Recover(options Options) (Result, error) { + options, err := validate(options) + if err != nil { + return Result{}, err + } + data, err := os.ReadFile(journalPath(options)) + if errors.Is(err, os.ErrNotExist) { + return Inspect(options) + } + if err != nil { + return Result{}, err + } + var transaction journal + if err := json.Unmarshal(data, &transaction); err != nil || transaction.Version != 1 { + return Result{}, errors.New("invalid session namespace journal") + } + if transaction.Action != actionActivate && transaction.Action != actionDeactivate { + return Result{}, errors.New("unknown session namespace journal action") + } + for _, name := range sessionDirectories { + homePath := filepath.Join(options.Home, name) + nativePath := filepath.Join(options.NativeRoot, name) + if info, err := os.Lstat(homePath); err == nil && info.Mode()&os.ModeSymlink != 0 { + if err := os.Remove(homePath); err != nil { + return Result{}, err + } + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return Result{}, err + } + if _, err := os.Lstat(homePath); errors.Is(err, os.ErrNotExist) { + if info, nativeErr := os.Stat(nativePath); nativeErr == nil && info.IsDir() { + if err := os.Rename(nativePath, homePath); err != nil { + return Result{}, err + } + } else if nativeErr != nil && !errors.Is(nativeErr, os.ErrNotExist) { + return Result{}, nativeErr + } + } + } + if err := removeRouteGuard(options); err != nil { + return Result{}, err + } + if err := removeJournal(options); err != nil { + return Result{}, err + } + result, err := Inspect(options) + if err != nil { + return Result{}, err + } + result.Recovered = true + return result, nil +} + +func rollbackAfterError(options Options, cause error) (Result, error) { + _, recoverErr := Recover(options) + if recoverErr != nil { + return Result{}, errors.Join(cause, recoverErr) + } + return Result{}, cause +} + +func finishAfterError(options Options, cause error) (Result, error) { + _, recoverErr := Recover(options) + if recoverErr != nil { + return Result{}, errors.Join(cause, recoverErr) + } + return Result{}, cause +} + +func validate(options Options) (Options, error) { + if !filepath.IsAbs(options.Home) || !filepath.IsAbs(options.Mount) || !filepath.IsAbs(options.NativeRoot) { + return Options{}, errors.New("absolute home, mount, and native root paths are required") + } + options.Home = filepath.Clean(options.Home) + options.Mount = filepath.Clean(options.Mount) + options.NativeRoot = filepath.Clean(options.NativeRoot) + if options.Home == options.Mount || options.Home == options.NativeRoot || options.Mount == options.NativeRoot { + return Options{}, errors.New("home, mount, and native root paths must be distinct") + } + return options, nil +} + +func resultFor(options Options) Result { + return Result{Home: options.Home, Mount: options.Mount, NativeRoot: options.NativeRoot, Journal: journalPath(options)} +} + +func journalPath(options Options) string { + return filepath.Join(options.Home, ".codexfold-namespace.json") +} + +func writeJournal(options Options, transaction journal) error { + data, err := json.Marshal(transaction) + if err != nil { + return err + } + temporary, err := os.CreateTemp(options.Home, ".codexfold-namespace-*.tmp") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return err + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + if err := os.Rename(temporaryPath, journalPath(options)); err != nil { + return err + } + return syncDirectory(options.Home) +} + +func removeJournal(options Options) error { + if err := os.Remove(journalPath(options)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return syncDirectory(options.Home) +} + +func removeEmptyDirectory(path string) error { + entries, err := os.ReadDir(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if len(entries) != 0 { + return fmt.Errorf("native namespace destination is not empty: %s", path) + } + return os.Remove(path) +} + +func syncDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} diff --git a/internal/sessionns/activation_test.go b/internal/sessionns/activation_test.go new file mode 100644 index 0000000..970a051 --- /dev/null +++ b/internal/sessionns/activation_test.go @@ -0,0 +1,259 @@ +package sessionns + +import ( + "database/sql" + "os" + "path/filepath" + "testing" + + _ "modernc.org/sqlite" +) + +func TestActivateAndDeactivatePreserveSessionTrees(t *testing.T) { + root := t.TempDir() + home := filepath.Join(root, "home") + mount := filepath.Join(root, "mount") + nativeRoot := filepath.Join(home, "fold-native") + writeFixture(t, filepath.Join(home, "sessions", "2026", "07", "12", "active.jsonl"), "active\n") + writeFixture(t, filepath.Join(home, "archived_sessions", "archived.jsonl"), "archived\n") + createStateDatabase(t, home) + for _, directory := range []string{"sessions", "archived_sessions"} { + if err := os.MkdirAll(filepath.Join(mount, directory), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(nativeRoot, directory), 0o700); err != nil { + t.Fatal(err) + } + } + + result, err := Activate(Options{Home: home, Mount: mount, NativeRoot: nativeRoot, MountProbe: healthyMountProbe}) + if err != nil { + t.Fatal(err) + } + if !result.Active || result.Recovered { + t.Fatalf("activation result = %#v", result) + } + assertLink(t, filepath.Join(home, "sessions"), filepath.Join(mount, "sessions")) + assertLink(t, filepath.Join(home, "archived_sessions"), filepath.Join(mount, "archived_sessions")) + assertFile(t, filepath.Join(nativeRoot, "sessions", "2026", "07", "12", "active.jsonl"), "active\n") + assertFile(t, filepath.Join(nativeRoot, "archived_sessions", "archived.jsonl"), "archived\n") + + status, err := Inspect(Options{Home: home, Mount: mount, NativeRoot: nativeRoot}) + if err != nil || !status.Active { + t.Fatalf("active status = %#v err=%v", status, err) + } + result, err = Deactivate(Options{Home: home, Mount: mount, NativeRoot: nativeRoot}) + if err != nil { + t.Fatal(err) + } + if result.Active { + t.Fatalf("deactivation result = %#v", result) + } + assertFile(t, filepath.Join(home, "sessions", "2026", "07", "12", "active.jsonl"), "active\n") + assertFile(t, filepath.Join(home, "archived_sessions", "archived.jsonl"), "archived\n") + for _, directory := range []string{"sessions", "archived_sessions"} { + info, err := os.Lstat(filepath.Join(home, directory)) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + t.Fatalf("restored %s = %#v err=%v", directory, info, err) + } + } +} + +func TestActivateRejectsOrdinaryDirectoryThatLooksLikeMount(t *testing.T) { + root := t.TempDir() + home := filepath.Join(root, "home") + mount := filepath.Join(root, "mount") + nativeRoot := filepath.Join(home, "fold-native") + writeFixture(t, filepath.Join(home, "sessions", "active.jsonl"), "active\n") + writeFixture(t, filepath.Join(home, "archived_sessions", "archived.jsonl"), "archived\n") + for _, directory := range []string{"sessions", "archived_sessions"} { + if err := os.MkdirAll(filepath.Join(mount, directory), 0o700); err != nil { + t.Fatal(err) + } + } + + _, err := Activate(Options{ + Home: home, Mount: mount, NativeRoot: nativeRoot, + MountProbe: func(string) error { return os.ErrInvalid }, + }) + if err == nil { + t.Fatal("activation must reject an ordinary directory even when it has canonical subdirectories") + } + assertFile(t, filepath.Join(home, "sessions", "active.jsonl"), "active\n") + assertFile(t, filepath.Join(home, "archived_sessions", "archived.jsonl"), "archived\n") +} + +func TestActiveNamespaceNormalizesDesktopMountAliasesInStateDatabase(t *testing.T) { + root := t.TempDir() + home := filepath.Join(root, "home") + mount := filepath.Join(home, "fold-fs") + nativeRoot := filepath.Join(home, "fold-native") + activeRoute := filepath.Join(home, "sessions", "2026", "07", "13", "rollout-session.jsonl") + writeFixture(t, activeRoute, "active\n") + if err := os.MkdirAll(filepath.Join(home, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + for _, directory := range sessionDirectories { + if err := os.MkdirAll(filepath.Join(mount, directory), 0o700); err != nil { + t.Fatal(err) + } + } + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`create table threads (id text primary key, rollout_path text not null); insert into threads values ('session', ?)`, activeRoute); err != nil { + _ = db.Close() + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + + options := Options{Home: home, Mount: mount, NativeRoot: nativeRoot, MountProbe: healthyMountProbe} + if _, err := Activate(options); err != nil { + t.Fatal(err) + } + db, err = sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + mountAlias := filepath.Join(mount, "sessions", "2026", "07", "13", "rollout-session.jsonl") + if _, err := db.Exec(`update threads set rollout_path = ? where id = 'session'`, mountAlias); err != nil { + _ = db.Close() + t.Fatal(err) + } + var normalized string + if err := db.QueryRow(`select rollout_path from threads where id = 'session'`).Scan(&normalized); err != nil { + _ = db.Close() + t.Fatal(err) + } + if filepath.Clean(normalized) != filepath.Clean(activeRoute) { + _ = db.Close() + t.Fatalf("normalized route = %q, want %q", normalized, activeRoute) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + + if _, err := Deactivate(options); err != nil { + t.Fatal(err) + } + db, err = sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var triggers int + if err := db.QueryRow(`select count(*) from sqlite_master where type = 'trigger' and name like 'codexfold_normalize_rollout_path_%'`).Scan(&triggers); err != nil { + t.Fatal(err) + } + if triggers != 0 { + t.Fatalf("route normalization triggers remained after deactivation: %d", triggers) + } +} + +func TestRouteGuardNormalizesMountAliasesWithUnicodePaths(t *testing.T) { + home := filepath.Join(t.TempDir(), "用户", ".codex") + mount := filepath.Join(home, "fold-fs") + if err := os.MkdirAll(home, 0o700); err != nil { + t.Fatal(err) + } + createStateDatabase(t, home) + options := Options{Home: home, Mount: mount, NativeRoot: filepath.Join(home, "fold-native")} + if err := installRouteGuard(options); err != nil { + t.Fatal(err) + } + database, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + mountAlias := filepath.Join(mount, "archived_sessions", "rollout-session.jsonl") + if _, err := database.Exec(`insert into threads values ('session', ?)`, mountAlias); err != nil { + t.Fatal(err) + } + var normalized string + if err := database.QueryRow(`select rollout_path from threads where id = 'session'`).Scan(&normalized); err != nil { + t.Fatal(err) + } + want := filepath.Join(home, "archived_sessions", "rollout-session.jsonl") + if filepath.Clean(normalized) != filepath.Clean(want) { + t.Fatalf("normalized Unicode route = %q, want %q", normalized, want) + } +} + +func TestRecoverRollsBackInterruptedActivation(t *testing.T) { + root := t.TempDir() + home := filepath.Join(root, "home") + mount := filepath.Join(root, "mount") + nativeRoot := filepath.Join(home, "fold-native") + writeFixture(t, filepath.Join(home, "sessions", "active.jsonl"), "active\n") + writeFixture(t, filepath.Join(home, "archived_sessions", "archived.jsonl"), "archived\n") + if err := os.MkdirAll(nativeRoot, 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(filepath.Join(home, "sessions"), filepath.Join(nativeRoot, "sessions")); err != nil { + t.Fatal(err) + } + options := Options{Home: home, Mount: mount, NativeRoot: nativeRoot} + if err := writeJournal(options, journal{Version: 1, Action: actionActivate}); err != nil { + t.Fatal(err) + } + + result, err := Recover(options) + if err != nil { + t.Fatal(err) + } + if result.Active || !result.Recovered { + t.Fatalf("recovery result = %#v", result) + } + assertFile(t, filepath.Join(home, "sessions", "active.jsonl"), "active\n") + assertFile(t, filepath.Join(home, "archived_sessions", "archived.jsonl"), "archived\n") + if _, err := os.Stat(journalPath(options)); !os.IsNotExist(err) { + t.Fatalf("journal remained after recovery: %v", err) + } +} + +func writeFixture(t *testing.T, path string, contents string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } +} + +func assertFile(t *testing.T, path string, want string) { + t.Helper() + got, err := os.ReadFile(path) + if err != nil || string(got) != want { + t.Fatalf("file %s = %q err=%v", path, got, err) + } +} + +func assertLink(t *testing.T, path string, want string) { + t.Helper() + got, err := os.Readlink(path) + if err != nil || filepath.Clean(got) != filepath.Clean(want) { + t.Fatalf("link %s = %q err=%v", path, got, err) + } +} + +func healthyMountProbe(string) error { return nil } + +func createStateDatabase(t *testing.T, home string) { + t.Helper() + database, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := database.Exec(`create table threads (id text primary key, rollout_path text not null)`); err != nil { + _ = database.Close() + t.Fatal(err) + } + if err := database.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/sessionns/routing_guard.go b/internal/sessionns/routing_guard.go new file mode 100644 index 0000000..6a94797 --- /dev/null +++ b/internal/sessionns/routing_guard.go @@ -0,0 +1,125 @@ +package sessionns + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + _ "modernc.org/sqlite" +) + +const ( + routeInsertTrigger = "codexfold_normalize_rollout_path_insert" + routeUpdateTrigger = "codexfold_normalize_rollout_path_update" +) + +func installRouteGuard(options Options) error { + return updateRouteGuard(options, true) +} + +func removeRouteGuard(options Options) error { + return updateRouteGuard(options, false) +} + +func updateRouteGuard(options Options, install bool) error { + databasePath := filepath.Join(options.Home, "state_5.sqlite") + if _, err := os.Stat(databasePath); err != nil { + if !install && errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("locate Codex state database: %w", err) + } + database, err := sql.Open("sqlite", databasePath) + if err != nil { + return fmt.Errorf("open Codex state database: %w", err) + } + defer database.Close() + connection, err := database.Conn(context.Background()) + if err != nil { + return err + } + defer connection.Close() + if _, err := connection.ExecContext(context.Background(), `pragma busy_timeout = 10000`); err != nil { + return err + } + if _, err := connection.ExecContext(context.Background(), `begin immediate`); err != nil { + return fmt.Errorf("begin route guard transaction: %w", err) + } + committed := false + defer func() { + if !committed { + _, _ = connection.ExecContext(context.Background(), `rollback`) + } + }() + for _, name := range []string{routeInsertTrigger, routeUpdateTrigger} { + if _, err := connection.ExecContext(context.Background(), `drop trigger if exists `+name); err != nil { + return err + } + } + if install { + for _, statement := range routeGuardTriggerStatements(options) { + if _, err := connection.ExecContext(context.Background(), statement); err != nil { + return fmt.Errorf("install Codex route guard: %w", err) + } + } + } + if _, err := connection.ExecContext(context.Background(), normalizeExistingRoutesStatement(options)); err != nil { + return fmt.Errorf("normalize existing Codex routes: %w", err) + } + if _, err := connection.ExecContext(context.Background(), `commit`); err != nil { + return fmt.Errorf("commit route guard transaction: %w", err) + } + committed = true + return nil +} + +func routeGuardTriggerStatements(options Options) []string { + body := routeGuardBody(options) + condition := routeGuardCondition(options, "NEW.rollout_path") + return []string{ + fmt.Sprintf(`create trigger %s after insert on threads when %s begin %s end`, routeInsertTrigger, condition, body), + fmt.Sprintf(`create trigger %s after update of rollout_path on threads when %s begin %s end`, routeUpdateTrigger, condition, body), + } +} + +func routeGuardBody(options Options) string { + return fmt.Sprintf(`update threads set rollout_path = %s where id = NEW.id;`, routeGuardCase(options, "NEW.rollout_path")) +} + +func normalizeExistingRoutesStatement(options Options) string { + return fmt.Sprintf(`update threads set rollout_path = %s where %s`, routeGuardCase(options, "rollout_path"), routeGuardCondition(options, "rollout_path")) +} + +func routeGuardCase(options Options, value string) string { + activeMount := filepath.Join(options.Mount, "sessions") + string(filepath.Separator) + archiveMount := filepath.Join(options.Mount, "archived_sessions") + string(filepath.Separator) + activeHome := filepath.Join(options.Home, "sessions") + string(filepath.Separator) + archiveHome := filepath.Join(options.Home, "archived_sessions") + string(filepath.Separator) + activeMountSQL := quoteSQLString(activeMount) + archiveMountSQL := quoteSQLString(archiveMount) + return fmt.Sprintf( + `case when substr(%s, 1, length(%s)) = %s then %s || substr(%s, length(%s) + 1) else %s || substr(%s, length(%s) + 1) end`, + value, activeMountSQL, activeMountSQL, quoteSQLString(activeHome), value, activeMountSQL, + quoteSQLString(archiveHome), value, archiveMountSQL, + ) +} + +func routeGuardCondition(options Options, value string) string { + activeMount := filepath.Join(options.Mount, "sessions") + string(filepath.Separator) + archiveMount := filepath.Join(options.Mount, "archived_sessions") + string(filepath.Separator) + activeMountSQL := quoteSQLString(activeMount) + archiveMountSQL := quoteSQLString(archiveMount) + return fmt.Sprintf( + `substr(%s, 1, length(%s)) = %s or substr(%s, 1, length(%s)) = %s`, + value, activeMountSQL, activeMountSQL, + value, archiveMountSQL, archiveMountSQL, + ) +} + +func quoteSQLString(value string) string { + return "'" + strings.ReplaceAll(value, "'", "''") + "'" +} diff --git a/scripts/activate-canonical-after-codex-exit.sh b/scripts/activate-canonical-after-codex-exit.sh new file mode 100755 index 0000000..3c81789 --- /dev/null +++ b/scripts/activate-canonical-after-codex-exit.sh @@ -0,0 +1,114 @@ +#!/bin/zsh +set -euo pipefail + +export PATH="/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin" + +CODEX_HOME="${1:?Codex home is required}" +STORE="${2:?CodexFold store is required}" +MOUNT="${3:?mount path is required}" +NATIVE_ROOT="${4:?native root is required}" +BIN="${5:?CodexFold binary is required}" +REOPEN_APP="${6:-1}" +CRITICAL_IDS_FILE="${7:-${CODEXFOLD_CRITICAL_IDS_FILE:-}}" +RUN_ROOT="${STORE}/activation/canonical-$(date '+%Y%m%d-%H%M%S')" + +mkdir -p "${RUN_ROOT}" +exec >"${RUN_ROOT}/run.log" 2>&1 + +activated=0 +finish() { + status=$? + if (( status != 0 )); then + if (( activated == 1 )); then + "${BIN}" fs namespace deactivate --apply \ + --codex-home "${CODEX_HOME}" --mount "${MOUNT}" --native-root "${NATIVE_ROOT}" || true + fi + date '+%Y-%m-%dT%H:%M:%S%z' >"${RUN_ROOT}/FAILED" + fi + if [[ "${REOPEN_APP}" == "1" ]]; then + open -a /Applications/ChatGPT.app || true + fi + exit "${status}" +} +trap finish EXIT + +codex_running() { + pgrep -f '/Applications/ChatGPT.app/Contents/MacOS/ChatGPT($| )' >/dev/null 2>&1 || + pgrep -f '/Applications/ChatGPT.app/Contents/Resources/codex .*app-server' >/dev/null 2>&1 || + pgrep -f '/opt/homebrew/(Cellar/codex/[^/]+/bin|bin)/codex($| )' >/dev/null 2>&1 +} + +echo "waiting for Codex Desktop and CLI to exit" +while codex_running; do + sleep 2 +done +for _ in 1 2 3; do + sleep 1 + if codex_running; then + while codex_running; do + sleep 2 + done + fi +done + +service_status="$(${BIN} fs service status --json)" +jq -e '.daemon_running == true and .mount_healthy == true' <<<"${service_status}" >/dev/null +compatibility="$(${BIN} fs compatibility --codex-home "${CODEX_HOME}" --store "${STORE}" --json)" +jq -e '.evaluation.approved == true and .evaluation.quarantine == false' <<<"${compatibility}" >/dev/null + +managed_count="$(find "${STORE}/fs/sessions" -type f -name state.json 2>/dev/null | wc -l | tr -d ' ')" +[[ "${managed_count}" == "0" ]] +fold_route_count="$(sqlite3 "${CODEX_HOME}/state_5.sqlite" "select count(*) from threads where rollout_path like '${MOUNT}/%';")" +[[ "${fold_route_count}" == "0" ]] + +snapshot_tree() { + output="$1" + ( + cd "${CODEX_HOME}" + find sessions archived_sessions -type f ! -name '._*' -print0 | sort -z | xargs -0 stat -f '%N\t%z' + ) >"${output}" +} + +snapshot_critical() { + output="$1" + : >"${output}" + [[ -z "${CRITICAL_IDS_FILE}" ]] && return 0 + [[ -f "${CRITICAL_IDS_FILE}" ]] + while IFS= read -r id || [[ -n "${id}" ]]; do + [[ -z "${id}" || "${id}" == \#* ]] && continue + grep -Eq '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' <<<"${id}" + rollout="$(find "${CODEX_HOME}/sessions" "${CODEX_HOME}/archived_sessions" -type f -name "rollout-*-${id}.jsonl" ! -name '._*' -print -quit)" + [[ -n "${rollout}" ]] + digest="$(shasum -a 256 "${rollout}" | awk '{print $1}')" + printf '%s\t%s\n' "${id}" "${digest}" >>"${output}" + done <"${CRITICAL_IDS_FILE}" +} + +snapshot_tree "${RUN_ROOT}/tree.before" +snapshot_critical "${RUN_ROOT}/critical.before" + +"${BIN}" fs namespace activate --apply \ + --codex-home "${CODEX_HOME}" --mount "${MOUNT}" --native-root "${NATIVE_ROOT}" --json \ + >"${RUN_ROOT}/activate.json" +activated=1 + +[[ "$(readlink "${CODEX_HOME}/sessions")" == "${MOUNT}/sessions" ]] +[[ "$(readlink "${CODEX_HOME}/archived_sessions")" == "${MOUNT}/archived_sessions" ]] +trigger_count="$(sqlite3 "${CODEX_HOME}/state_5.sqlite" "select count(*) from sqlite_master where type='trigger' and name like 'codexfold_normalize_rollout_path_%';")" +[[ "${trigger_count}" == "2" ]] + +snapshot_tree "${RUN_ROOT}/tree.after" +snapshot_critical "${RUN_ROOT}/critical.after" +diff -u "${RUN_ROOT}/tree.before" "${RUN_ROOT}/tree.after" +diff -u "${RUN_ROOT}/critical.before" "${RUN_ROOT}/critical.after" + +service_status="$(${BIN} fs service status --json)" +jq -e '.daemon_running == true and .mount_healthy == true' <<<"${service_status}" >/dev/null +namespace_status="$(${BIN} fs namespace status --codex-home "${CODEX_HOME}" --mount "${MOUNT}" --native-root "${NATIVE_ROOT}" --json)" +jq -e '.active == true' <<<"${namespace_status}" >/dev/null + +date '+%Y-%m-%dT%H:%M:%S%z' >"${RUN_ROOT}/COMPLETE" +trap - EXIT +if [[ "${REOPEN_APP}" == "1" ]]; then + open -a /Applications/ChatGPT.app +fi diff --git a/scripts/prepare-isolated-codex-home.sh b/scripts/prepare-isolated-codex-home.sh new file mode 100755 index 0000000..b6f4b96 --- /dev/null +++ b/scripts/prepare-isolated-codex-home.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + printf 'usage: %s SOURCE_CODEX_HOME TARGET_CODEX_HOME\n' "$0" >&2 + exit 2 +fi + +source_home=$(cd "$1" && pwd) +target_home=$2 + +if [[ "$source_home" == "$target_home" ]]; then + echo "source and target CODEX_HOME must differ" >&2 + exit 2 +fi +if [[ ! -f "$source_home/config.toml" || ! -f "$source_home/auth.json" ]]; then + echo "source CODEX_HOME must contain config.toml and auth.json" >&2 + exit 2 +fi +if [[ -e "$target_home" ]]; then + echo "target CODEX_HOME already exists: $target_home" >&2 + exit 2 +fi + +mkdir -p "$target_home" +chmod 700 "$target_home" + +# Keep provider/auth state byte-identical; only the home directory around it is isolated. +for name in config.toml auth.json; do + cp -p "$source_home/$name" "$target_home/$name" + chmod 600 "$target_home/$name" +done +if [[ -f "$source_home/models_cache.json" ]]; then + cp -p "$source_home/models_cache.json" "$target_home/models_cache.json" + chmod 600 "$target_home/models_cache.json" +fi + +mkdir -p "$target_home/sessions" "$target_home/archived_sessions" + +# These assets are immutable inputs for the canary. APFS clone avoids duplicating their +# contents while copy-on-write keeps a canary update from touching the real home. +for name in plugins skills vendor_sources computer-use; do + if [[ -e "$source_home/$name" ]]; then + cp -cR "$source_home/$name" "$target_home/$name" + fi +done + +printf 'prepared isolated CODEX_HOME: %s\n' "$target_home" diff --git a/scripts/tests/test-prepare-isolated-codex-home.sh b/scripts/tests/test-prepare-isolated-codex-home.sh new file mode 100755 index 0000000..96605f5 --- /dev/null +++ b/scripts/tests/test-prepare-isolated-codex-home.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PREPARE="$ROOT_DIR/scripts/prepare-isolated-codex-home.sh" +TEST_ROOT="$(mktemp -d)" +trap 'rm -rf "$TEST_ROOT"' EXIT + +source_home="$TEST_ROOT/source" +target_home="$TEST_ROOT/target" +mkdir -p "$source_home/plugins/cache/example" "$source_home/skills/example" +printf 'model_provider = "main"\n[model_providers.main]\nname = "third-party"\nbase_url = "https://third-party.example/v1"\n' > "$source_home/config.toml" +printf '{"access_token":"test-token"}\n' > "$source_home/auth.json" +printf '{"client_version":"test","models":[]}\n' > "$source_home/models_cache.json" +printf 'plugin-source\n' > "$source_home/plugins/cache/example/state" +printf 'skill-source\n' > "$source_home/skills/example/SKILL.md" +chmod 700 "$source_home" +chmod 600 "$source_home/config.toml" "$source_home/auth.json" + +"$PREPARE" "$source_home" "$target_home" >/dev/null + +cmp -s "$source_home/config.toml" "$target_home/config.toml" +cmp -s "$source_home/auth.json" "$target_home/auth.json" +cmp -s "$source_home/models_cache.json" "$target_home/models_cache.json" +[[ "$(stat -f '%Lp' "$target_home/config.toml")" == 600 ]] +[[ "$(stat -f '%Lp' "$target_home/auth.json")" == 600 ]] +[[ "$(stat -f '%Lp' "$target_home/models_cache.json")" == 600 ]] +[[ -d "$target_home/plugins" && ! -L "$target_home/plugins" ]] +printf 'plugin-target\n' > "$target_home/plugins/cache/example/state" +grep -Fqx 'plugin-source' "$source_home/plugins/cache/example/state" + +echo "PASS: isolated CODEX_HOME preserves current provider/auth and clones mutable canary assets" diff --git a/scripts/tests/test-public-scripts-sanitized.sh b/scripts/tests/test-public-scripts-sanitized.sh new file mode 100755 index 0000000..d49407c --- /dev/null +++ b/scripts/tests/test-public-scripts-sanitized.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "$0")/../.." && pwd) + +if rg -n '/Users/jstar|/Volumes/JSData|019[0-9a-f]{5,}|mcxin|jstarctl' "$repo_root/scripts" --glob '!**/test-public-scripts-sanitized.sh'; then + echo "public scripts contain private paths, session IDs, or control-plane names" >&2 + exit 1 +fi + +grep -q 'CRITICAL_IDS_FILE' "$repo_root/scripts/activate-canonical-after-codex-exit.sh" + +echo "PASS: public scripts are sanitized and critical session checks are parameterized" From 59d169c014b59299705f662e709e065a67fc9a28 Mon Sep 17 00:00:00 2001 From: King Star Date: Mon, 13 Jul 2026 17:04:03 +0800 Subject: [PATCH 19/33] fix: follow canonical session symlinks during activation --- .../activate-canonical-after-codex-exit.sh | 4 ++-- ...est-activate-canonical-symlink-snapshot.sh | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100755 scripts/tests/test-activate-canonical-symlink-snapshot.sh diff --git a/scripts/activate-canonical-after-codex-exit.sh b/scripts/activate-canonical-after-codex-exit.sh index 3c81789..0cefebf 100755 --- a/scripts/activate-canonical-after-codex-exit.sh +++ b/scripts/activate-canonical-after-codex-exit.sh @@ -65,7 +65,7 @@ snapshot_tree() { output="$1" ( cd "${CODEX_HOME}" - find sessions archived_sessions -type f ! -name '._*' -print0 | sort -z | xargs -0 stat -f '%N\t%z' + find -H sessions archived_sessions -type f ! -name '._*' -print0 | sort -z | xargs -0 stat -f '%N\t%z' ) >"${output}" } @@ -77,7 +77,7 @@ snapshot_critical() { while IFS= read -r id || [[ -n "${id}" ]]; do [[ -z "${id}" || "${id}" == \#* ]] && continue grep -Eq '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' <<<"${id}" - rollout="$(find "${CODEX_HOME}/sessions" "${CODEX_HOME}/archived_sessions" -type f -name "rollout-*-${id}.jsonl" ! -name '._*' -print -quit)" + rollout="$(find -H "${CODEX_HOME}/sessions" "${CODEX_HOME}/archived_sessions" -type f -name "rollout-*-${id}.jsonl" ! -name '._*' -print -quit)" [[ -n "${rollout}" ]] digest="$(shasum -a 256 "${rollout}" | awk '{print $1}')" printf '%s\t%s\n' "${id}" "${digest}" >>"${output}" diff --git a/scripts/tests/test-activate-canonical-symlink-snapshot.sh b/scripts/tests/test-activate-canonical-symlink-snapshot.sh new file mode 100755 index 0000000..4b5e07d --- /dev/null +++ b/scripts/tests/test-activate-canonical-symlink-snapshot.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "$0")/../.." && pwd) +script="$repo_root/scripts/activate-canonical-after-codex-exit.sh" + +grep -Fq 'find -H sessions archived_sessions' "$script" +grep -Fq 'find -H "${CODEX_HOME}/sessions" "${CODEX_HOME}/archived_sessions"' "$script" + +root=$(mktemp -d) +trap 'rm -rf "$root"' EXIT +mkdir -p "$root/native/sessions/2026/07/13" "$root/native/archived_sessions" +touch "$root/native/sessions/2026/07/13/rollout.jsonl" +ln -s "$root/native/sessions" "$root/sessions" +ln -s "$root/native/archived_sessions" "$root/archived_sessions" + +count=$(cd "$root" && find -H sessions archived_sessions -type f | wc -l | tr -d ' ') +[[ "$count" == "1" ]] + +echo "PASS: activation snapshots traverse canonical session symlinks" From f7c2e29762536e7052b7d302a3fbb812e32477ea Mon Sep 17 00:00:00 2001 From: King Star Date: Mon, 13 Jul 2026 17:07:46 +0800 Subject: [PATCH 20/33] fix: ignore orphaned desktop app servers during activation --- scripts/activate-canonical-after-codex-exit.sh | 1 - scripts/tests/test-activate-canonical-symlink-snapshot.sh | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/activate-canonical-after-codex-exit.sh b/scripts/activate-canonical-after-codex-exit.sh index 0cefebf..6e0578b 100755 --- a/scripts/activate-canonical-after-codex-exit.sh +++ b/scripts/activate-canonical-after-codex-exit.sh @@ -34,7 +34,6 @@ trap finish EXIT codex_running() { pgrep -f '/Applications/ChatGPT.app/Contents/MacOS/ChatGPT($| )' >/dev/null 2>&1 || - pgrep -f '/Applications/ChatGPT.app/Contents/Resources/codex .*app-server' >/dev/null 2>&1 || pgrep -f '/opt/homebrew/(Cellar/codex/[^/]+/bin|bin)/codex($| )' >/dev/null 2>&1 } diff --git a/scripts/tests/test-activate-canonical-symlink-snapshot.sh b/scripts/tests/test-activate-canonical-symlink-snapshot.sh index 4b5e07d..8bf3497 100755 --- a/scripts/tests/test-activate-canonical-symlink-snapshot.sh +++ b/scripts/tests/test-activate-canonical-symlink-snapshot.sh @@ -6,6 +6,10 @@ script="$repo_root/scripts/activate-canonical-after-codex-exit.sh" grep -Fq 'find -H sessions archived_sessions' "$script" grep -Fq 'find -H "${CODEX_HOME}/sessions" "${CODEX_HOME}/archived_sessions"' "$script" +if grep -Fq '/Applications/ChatGPT.app/Contents/Resources/codex .*app-server' "$script"; then + echo "activation must not wait forever on orphaned Desktop app-server processes" >&2 + exit 1 +fi root=$(mktemp -d) trap 'rm -rf "$root"' EXIT From f7cf369cfde4d5812fc1bb4813b214893bbc911e Mon Sep 17 00:00:00 2001 From: King Star Date: Mon, 13 Jul 2026 17:13:20 +0800 Subject: [PATCH 21/33] fix: drain real-home app servers before activation --- .../activate-canonical-after-codex-exit.sh | 34 +++++++++++++++++-- ...est-activate-canonical-symlink-snapshot.sh | 12 ++++--- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/scripts/activate-canonical-after-codex-exit.sh b/scripts/activate-canonical-after-codex-exit.sh index 6e0578b..7364038 100755 --- a/scripts/activate-canonical-after-codex-exit.sh +++ b/scripts/activate-canonical-after-codex-exit.sh @@ -20,8 +20,12 @@ finish() { status=$? if (( status != 0 )); then if (( activated == 1 )); then - "${BIN}" fs namespace deactivate --apply \ - --codex-home "${CODEX_HOME}" --mount "${MOUNT}" --native-root "${NATIVE_ROOT}" || true + "${BIN}" fs service stop --apply || true + if "${BIN}" fs namespace deactivate --apply \ + --codex-home "${CODEX_HOME}" --mount "${MOUNT}" --native-root "${NATIVE_ROOT}"; then + "${BIN}" fs service start --apply \ + --codex-home "${CODEX_HOME}" --mount "${MOUNT}" || true + fi fi date '+%Y-%m-%dT%H:%M:%S%z' >"${RUN_ROOT}/FAILED" fi @@ -37,6 +41,20 @@ codex_running() { pgrep -f '/opt/homebrew/(Cellar/codex/[^/]+/bin|bin)/codex($| )' >/dev/null 2>&1 } +app_servers_running() { + local pid command_line app_home + while IFS= read -r pid; do + [[ -z "${pid}" ]] && continue + command_line="$(ps eww -p "${pid}" -o command= 2>/dev/null || true)" + [[ -z "${command_line}" ]] && continue + app_home="$(sed -n 's/.* CODEX_HOME=\([^ ]*\).*/\1/p' <<<"${command_line}")" + if [[ -z "${app_home}" || "${app_home}" == "${CODEX_HOME}" ]]; then + return 0 + fi + done < <(pgrep -f '/Applications/ChatGPT.app/Contents/Resources/codex .*app-server' 2>/dev/null || true) + return 1 +} + echo "waiting for Codex Desktop and CLI to exit" while codex_running; do sleep 2 @@ -49,6 +67,18 @@ for _ in 1 2 3; do done fi done +drained=0 +for _ in {1..30}; do + if ! app_servers_running; then + drained=1 + break + fi + sleep 1 +done +if (( drained == 0 )); then + echo "real-home Codex app servers did not drain" + exit 1 +fi service_status="$(${BIN} fs service status --json)" jq -e '.daemon_running == true and .mount_healthy == true' <<<"${service_status}" >/dev/null diff --git a/scripts/tests/test-activate-canonical-symlink-snapshot.sh b/scripts/tests/test-activate-canonical-symlink-snapshot.sh index 8bf3497..71aeb70 100755 --- a/scripts/tests/test-activate-canonical-symlink-snapshot.sh +++ b/scripts/tests/test-activate-canonical-symlink-snapshot.sh @@ -6,10 +6,14 @@ script="$repo_root/scripts/activate-canonical-after-codex-exit.sh" grep -Fq 'find -H sessions archived_sessions' "$script" grep -Fq 'find -H "${CODEX_HOME}/sessions" "${CODEX_HOME}/archived_sessions"' "$script" -if grep -Fq '/Applications/ChatGPT.app/Contents/Resources/codex .*app-server' "$script"; then - echo "activation must not wait forever on orphaned Desktop app-server processes" >&2 - exit 1 -fi +grep -Fq 'app_servers_running()' "$script" +grep -Fq 'real-home Codex app servers did not drain' "$script" + +stop_line=$(grep -n 'fs service stop --apply' "$script" | head -n 1 | cut -d: -f1) +deactivate_line=$(grep -n 'fs namespace deactivate --apply' "$script" | head -n 1 | cut -d: -f1) +start_line=$(grep -n 'fs service start --apply' "$script" | head -n 1 | cut -d: -f1) +[[ -n "$stop_line" && -n "$deactivate_line" && -n "$start_line" ]] +(( stop_line < deactivate_line && deactivate_line < start_line )) root=$(mktemp -d) trap 'rm -rf "$root"' EXIT From 7ce4940e7fa7ae5396ee8c4f900b8a7c3dbbdd41 Mon Sep 17 00:00:00 2001 From: King Star Date: Tue, 14 Jul 2026 14:15:44 +0800 Subject: [PATCH 22/33] fix: make canonical session cutovers fail closed --- docs/validation-macos-canary.md | 8 +++ internal/cli/fs.go | 11 +++ internal/cli/fs_test.go | 68 ++++++++++++++++--- internal/mountfs/fuse_integration_test.go | 38 +++++++++++ .../activate-canonical-after-codex-exit.sh | 6 +- ...est-activate-canonical-symlink-snapshot.sh | 1 + 6 files changed, 123 insertions(+), 9 deletions(-) diff --git a/docs/validation-macos-canary.md b/docs/validation-macos-canary.md index 42538af..efe878a 100644 --- a/docs/validation-macos-canary.md +++ b/docs/validation-macos-canary.md @@ -4,6 +4,14 @@ The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have passed for read, append, resume, fork, child-session enrollment, canonical archive/unarchive moves, launchd restart, rollback, namespace deactivation, and unknown-version quarantine. The project remains at `fs-engine-preview` because the user Codex home is intentionally not enrolled and sleep or full host restart has not been exercised. +Additional failure-containment evidence on 2026-07-14: + +- Exact contracts were imported from real FUSE traces for PATH CLI `0.144.3` and Desktop `26.707.71524+5263`. +- Canonical migration now verifies the mounted managed target before removing the native directory entry. A clean first migration passed without a retry. +- A real Desktop canary preserved the exact 79,067-byte, 16-record source prefix, appended an 8,414-byte, 12-record managed delta, and rolled back to their exact 87,481-byte concatenation. A subsequent native Desktop turn appended 5,590 bytes and 9 records. The final 93,071-byte, 37-record JSONL parsed completely and preserved byte order. +- Rollback now holds an exclusive writer lease across materialization and state retirement. A live FUSE writer and a real Desktop app-server both caused rollback to fail closed; after every writer drained, rollback preserved the exact visible SHA-256 and a new Desktop turn persisted to the native JSONL. +- The canonical activation gate hashes every rollout before and after namespace activation. File size alone is no longer accepted as full-history evidence. + Additional failure-containment evidence on 2026-07-13: - The mount backing directory rejects symlinks and any ordinary files before the host starts. diff --git a/internal/cli/fs.go b/internal/cli/fs.go index 45f4a4e..2351102 100644 --- a/internal/cli/fs.go +++ b/internal/cli/fs.go @@ -581,6 +581,9 @@ func newFSMigrateCommand() *cobra.Command { if err != nil || filepath.Clean(current.RolloutPath) != filepath.Clean(session.RolloutPath) { return rollbackCanonicalMigration(errors.New("canonical Codex route changed during migration")) } + if _, err := waitForTargetMatch(command.Context(), target, vfs.NativeFile{Bytes: shadow.Bytes, SHA256: shadow.SHA256}, mountWait); err != nil { + return rollbackCanonicalMigration(fmt.Errorf("verify managed target before canonical cutover: %w", err)) + } if err := finalizeCanonicalSnapshotSource(canonicalSource, native); err != nil { return rollbackCanonicalMigration(err) } @@ -708,6 +711,14 @@ func newFSRollbackCommand() *cobra.Command { return err } defer resolver.Close() + rollbackLease, err := managed.OpenWriter() + if errors.Is(err, vfs.ErrWriterBusy) { + return errors.New("cannot rollback while the session has an active writer") + } + if err != nil { + return err + } + defer rollbackLease.Close() target, err := managed.MaterializeCurrent(command.Context(), filepath.Clean(targetPath), true) if err != nil { return err diff --git a/internal/cli/fs_test.go b/internal/cli/fs_test.go index d583f69..4385dc2 100644 --- a/internal/cli/fs_test.go +++ b/internal/cli/fs_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "testing" "time" @@ -618,12 +619,6 @@ func TestFSMigrateCanonicalKeepsCodexRouteAndHidesRetainedSnapshot(t *testing.T) } mount := filepath.Join(home, "fold-fs") target := filepath.Join(mount, "archived_sessions", filepath.Base(route)) - if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(target, source, 0o600); err != nil { - t.Fatal(err) - } cliPath := approvedCLIContract(t, storeDir, "1.2.3") acknowledged := make(chan error, 1) go func() { @@ -632,7 +627,19 @@ func TestFSMigrateCanonicalKeepsCodexRouteAndHidesRetainedSnapshot(t *testing.T) for time.Now().Before(deadline) { state, err := vfs.LoadSessionState(statePath) if err == nil { - acknowledged <- writeMountAcknowledgement(storeDir, "session", state.Generation, "/archived_sessions/"+filepath.Base(route)) + if err := writeMountAcknowledgement(storeDir, "session", state.Generation, "/archived_sessions/"+filepath.Base(route)); err != nil { + acknowledged <- err + return + } + time.Sleep(100 * time.Millisecond) + if _, err := os.Stat(nativePath); err != nil { + acknowledged <- errors.New("canonical source was hidden before mounted target verification") + return + } + if err := os.MkdirAll(filepath.Dir(target), 0o700); err == nil { + err = os.WriteFile(target, source, 0o600) + } + acknowledged <- err return } time.Sleep(10 * time.Millisecond) @@ -642,7 +649,7 @@ func TestFSMigrateCanonicalKeepsCodexRouteAndHidesRetainedSnapshot(t *testing.T) executeFS(t, []string{ "fs", "migrate", "session", "--apply", "--canonical-namespace", "--codex-home", home, "--store", storeDir, "--mount", mount, "--native-root", nativeRoot, - "--cli", cliPath, "--desktop-app", "none", + "--cli", cliPath, "--desktop-app", "none", "--mount-wait", "500ms", }) if err := <-acknowledged; err != nil { t.Fatal(err) @@ -717,6 +724,51 @@ func TestFSRollbackUsesLatestVisibleBytesAfterVirtualAppend(t *testing.T) { } } +func TestFSRollbackRejectsActiveWriter(t *testing.T) { + home, storeDir, originalPath := fsFixture(t, true) + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + defer resolver.Close() + native, err := hashPath(originalPath) + if err != nil { + t.Fatal(err) + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: native, + }) + if err != nil { + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + defer writer.Close() + + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "rollback", "session", "--codex-home", home, "--store", storeDir, "--apply"}) + err = root.Execute() + if err == nil || !strings.Contains(err.Error(), "active writer") { + t.Fatalf("rollback error = %v, want active writer rejection", err) + } + if _, err := managedState(storeDir, "session"); err != nil { + t.Fatalf("active-writer rejection retired managed state: %v", err) + } + sessions, err := codex.LoadSessions(home) + if err != nil || len(sessions) != 1 || filepath.Clean(sessions[0].RolloutPath) != filepath.Clean(originalPath) { + t.Fatalf("active-writer rejection changed route: sessions=%#v err=%v", sessions, err) + } +} + func TestFSRollbackCanonicalRetiresManagedStateAndKeepsRoute(t *testing.T) { allowFixtureMount(t) home, storeDir, originalPath := fsFixture(t, true) diff --git a/internal/mountfs/fuse_integration_test.go b/internal/mountfs/fuse_integration_test.go index 8fbb689..7c09ead 100644 --- a/internal/mountfs/fuse_integration_test.go +++ b/internal/mountfs/fuse_integration_test.go @@ -152,6 +152,44 @@ func TestRealFuseMountNativeFileOperations(t *testing.T) { waitForRealUnmount(t, mountPoint) } +func TestRealFuseCanonicalNativeToManagedCutover(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := filepath.Join("archived_sessions", "rollout-cutover.jsonl") + nativePath := filepath.Join(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + source := []byte("{\"cutover\":true}\n") + if err := os.WriteFile(nativePath, source, 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealMount(t, mountPoint, filesystem) + target := filepath.Join(mountPoint, route) + + managed := mountSessionFixture(t, "cutover", source) + if err := filesystem.UpsertSessionAt("cutover", "/"+filepath.ToSlash(route), managed); err != nil { + t.Fatal(err) + } + if err := os.Remove(nativePath); err != nil { + t.Fatal(err) + } + waitForRealFile(t, target, source) + + stopMount() + waitForRealUnmount(t, mountPoint) +} + func TestRealFuseMountCanonicalManagedRename(t *testing.T) { if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") diff --git a/scripts/activate-canonical-after-codex-exit.sh b/scripts/activate-canonical-after-codex-exit.sh index 7364038..7115003 100755 --- a/scripts/activate-canonical-after-codex-exit.sh +++ b/scripts/activate-canonical-after-codex-exit.sh @@ -94,7 +94,11 @@ snapshot_tree() { output="$1" ( cd "${CODEX_HOME}" - find -H sessions archived_sessions -type f ! -name '._*' -print0 | sort -z | xargs -0 stat -f '%N\t%z' + while IFS= read -r -d '' rollout; do + size="$(stat -f '%z' "${rollout}")" + digest="$(shasum -a 256 "${rollout}" | awk '{print $1}')" + printf '%s\t%s\t%s\n' "${rollout}" "${size}" "${digest}" + done < <(find -H sessions archived_sessions -type f ! -name '._*' -print0 | sort -z) ) >"${output}" } diff --git a/scripts/tests/test-activate-canonical-symlink-snapshot.sh b/scripts/tests/test-activate-canonical-symlink-snapshot.sh index 71aeb70..a77fa65 100755 --- a/scripts/tests/test-activate-canonical-symlink-snapshot.sh +++ b/scripts/tests/test-activate-canonical-symlink-snapshot.sh @@ -6,6 +6,7 @@ script="$repo_root/scripts/activate-canonical-after-codex-exit.sh" grep -Fq 'find -H sessions archived_sessions' "$script" grep -Fq 'find -H "${CODEX_HOME}/sessions" "${CODEX_HOME}/archived_sessions"' "$script" +grep -Fq 'shasum -a 256 "${rollout}"' "$script" grep -Fq 'app_servers_running()' "$script" grep -Fq 'real-home Codex app servers did not drain' "$script" From 2724fe7696ee367294a67bd11acf6fa8d1d4c771 Mon Sep 17 00:00:00 2001 From: King Star Date: Tue, 14 Jul 2026 15:11:56 +0800 Subject: [PATCH 23/33] fix: serialize canonical migration cutovers --- internal/cli/fs.go | 3 +- internal/cli/fs_test.go | 96 +++++++++++++++++++ internal/vfs/session.go | 91 ++++++++++++++---- .../activate-canonical-after-codex-exit.sh | 6 +- ...est-activate-canonical-symlink-snapshot.sh | 50 +++++++++- 5 files changed, 220 insertions(+), 26 deletions(-) diff --git a/internal/cli/fs.go b/internal/cli/fs.go index 2351102..3e82143 100644 --- a/internal/cli/fs.go +++ b/internal/cli/fs.go @@ -565,10 +565,11 @@ func newFSMigrateCommand() *cobra.Command { } return cause } - managed, err := vfs.OpenSession(command.Context(), vfs.SessionOptions{Root: store, ManifestPath: fold.ManifestPath(store, session.ID), Manifest: manifest, Reader: resolver, NativeSnapshot: native}) + managed, migrationLease, err := vfs.OpenSessionWithWriter(command.Context(), vfs.SessionOptions{Root: store, ManifestPath: fold.ManifestPath(store, session.ID), Manifest: manifest, Reader: resolver, NativeSnapshot: native}) if err != nil { return rollbackCanonicalMigration(err) } + defer migrationLease.Close() if canonicalNamespace { if err := waitForMountAcknowledgement(command.Context(), store, session.ID, managed.State().Generation, canonicalRoute, mountWait); err != nil { return rollbackCanonicalMigration(fmt.Errorf("wait for canonical mount acknowledgement: %w", err)) diff --git a/internal/cli/fs_test.go b/internal/cli/fs_test.go index 4385dc2..184c449 100644 --- a/internal/cli/fs_test.go +++ b/internal/cli/fs_test.go @@ -671,6 +671,102 @@ func TestFSMigrateCanonicalKeepsCodexRouteAndHidesRetainedSnapshot(t *testing.T) } } +func TestFSMigrateCanonicalReservesWriterDuringCutover(t *testing.T) { + allowFixtureMount(t) + home := t.TempDir() + storeDir := filepath.Join(home, "fold-store") + route := filepath.Join(home, "archived_sessions", "rollout-session.jsonl") + if err := os.MkdirAll(filepath.Dir(route), 0o700); err != nil { + t.Fatal(err) + } + source := []byte("{\"type\":\"session_meta\"}\n{\"canonical\":true}\n") + if err := os.WriteFile(route, source, 0o600); err != nil { + t.Fatal(err) + } + writeStateFixture(t, home, route) + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`update threads set archived = 1, id = 'session' where id = 'fixture'`); err != nil { + _ = db.Close() + t.Fatal(err) + } + _ = db.Close() + if _, err := fold.Fold(context.Background(), codex.Session{ID: "session", RolloutPath: route, Archived: true}, fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { + t.Fatal(err) + } + if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { + t.Fatal(err) + } + + nativeRoot := filepath.Join(home, "fold-native") + nativePath := filepath.Join(nativeRoot, "archived_sessions", filepath.Base(route)) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(route, nativePath); err != nil { + t.Fatal(err) + } + mount := filepath.Join(home, "fold-fs") + target := filepath.Join(mount, "archived_sessions", filepath.Base(route)) + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + writerAttempt := make(chan error, 1) + releaseWriter := make(chan struct{}) + go func() { + statePath := filepath.Join(storeDir, "fs", "sessions", "session", "state.json") + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + state, stateErr := vfs.LoadSessionState(statePath) + if stateErr == nil { + if ackErr := writeMountAcknowledgement(storeDir, "session", state.Generation, "/archived_sessions/"+filepath.Base(route)); ackErr != nil { + writerAttempt <- ackErr + return + } + managed, resolver, openErr := openManagedSession(context.Background(), storeDir, state) + if openErr != nil { + writerAttempt <- openErr + return + } + writer, writerErr := managed.OpenWriter() + if err := os.MkdirAll(filepath.Dir(target), 0o700); err == nil { + err = os.WriteFile(target, source, 0o600) + } + writerAttempt <- writerErr + if writer != nil { + <-releaseWriter + _ = writer.Close() + } + _ = resolver.Close() + return + } + time.Sleep(5 * time.Millisecond) + } + writerAttempt <- errors.New("managed state was not created") + }() + + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "migrate", "session", "--apply", "--canonical-namespace", + "--codex-home", home, "--store", storeDir, "--mount", mount, "--native-root", nativeRoot, + "--cli", cliPath, "--desktop-app", "none", "--mount-wait", "500ms", + }) + migrateErr := root.Execute() + writerErr := <-writerAttempt + close(releaseWriter) + if migrateErr != nil { + t.Fatalf("canonical migration failed: %v", migrateErr) + } + if !errors.Is(writerErr, vfs.ErrWriterBusy) { + t.Fatalf("concurrent writer error = %v, want %v", writerErr, vfs.ErrWriterBusy) + } + if _, err := managedState(storeDir, "session"); err != nil { + t.Fatalf("canonical migration did not retain managed state: %v", err) + } +} + func TestFSRollbackUsesLatestVisibleBytesAfterVirtualAppend(t *testing.T) { allowFixtureMount(t) home, storeDir, nativePath := fsFixture(t, true) diff --git a/internal/vfs/session.go b/internal/vfs/session.go index a3a5e93..b3c540e 100644 --- a/internal/vfs/session.go +++ b/internal/vfs/session.go @@ -44,71 +44,112 @@ type VisibleInfo struct { var ErrWriterBusy = errors.New("session writer lease is already held") func OpenSession(ctx context.Context, options SessionOptions) (*Session, error) { + session, _, err := openSession(ctx, options, false) + return session, err +} + +func OpenSessionWithWriter(ctx context.Context, options SessionOptions) (*Session, *WriteHandle, error) { + return openSession(ctx, options, true) +} + +func openSession(ctx context.Context, options SessionOptions, reserveWriter bool) (*Session, *WriteHandle, error) { if err := ctx.Err(); err != nil { - return nil, err + return nil, nil, err } if options.Root == "" || options.ManifestPath == "" || !safeSessionID(options.Manifest.Session.ID) { - return nil, errors.New("session root, manifest path, and safe session ID are required") + return nil, nil, errors.New("session root, manifest path, and safe session ID are required") } view, err := NewView(options.Manifest, options.Reader) if err != nil { - return nil, err + return nil, nil, err } directory := filepath.Join(options.Root, "fs", "sessions", options.Manifest.Session.ID) if err := os.MkdirAll(directory, 0o700); err != nil { - return nil, fmt.Errorf("create virtual session directory: %w", err) + return nil, nil, fmt.Errorf("create virtual session directory: %w", err) } - statePath := filepath.Join(directory, "state.json") - if err := cleanupStaleWriterLease(filepath.Join(directory, "writer.lease")); err != nil { - return nil, err + leasePath := filepath.Join(directory, "writer.lease") + var reservedLease *os.File + if reserveWriter { + reservedLease, err = acquireWriterLease(leasePath) + if err != nil { + return nil, nil, err + } + } else if err := cleanupStaleWriterLease(leasePath); err != nil { + return nil, nil, err } + cleanupReservedLease := func() { + if reservedLease == nil { + return + } + _ = unlockWriterFile(reservedLease) + _ = reservedLease.Close() + } + statePath := filepath.Join(directory, "state.json") state, err := loadSessionState(statePath) if errors.Is(err, os.ErrNotExist) { if err := verifyNativeFile(options.NativeSnapshot); err != nil { - return nil, err + cleanupReservedLease() + return nil, nil, err } deltaPath := filepath.Join(directory, "delta.jsonl") delta, err := os.OpenFile(deltaPath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600) if err != nil { - return nil, fmt.Errorf("create session delta: %w", err) + cleanupReservedLease() + return nil, nil, fmt.Errorf("create session delta: %w", err) } if err := delta.Sync(); err != nil { _ = delta.Close() - return nil, fmt.Errorf("sync session delta: %w", err) + cleanupReservedLease() + return nil, nil, fmt.Errorf("sync session delta: %w", err) } if err := delta.Close(); err != nil { - return nil, fmt.Errorf("close session delta: %w", err) + cleanupReservedLease() + return nil, nil, fmt.Errorf("close session delta: %w", err) } state = SessionState{Version: sessionStateVersion, SessionID: options.Manifest.Session.ID, Generation: 1, ManifestPath: filepath.Clean(options.ManifestPath), BaseBytes: view.Size(), BaseSHA256: options.Manifest.Source.SHA256, DeltaPath: deltaPath, NativeSnapshot: options.NativeSnapshot} if err := writeSessionState(statePath, state); err != nil { - return nil, err + cleanupReservedLease() + return nil, nil, err } } else if err != nil { - return nil, err + cleanupReservedLease() + return nil, nil, err } else { if state.SessionID != options.Manifest.Session.ID || state.ManifestPath != filepath.Clean(options.ManifestPath) || state.BaseBytes != view.Size() || state.BaseSHA256 != options.Manifest.Source.SHA256 || state.NativeSnapshot != options.NativeSnapshot { - return nil, errors.New("persisted session state does not match the requested manifest") + cleanupReservedLease() + return nil, nil, errors.New("persisted session state does not match the requested manifest") } if !pathWithin(directory, state.DeltaPath) || (state.BackingPath != "" && !pathWithin(directory, state.BackingPath)) { - return nil, errors.New("persisted session state contains an unsafe data path") + cleanupReservedLease() + return nil, nil, errors.New("persisted session state contains an unsafe data path") } if _, err := os.Stat(state.DeltaPath); err != nil { - return nil, fmt.Errorf("stat session delta: %w", err) + cleanupReservedLease() + return nil, nil, fmt.Errorf("stat session delta: %w", err) } if state.BackingPath != "" { if _, err := os.Stat(state.BackingPath); err != nil { - return nil, fmt.Errorf("stat session backing: %w", err) + cleanupReservedLease() + return nil, nil, fmt.Errorf("stat session backing: %w", err) } } } session := &Session{state: state, statePath: statePath, directory: directory, view: view, readerLeases: make(map[uint64]int), beforeCOWPhase: options.BeforeCOWPhase} + var writer *WriteHandle + if reservedLease != nil { + session.writerOpen = true + writer = &WriteHandle{session: session, leasePath: leasePath, lease: reservedLease} + } if err := session.recover(ctx); err != nil { - return nil, err + if writer != nil { + _ = writer.Close() + } + return nil, nil, err } if recovered, err := loadSessionState(statePath); err == nil { session.state = recovered } - return session, nil + return session, writer, nil } func (s *Session) State() SessionState { @@ -189,6 +230,15 @@ func (s *Session) OpenWriter() (*WriteHandle, error) { return nil, ErrWriterBusy } leasePath := filepath.Join(s.directory, "writer.lease") + lease, err := acquireWriterLease(leasePath) + if err != nil { + return nil, err + } + s.writerOpen = true + return &WriteHandle{session: s, leasePath: leasePath, lease: lease}, nil +} + +func acquireWriterLease(leasePath string) (*os.File, error) { lease, err := os.OpenFile(leasePath, os.O_CREATE|os.O_RDWR, 0o600) if err != nil { return nil, fmt.Errorf("create writer lease: %w", err) @@ -217,8 +267,7 @@ func (s *Session) OpenWriter() (*WriteHandle, error) { _ = lease.Close() return nil, fmt.Errorf("sync writer lease: %w", err) } - s.writerOpen = true - return &WriteHandle{session: s, leasePath: leasePath, lease: lease}, nil + return lease, nil } func (s *Session) ensureBacking(ctx context.Context) (string, error) { diff --git a/scripts/activate-canonical-after-codex-exit.sh b/scripts/activate-canonical-after-codex-exit.sh index 7115003..bc58c96 100755 --- a/scripts/activate-canonical-after-codex-exit.sh +++ b/scripts/activate-canonical-after-codex-exit.sh @@ -17,8 +17,8 @@ exec >"${RUN_ROOT}/run.log" 2>&1 activated=0 finish() { - status=$? - if (( status != 0 )); then + exit_code=$? + if (( exit_code != 0 )); then if (( activated == 1 )); then "${BIN}" fs service stop --apply || true if "${BIN}" fs namespace deactivate --apply \ @@ -32,7 +32,7 @@ finish() { if [[ "${REOPEN_APP}" == "1" ]]; then open -a /Applications/ChatGPT.app || true fi - exit "${status}" + exit "${exit_code}" } trap finish EXIT diff --git a/scripts/tests/test-activate-canonical-symlink-snapshot.sh b/scripts/tests/test-activate-canonical-symlink-snapshot.sh index a77fa65..2a22efb 100755 --- a/scripts/tests/test-activate-canonical-symlink-snapshot.sh +++ b/scripts/tests/test-activate-canonical-symlink-snapshot.sh @@ -17,7 +17,8 @@ start_line=$(grep -n 'fs service start --apply' "$script" | head -n 1 | cut -d: (( stop_line < deactivate_line && deactivate_line < start_line )) root=$(mktemp -d) -trap 'rm -rf "$root"' EXIT +runtime_root="" +trap 'rm -rf "$root"; [[ -z "$runtime_root" ]] || rm -rf "$runtime_root"' EXIT mkdir -p "$root/native/sessions/2026/07/13" "$root/native/archived_sessions" touch "$root/native/sessions/2026/07/13/rollout.jsonl" ln -s "$root/native/sessions" "$root/sessions" @@ -27,3 +28,50 @@ count=$(cd "$root" && find -H sessions archived_sessions -type f | wc -l | tr -d [[ "$count" == "1" ]] echo "PASS: activation snapshots traverse canonical session symlinks" + +runtime_root=$(mktemp -d) +mkdir -p "$runtime_root/home/sessions" "$runtime_root/home/archived_sessions" "$runtime_root/store/fs/sessions" "$runtime_root/mount" "$runtime_root/native" "$runtime_root/bin" +sqlite3 "$runtime_root/home/state_5.sqlite" 'create table threads (rollout_path text);' + +cat >"$runtime_root/bin/pgrep" <<'EOF' +#!/bin/sh +exit 1 +EOF +cat >"$runtime_root/bin/sleep" <<'EOF' +#!/bin/sh +exit 0 +EOF +cat >"$runtime_root/bin/codexfold" <<'EOF' +#!/bin/sh +printf '%s\n' "$*" >>"$CODEXFOLD_FAKE_LOG" +case "$*" in + 'fs service status --json') + printf '%s\n' '{"daemon_running":true,"mount_healthy":true}' + ;; + fs\ compatibility*) + printf '%s\n' '{"evaluation":{"approved":true,"quarantine":false}}' + ;; + 'fs namespace activate'*) + printf '%s\n' '{"active":true}' + ;; +esac +EOF +chmod +x "$runtime_root/bin/pgrep" "$runtime_root/bin/sleep" "$runtime_root/bin/codexfold" + +runtime_script="$runtime_root/activate.zsh" +sed "s|^export PATH=.*|export PATH=\"$runtime_root/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin\"|" "$script" >"$runtime_script" +runtime_log="$runtime_root/commands.log" +if CODEXFOLD_FAKE_LOG="$runtime_log" /bin/zsh "$runtime_script" \ + "$runtime_root/home" "$runtime_root/store" "$runtime_root/mount" "$runtime_root/native" \ + "$runtime_root/bin/codexfold" 0; then + echo "activation unexpectedly succeeded" >&2 + exit 1 +fi + +grep -Fqx 'fs service stop --apply' "$runtime_log" +grep -Fq 'fs namespace deactivate --apply' "$runtime_log" +grep -Fq 'fs service start --apply' "$runtime_log" +failed_marker=$(find "$runtime_root/store/activation" -type f -name FAILED -print -quit) +[[ -n "$failed_marker" ]] + +echo "PASS: activation failure trap restores the namespace under zsh" From c41e4ac55b8d28517d937041bfb27bc1375b0599 Mon Sep 17 00:00:00 2001 From: King Star Date: Tue, 14 Jul 2026 16:41:49 +0800 Subject: [PATCH 24/33] fix: restore managed routes after rollback failure --- internal/cli/fs.go | 25 +++-- internal/cli/fs_service.go | 8 +- internal/cli/fs_test.go | 121 ++++++++++++++++++++++ internal/mountfs/fuse_integration_test.go | 56 ++++++++++ internal/vfs/state.go | 15 +++ internal/vfs/state_discovery_test.go | 37 +++++++ 6 files changed, 254 insertions(+), 8 deletions(-) diff --git a/internal/cli/fs.go b/internal/cli/fs.go index 3e82143..9d789d3 100644 --- a/internal/cli/fs.go +++ b/internal/cli/fs.go @@ -729,19 +729,30 @@ func newFSRollbackCommand() *cobra.Command { if err != nil { return err } - retiredSnapshot, err := retireCanonicalNativeSnapshot(store, nativeRoot, state.SessionID, state.NativeSnapshot.Path, target.Path, retiredState) + mountedTarget, err := canonicalMountRoute(home, mount, current.RolloutPath) if err != nil { _ = restoreManagedState(store, state.SessionID, retiredState) return err } - mountedTarget, err := canonicalMountRoute(home, mount, current.RolloutPath) - if err == nil { - _, err = waitForTargetMatch(command.Context(), mountedTarget, target, mountWait) + restoreManagedRoute := func(cause error, retiredSnapshot string) error { + var restoreErrors []error + if err := restoreCanonicalNativeSnapshot(state.NativeSnapshot.Path, retiredSnapshot); err != nil { + restoreErrors = append(restoreErrors, err) + } + if err := restoreManagedState(store, state.SessionID, retiredState); err != nil { + restoreErrors = append(restoreErrors, err) + } else if _, err := waitForTargetMatch(command.Context(), mountedTarget, target, mountWait); err != nil { + restoreErrors = append(restoreErrors, fmt.Errorf("verify restored managed route: %w", err)) + } + return errors.Join(append([]error{cause}, restoreErrors...)...) } + retiredSnapshot, err := retireCanonicalNativeSnapshot(store, nativeRoot, state.SessionID, state.NativeSnapshot.Path, target.Path, retiredState) if err != nil { - _ = restoreCanonicalNativeSnapshot(state.NativeSnapshot.Path, retiredSnapshot) - _ = restoreManagedState(store, state.SessionID, retiredState) - return fmt.Errorf("verify canonical native rollback: %w", err) + return restoreManagedRoute(err, "") + } + _, err = waitForTargetMatch(command.Context(), mountedTarget, target, mountWait) + if err != nil { + return restoreManagedRoute(fmt.Errorf("verify canonical native rollback: %w", err), retiredSnapshot) } result.RetiredState = retiredState } else { diff --git a/internal/cli/fs_service.go b/internal/cli/fs_service.go index e6e8a61..5918a0b 100644 --- a/internal/cli/fs_service.go +++ b/internal/cli/fs_service.go @@ -417,7 +417,13 @@ func restoreManagedState(store string, sessionID string, retiredPath string) err return errors.New("store, session ID, and retired state path are required") } target := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) - return os.Rename(filepath.Clean(retiredPath), target) + if err := os.Rename(filepath.Clean(retiredPath), target); err != nil { + return err + } + if _, err := vfs.RepublishSessionState(filepath.Join(target, "state.json")); err != nil { + return fmt.Errorf("republish restored managed state: %w", err) + } + return nil } func retainCanonicalSnapshot(store string, sessionID string, source vfs.NativeFile) (vfs.NativeFile, error) { diff --git a/internal/cli/fs_test.go b/internal/cli/fs_test.go index 184c449..44514d5 100644 --- a/internal/cli/fs_test.go +++ b/internal/cli/fs_test.go @@ -981,6 +981,127 @@ func TestFSRollbackCanonicalRetiresManagedStateAndKeepsRoute(t *testing.T) { } } +func TestFSRollbackCanonicalFailureWaitsForManagedRouteRestoration(t *testing.T) { + allowFixtureMount(t) + home, storeDir, originalPath := fsFixture(t, true) + original, err := os.ReadFile(originalPath) + if err != nil { + t.Fatal(err) + } + filename := "rollout-session.jsonl" + route := filepath.Join(home, "sessions", "2026", "07", "12", filename) + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`update threads set rollout_path = ? where id = 'session'`, route); err != nil { + _ = db.Close() + t.Fatal(err) + } + _ = db.Close() + + nativeRoot := filepath.Join(home, "fold-native") + nativePath := filepath.Join(nativeRoot, "archived_sessions", filename) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(originalPath, nativePath); err != nil { + t.Fatal(err) + } + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + native, err := hashPath(nativePath) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: native, + }) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + tail := []byte("{\"rollback_failure\":true}\n") + if _, err := writer.Append(context.Background(), tail); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + _ = writer.Close() + _ = resolver.Close() + want := append(append([]byte(nil), original...), tail...) + + mount := filepath.Join(home, "fold-fs") + mountedTarget := filepath.Join(mount, "sessions", "2026", "07", "12", filename) + if err := os.MkdirAll(filepath.Dir(mountedTarget), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(mountedTarget, original, 0o600); err != nil { + t.Fatal(err) + } + stateDirectory := filepath.Join(storeDir, "fs", "sessions", "session") + restored := make(chan error, 1) + go func() { + deadline := time.Now().Add(5 * time.Second) + retired := false + for time.Now().Before(deadline) { + _, statErr := os.Stat(stateDirectory) + if !retired && os.IsNotExist(statErr) { + retired = true + _ = os.Remove(mountedTarget) + } + if retired && statErr == nil { + time.Sleep(50 * time.Millisecond) + restored <- os.WriteFile(mountedTarget, want, 0o600) + return + } + time.Sleep(5 * time.Millisecond) + } + restored <- errors.New("managed state was not restored") + }() + + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "rollback", "session", "--apply", "--canonical-namespace", + "--codex-home", home, "--store", storeDir, "--mount", mount, "--native-root", nativeRoot, + "--mount-wait", "200ms", + }) + err = root.Execute() + if err == nil || !strings.Contains(err.Error(), "verify canonical native rollback") { + t.Fatalf("rollback error = %v, want canonical verification failure", err) + } + select { + case restoreErr := <-restored: + if restoreErr != nil { + t.Fatal(restoreErr) + } + default: + t.Fatal("rollback returned before the managed route became readable again") + } + state, err := managedState(storeDir, "session") + if err != nil { + t.Fatal(err) + } + if state.Generation != 2 { + t.Fatalf("restored generation = %d, want 2", state.Generation) + } +} + func TestFSRollbackCanonicalRetiresHiddenSnapshot(t *testing.T) { allowFixtureMount(t) home, storeDir, originalPath := fsFixture(t, true) diff --git a/internal/mountfs/fuse_integration_test.go b/internal/mountfs/fuse_integration_test.go index 7c09ead..12cd9c6 100644 --- a/internal/mountfs/fuse_integration_test.go +++ b/internal/mountfs/fuse_integration_test.go @@ -190,6 +190,46 @@ func TestRealFuseCanonicalNativeToManagedCutover(t *testing.T) { waitForRealUnmount(t, mountPoint) } +func TestRealFuseCanonicalManagedRemovalRevealsNative(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := filepath.Join("sessions", "2026", "07", "14", "rollout-rollback.jsonl") + nativePath := filepath.Join(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + managedBytes := []byte("{\"managed\":true}\n") + nativeBytes := []byte("{\"native\":true}\n") + managed := mountSessionFixture(t, "rollback", managedBytes) + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + if err := filesystem.AddSessionAt("rollback", "/"+filepath.ToSlash(route), managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealMount(t, mountPoint, filesystem) + target := filepath.Join(mountPoint, route) + waitForRealFile(t, target, managedBytes) + + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + if err := filesystem.RemoveSession("rollback"); err != nil { + t.Fatal(err) + } + waitForRealFileTransition(t, target, nativeBytes) + + stopMount() + waitForRealUnmount(t, mountPoint) +} + func TestRealFuseMountCanonicalManagedRename(t *testing.T) { if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") @@ -361,6 +401,22 @@ func waitForRealFile(t *testing.T, path string, want []byte) { t.Fatal("hot-loaded file did not become visible") } +func waitForRealFileTransition(t *testing.T, path string, want []byte) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(path) + if err == nil && bytes.Equal(data, want) { + return + } + if err != nil && !os.IsNotExist(err) { + t.Fatalf("read transitioned file: %v", err) + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("managed file did not transition to native bytes") +} + type fuseFixtureReader map[string][]byte func (r fuseFixtureReader) ReadAt(_ context.Context, ref fold.ObjectRef, destination []byte, offset int64) (int, error) { diff --git a/internal/vfs/state.go b/internal/vfs/state.go index 367bc01..841f197 100644 --- a/internal/vfs/state.go +++ b/internal/vfs/state.go @@ -63,6 +63,21 @@ func LoadSessionState(path string) (SessionState, error) { return state, nil } +func RepublishSessionState(path string) (SessionState, error) { + state, err := LoadSessionState(path) + if err != nil { + return SessionState{}, err + } + if state.Generation == ^uint64(0) { + return SessionState{}, errors.New("session generation cannot advance") + } + state.Generation++ + if err := writeSessionState(path, state); err != nil { + return SessionState{}, err + } + return state, nil +} + func DiscoverSessionStates(root string) ([]SessionState, error) { directory := filepath.Join(root, "fs", "sessions") entries, err := os.ReadDir(directory) diff --git a/internal/vfs/state_discovery_test.go b/internal/vfs/state_discovery_test.go index f26ab7c..aa8c113 100644 --- a/internal/vfs/state_discovery_test.go +++ b/internal/vfs/state_discovery_test.go @@ -53,3 +53,40 @@ func TestLoadSessionStateRejectsStateOutsideManagedSessionDirectory(t *testing.T t.Fatal("LoadSessionState should reject data paths outside the managed session directory") } } + +func TestRepublishSessionStateAdvancesGeneration(t *testing.T) { + root := t.TempDir() + directory := filepath.Join(root, "fs", "sessions", "session") + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + statePath := filepath.Join(directory, "state.json") + state := SessionState{ + Version: sessionStateVersion, SessionID: "session", Generation: 7, + ManifestPath: filepath.Join(root, "manifest.json"), BaseBytes: 1, + BaseSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + DeltaPath: filepath.Join(directory, "delta.jsonl"), + NativeSnapshot: NativeFile{Path: filepath.Join(root, "native.jsonl"), Bytes: 1, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + } + if err := os.WriteFile(state.DeltaPath, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := writeSessionState(statePath, state); err != nil { + t.Fatal(err) + } + + republished, err := RepublishSessionState(statePath) + if err != nil { + t.Fatalf("RepublishSessionState: %v", err) + } + if republished.Generation != 8 { + t.Fatalf("republished generation = %d, want 8", republished.Generation) + } + loaded, err := LoadSessionState(statePath) + if err != nil { + t.Fatal(err) + } + if loaded.Generation != 8 { + t.Fatalf("persisted generation = %d, want 8", loaded.Generation) + } +} From 045eea177baddc66c176acd5c36dfaaef98fc2ce Mon Sep 17 00:00:00 2001 From: King Star Date: Tue, 14 Jul 2026 18:43:01 +0800 Subject: [PATCH 25/33] fix: make canonical rollback retirement restart-safe --- docs/validation-macos-canary.md | 6 + internal/cli/fs.go | 142 +++++- internal/cli/fs_service.go | 125 ++++- internal/cli/fs_test.go | 567 ++++++++++++++++++++-- internal/mountfs/filesystem.go | 27 +- internal/mountfs/filesystem_test.go | 60 +++ internal/mountfs/fuse_integration_test.go | 162 +++++++ 7 files changed, 1029 insertions(+), 60 deletions(-) diff --git a/docs/validation-macos-canary.md b/docs/validation-macos-canary.md index efe878a..6cda3c4 100644 --- a/docs/validation-macos-canary.md +++ b/docs/validation-macos-canary.md @@ -6,6 +6,12 @@ The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have p Additional failure-containment evidence on 2026-07-14: +- Canonical rollback now uses a two-stage retirement request and acknowledgement. The daemon keeps the managed session loaded while preferring a verified native target, so removing or changing that target falls back to managed bytes instead of creating an `ENOENT` window. +- A live pending-retirement restart loaded the managed fallback into a fresh daemon, acknowledged the exact generation and route, and preserved the complete SHA-256. Toggling the native target 100 times while opening the mounted route 2,000 times produced zero read failures. +- A second live restart began with an earlier successful acknowledgement after the native target had disappeared. The fresh daemon replaced it with `native rollback target is unavailable or changed`, remained running, and exposed the complete managed JSONL with the same SHA-256. +- A normal rollback completed with zero route-read failures, preserved the exact visible SHA-256, then passed archive, overwrite-fold, pack rebuild, 10,000-range shadow verification, canonical re-migration, unarchive, and a complete FUSE service restart. +- The isolated real Codex task resumed after that restart and performed a repository review, added and mutation-tested a restart regression, ran Go and race tests, and wrote a detailed verdict. Its mounted rollout grew from 1,226,174 to 1,579,063 bytes; the complete 1,226,174-byte prefix retained SHA-256 `eff00f0583833b1d9cb03b12ed5b19cb68240c37e953c086f028e8bc6a4de2f6`, and all 746 JSONL records parsed. +- Recovery before retirement is covered explicitly: the rollback request uses the recovered `managed.State().Generation`. A fresh-daemon regression also verifies that a stale successful acknowledgement is replaced with a rejection when its native target is no longer valid. - Exact contracts were imported from real FUSE traces for PATH CLI `0.144.3` and Desktop `26.707.71524+5263`. - Canonical migration now verifies the mounted managed target before removing the native directory entry. A clean first migration passed without a retry. - A real Desktop canary preserved the exact 79,067-byte, 16-record source prefix, appended an 8,414-byte, 12-record managed delta, and rolled back to their exact 87,481-byte concatenation. A subsequent native Desktop turn appended 5,590 bytes and 9 records. The final 93,071-byte, 37-record JSONL parsed completely and preserved byte order. diff --git a/internal/cli/fs.go b/internal/cli/fs.go index 9d789d3..6648e52 100644 --- a/internal/cli/fs.go +++ b/internal/cli/fs.go @@ -348,6 +348,13 @@ func newFSServeCommand() *cobra.Command { seen[state.SessionID] = struct{}{} if canonicalNamespace { route, exists := routes[state.SessionID] + handled, err := syncCanonicalRetirement(store, home, nativeRoot, filesystem, state, route, exists, known, knownRoutes, openState) + if err != nil { + return err + } + if handled { + continue + } if !exists { if _, mounted := known[state.SessionID]; mounted { if err := filesystem.RemoveSession(state.SessionID); err != nil && !errors.Is(err, os.ErrNotExist) { @@ -458,6 +465,66 @@ func newFSServeCommand() *cobra.Command { return command } +func syncCanonicalRetirement( + store string, + home string, + nativeRoot string, + filesystem *mountfs.Filesystem, + state vfs.SessionState, + route string, + routeExists bool, + known map[string]uint64, + knownRoutes map[string]string, + openState func(vfs.SessionState) (*vfs.Session, error), +) (bool, error) { + retirement, retiring, err := readRetirementRequest(store, state.SessionID) + if err != nil { + return false, err + } + if !retiring { + return false, removeIfExists(filepath.Join(store, "fs", "sessions", state.SessionID, retirementAcknowledgementFilename)) + } + reject := func(message string) (bool, error) { + rejected := retirement + rejected.Error = message + return true, writeRetirementAcknowledgement(store, state.SessionID, rejected) + } + if !routeExists || retirement.Route != route { + return reject("retirement request does not match the current session route") + } + generation := known[state.SessionID] + if generation != state.Generation || knownRoutes[state.SessionID] != route { + managed, err := openState(state) + if err != nil { + return true, err + } + if err := filesystem.UpsertSessionAt(state.SessionID, route, managed); err != nil { + return true, err + } + generation = managed.State().Generation + known[state.SessionID] = generation + knownRoutes[state.SessionID] = route + } + if retirement.Generation != generation { + return reject("retirement request generation does not match the current session state") + } + nativeTargetPath, err := canonicalNativeRoute(home, nativeRoot, filepath.Join(home, filepath.FromSlash(strings.TrimPrefix(route, "/")))) + if err != nil { + return true, err + } + nativeTarget, targetErr := hashPath(nativeTargetPath) + if targetErr != nil || nativeTarget.Bytes != retirement.Bytes || nativeTarget.SHA256 != retirement.SHA256 { + return reject("native rollback target is unavailable or changed") + } + if err := filesystem.PreferNativeSession(state.SessionID); err != nil { + return true, err + } + if err := writeRetirementAcknowledgement(store, state.SessionID, retirement); err != nil { + return true, err + } + return true, nil +} + func newFSMigrateCommand() *cobra.Command { var codexHome string var storeDir string @@ -725,34 +792,83 @@ func newFSRollbackCommand() *cobra.Command { return err } if canonicalNamespace { - retiredState, err := retireManagedState(store, state.SessionID) + mountedTarget, err := canonicalMountRoute(home, mount, current.RolloutPath) if err != nil { return err } - mountedTarget, err := canonicalMountRoute(home, mount, current.RolloutPath) + canonicalRoute, err := canonicalNamespaceRoute(home, mount, current.RolloutPath) if err != nil { - _ = restoreManagedState(store, state.SessionID, retiredState) return err } - restoreManagedRoute := func(cause error, retiredSnapshot string) error { + retirement, err := createRetirementRequest(store, state.SessionID, managed.State().Generation, canonicalRoute, target) + if err != nil { + return err + } + recoveryWait := mountWait + if recoveryWait < 15*time.Second { + recoveryWait = 15 * time.Second + } + restoreManagedRoute := func(cause error, retiredState string, retiredSnapshot string) error { var restoreErrors []error - if err := restoreCanonicalNativeSnapshot(state.NativeSnapshot.Path, retiredSnapshot); err != nil { - restoreErrors = append(restoreErrors, err) + if retiredSnapshot != "" { + if err := restoreCanonicalNativeSnapshot(state.NativeSnapshot.Path, retiredSnapshot); err != nil { + restoreErrors = append(restoreErrors, err) + } } - if err := restoreManagedState(store, state.SessionID, retiredState); err != nil { + var restored vfs.SessionState + if retiredState == "" { + directory := filepath.Join(store, "fs", "sessions", state.SessionID) + if err := clearRetirementControl(directory); err != nil { + restoreErrors = append(restoreErrors, err) + } else { + restored, err = vfs.RepublishSessionState(filepath.Join(directory, "state.json")) + if err != nil { + restoreErrors = append(restoreErrors, err) + } + } + } else if err := clearRetirementControl(retiredState); err != nil { + restoreErrors = append(restoreErrors, err) + } else if err := restoreManagedState(store, state.SessionID, retiredState); err != nil { restoreErrors = append(restoreErrors, err) - } else if _, err := waitForTargetMatch(command.Context(), mountedTarget, target, mountWait); err != nil { - restoreErrors = append(restoreErrors, fmt.Errorf("verify restored managed route: %w", err)) + } else { + restored, err = managedState(store, state.SessionID) + if err != nil { + restoreErrors = append(restoreErrors, err) + } + } + if restored.Generation != 0 { + if err := waitForMountAcknowledgement(command.Context(), store, state.SessionID, restored.Generation, canonicalRoute, recoveryWait); err != nil { + restoreErrors = append(restoreErrors, fmt.Errorf("wait for restored managed route: %w", err)) + } else if _, err := waitForTargetMatch(command.Context(), mountedTarget, target, recoveryWait); err != nil { + restoreErrors = append(restoreErrors, fmt.Errorf("verify restored managed route: %w", err)) + } } return errors.Join(append([]error{cause}, restoreErrors...)...) } - retiredSnapshot, err := retireCanonicalNativeSnapshot(store, nativeRoot, state.SessionID, state.NativeSnapshot.Path, target.Path, retiredState) - if err != nil { - return restoreManagedRoute(err, "") + if err := waitForRetirementAcknowledgement(command.Context(), store, state.SessionID, retirement, mountWait); err != nil { + return restoreManagedRoute(err, "", "") } _, err = waitForTargetMatch(command.Context(), mountedTarget, target, mountWait) if err != nil { - return restoreManagedRoute(fmt.Errorf("verify canonical native rollback: %w", err), retiredSnapshot) + return restoreManagedRoute(fmt.Errorf("verify canonical native rollback: %w", err), "", "") + } + nativeTarget, err := hashPath(target.Path) + if err != nil || nativeTarget.Bytes != target.Bytes || nativeTarget.SHA256 != target.SHA256 { + if err == nil { + err = errors.New("canonical native rollback target changed before retirement") + } + return restoreManagedRoute(fmt.Errorf("verify canonical native rollback target: %w", err), "", "") + } + retiredState, err := retireManagedState(store, state.SessionID) + if err != nil { + return restoreManagedRoute(err, "", "") + } + retiredSnapshot, err := retireCanonicalNativeSnapshot(store, nativeRoot, state.SessionID, state.NativeSnapshot.Path, target.Path, retiredState) + if err != nil { + return restoreManagedRoute(err, retiredState, "") + } + if err := clearRetirementControl(retiredState); err != nil { + return restoreManagedRoute(err, retiredState, retiredSnapshot) } result.RetiredState = retiredState } else { diff --git a/internal/cli/fs_service.go b/internal/cli/fs_service.go index 5918a0b..346dc6a 100644 --- a/internal/cli/fs_service.go +++ b/internal/cli/fs_service.go @@ -2,6 +2,8 @@ package cli import ( "context" + "crypto/rand" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -545,12 +547,131 @@ type mountAcknowledgement struct { Route string `json:"route"` } +const ( + retirementRequestFilename = "retire.request.json" + retirementAcknowledgementFilename = "retire.ack.json" +) + +type retirementControl struct { + Token string `json:"token"` + Generation uint64 `json:"generation"` + Route string `json:"route"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` + Error string `json:"error,omitempty"` +} + +func createRetirementRequest(store string, sessionID string, generation uint64, route string, target vfs.NativeFile) (retirementControl, error) { + if store == "" || !validSessionID(sessionID) || generation == 0 || route == "" || target.Bytes < 0 || len(target.SHA256) != 64 { + return retirementControl{}, errors.New("complete retirement request metadata is required") + } + directory := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) + requestPath := filepath.Join(directory, retirementRequestFilename) + if _, err := os.Lstat(requestPath); err == nil { + return retirementControl{}, errors.New("session retirement is already pending") + } else if !errors.Is(err, os.ErrNotExist) { + return retirementControl{}, err + } + if err := removeIfExists(filepath.Join(directory, retirementAcknowledgementFilename)); err != nil { + return retirementControl{}, err + } + tokenBytes := make([]byte, 16) + if _, err := rand.Read(tokenBytes); err != nil { + return retirementControl{}, err + } + request := retirementControl{Token: hex.EncodeToString(tokenBytes), Generation: generation, Route: route, Bytes: target.Bytes, SHA256: target.SHA256} + if err := writeSessionControlFile(directory, retirementRequestFilename, request); err != nil { + return retirementControl{}, err + } + return request, nil +} + +func readRetirementRequest(store string, sessionID string) (retirementControl, bool, error) { + path := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID, retirementRequestFilename) + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return retirementControl{}, false, nil + } + if err != nil { + return retirementControl{}, false, err + } + var request retirementControl + if err := json.Unmarshal(data, &request); err != nil { + return retirementControl{}, false, fmt.Errorf("decode retirement request: %w", err) + } + if len(request.Token) != 32 || request.Generation == 0 || request.Route == "" || request.Bytes < 0 || len(request.SHA256) != 64 || request.Error != "" { + return retirementControl{}, false, errors.New("invalid retirement request") + } + return request, true, nil +} + +func writeRetirementAcknowledgement(store string, sessionID string, acknowledgement retirementControl) error { + if len(acknowledgement.Token) != 32 || acknowledgement.Generation == 0 || acknowledgement.Route == "" || acknowledgement.Bytes < 0 || len(acknowledgement.SHA256) != 64 { + return errors.New("complete retirement acknowledgement metadata is required") + } + directory := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) + return writeSessionControlFile(directory, retirementAcknowledgementFilename, acknowledgement) +} + +func waitForRetirementAcknowledgement(ctx context.Context, store string, sessionID string, request retirementControl, timeout time.Duration) error { + if timeout <= 0 { + timeout = 15 * time.Second + } + path := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID, retirementAcknowledgementFilename) + deadline := time.Now().Add(timeout) + for { + data, err := os.ReadFile(path) + if err == nil { + var acknowledgement retirementControl + if json.Unmarshal(data, &acknowledgement) == nil && + acknowledgement.Token == request.Token && acknowledgement.Generation == request.Generation && + acknowledgement.Route == request.Route && acknowledgement.Bytes == request.Bytes && acknowledgement.SHA256 == request.SHA256 { + if acknowledgement.Error != "" { + return fmt.Errorf("retirement rejected: %s", acknowledgement.Error) + } + return nil + } + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if time.Now().After(deadline) { + return errors.New("timed out waiting for retirement acknowledgement") + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(25 * time.Millisecond): + } + } +} + +func clearRetirementControl(directory string) error { + var result error + for _, name := range []string{retirementRequestFilename, retirementAcknowledgementFilename} { + if err := removeIfExists(filepath.Join(filepath.Clean(directory), name)); err != nil { + result = errors.Join(result, err) + } + } + return result +} + +func removeIfExists(path string) error { + if err := os.Remove(filepath.Clean(path)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil +} + func writeMountAcknowledgement(store string, sessionID string, generation uint64, route string) error { if store == "" || !validSessionID(sessionID) || generation == 0 || route == "" { return errors.New("complete mount acknowledgement metadata is required") } directory := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) - data, err := json.Marshal(mountAcknowledgement{Generation: generation, Route: route}) + return writeSessionControlFile(directory, "mounted.json", mountAcknowledgement{Generation: generation, Route: route}) +} + +func writeSessionControlFile(directory string, name string, value any) error { + data, err := json.Marshal(value) if err != nil { return err } @@ -575,7 +696,7 @@ func writeMountAcknowledgement(store string, sessionID string, generation uint64 if err := temporary.Close(); err != nil { return err } - return os.Rename(temporaryPath, filepath.Join(directory, "mounted.json")) + return os.Rename(temporaryPath, filepath.Join(directory, name)) } func waitForMountAcknowledgement(ctx context.Context, store string, sessionID string, generation uint64, route string, timeout time.Duration) error { diff --git a/internal/cli/fs_test.go b/internal/cli/fs_test.go index 44514d5..fc50062 100644 --- a/internal/cli/fs_test.go +++ b/internal/cli/fs_test.go @@ -6,6 +6,8 @@ import ( "database/sql" "encoding/json" "errors" + "fmt" + "io" "os" "path/filepath" "runtime" @@ -934,22 +936,7 @@ func TestFSRollbackCanonicalRetiresManagedStateAndKeepsRoute(t *testing.T) { t.Fatal(err) } stateDirectory := filepath.Join(storeDir, "fs", "sessions", "session") - copyDone := make(chan error, 1) - go func() { - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - if _, err := os.Stat(stateDirectory); os.IsNotExist(err) { - data, readErr := os.ReadFile(targetNativePath) - if readErr == nil { - readErr = os.WriteFile(mountedTarget, data, 0o600) - } - copyDone <- readErr - return - } - time.Sleep(10 * time.Millisecond) - } - copyDone <- errors.New("managed state was not retired") - }() + copyDone := emulateCanonicalRetirement(storeDir, "session", mountedTarget, targetNativePath) executeFS(t, []string{ "fs", "rollback", "session", "--apply", "--canonical-namespace", "--codex-home", home, "--store", storeDir, "--mount", mount, "--native-root", nativeRoot, @@ -981,6 +968,361 @@ func TestFSRollbackCanonicalRetiresManagedStateAndKeepsRoute(t *testing.T) { } } +func TestFSRollbackCanonicalRetirementUsesRecoveredGeneration(t *testing.T) { + allowFixtureMount(t) + home, storeDir, originalPath := fsFixture(t, true) + original, err := os.ReadFile(originalPath) + if err != nil { + t.Fatal(err) + } + filename := "rollout-session.jsonl" + route := filepath.Join(home, "sessions", "2026", "07", "12", filename) + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`update threads set rollout_path = ? where id = 'session'`, route); err != nil { + _ = db.Close() + t.Fatal(err) + } + _ = db.Close() + + nativeRoot := filepath.Join(home, "fold-native") + nativePath := filepath.Join(nativeRoot, "archived_sessions", filename) + targetNativePath := filepath.Join(nativeRoot, "sessions", "2026", "07", "12", filename) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(originalPath, nativePath); err != nil { + t.Fatal(err) + } + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + native, err := hashPath(nativePath) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + recoveryStop := errors.New("stop after COW file publish") + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: native, + BeforeCOWPhase: func(phase string) error { + if phase == "after-file-publish" { + return recoveryStop + } + return nil + }, + }) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + if _, err := writer.WriteAt(context.Background(), []byte("X"), 0); !errors.Is(err, recoveryStop) { + _ = writer.Close() + _ = resolver.Close() + t.Fatalf("WriteAt error = %v, want %v", err, recoveryStop) + } + if err := writer.Close(); err != nil { + _ = resolver.Close() + t.Fatal(err) + } + if err := resolver.Close(); err != nil { + t.Fatal(err) + } + staleState, err := managedState(storeDir, "session") + if err != nil { + t.Fatal(err) + } + if staleState.Generation != 1 { + t.Fatalf("pre-recovery generation = %d, want 1", staleState.Generation) + } + + mount := filepath.Join(home, "fold-fs") + mountedTarget := filepath.Join(mount, "sessions", "2026", "07", "12", filename) + if err := os.MkdirAll(filepath.Dir(mountedTarget), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(mountedTarget, original, 0o600); err != nil { + t.Fatal(err) + } + retirementDone := emulateCanonicalRetirementGeneration(t, storeDir, "session", mountedTarget, targetNativePath, 2) + + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "rollback", "session", "--apply", "--canonical-namespace", + "--codex-home", home, "--store", storeDir, "--mount", mount, "--native-root", nativeRoot, + "--mount-wait", "500ms", + }) + rollbackErr := root.Execute() + if err := <-retirementDone; err != nil { + t.Fatal(err) + } + if rollbackErr != nil { + t.Fatalf("rollback with recovered session state: %v", rollbackErr) + } + if _, err := os.Stat(filepath.Join(storeDir, "fs", "sessions", "session")); !os.IsNotExist(err) { + t.Fatalf("managed state remained after canonical rollback: %v", err) + } + if got, err := os.ReadFile(targetNativePath); err != nil || !bytes.Equal(got, original) { + t.Fatalf("canonical rollback bytes = %q err=%v", got, err) + } +} + +func TestCanonicalRetirementColdLoadRestoresManagedFallbackBeforeNativePreference(t *testing.T) { + home, storeDir, originalPath := fsFixture(t, true) + original, err := os.ReadFile(originalPath) + if err != nil { + t.Fatal(err) + } + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + retainedPath := filepath.Join(storeDir, "fs", "snapshots", "session", "native.jsonl") + if err := os.MkdirAll(filepath.Dir(retainedPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(retainedPath, original, 0o600); err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + retained, err := hashPath(retainedPath) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: retained, + }) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + if err := resolver.Close(); err != nil { + t.Fatal(err) + } + + route := "/archived_sessions/rollout-session.jsonl" + nativeRoot := filepath.Join(home, "fold-native") + nativeTargetPath := filepath.Join(nativeRoot, "archived_sessions", "rollout-session.jsonl") + if err := os.MkdirAll(filepath.Dir(nativeTargetPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(nativeTargetPath, original, 0o600); err != nil { + t.Fatal(err) + } + nativeTarget, err := hashPath(nativeTargetPath) + if err != nil { + t.Fatal(err) + } + if _, err := createRetirementRequest(storeDir, "session", managed.State().Generation, route, nativeTarget); err != nil { + t.Fatal(err) + } + + filesystem := mountfs.NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + known := make(map[string]uint64) + knownRoutes := make(map[string]string) + var closers []io.Closer + t.Cleanup(func() { + for _, closer := range closers { + _ = closer.Close() + } + }) + opens := 0 + openState := func(state vfs.SessionState) (*vfs.Session, error) { + opens++ + current, nextResolver, err := openManagedSession(context.Background(), storeDir, state) + if err == nil { + closers = append(closers, nextResolver) + } + return current, err + } + state, err := managedState(storeDir, "session") + if err != nil { + t.Fatal(err) + } + for attempt := 0; attempt < 2; attempt++ { + handled, err := syncCanonicalRetirement(storeDir, home, nativeRoot, filesystem, state, route, true, known, knownRoutes, openState) + if err != nil || !handled { + t.Fatalf("sync retirement attempt %d: handled=%t err=%v", attempt, handled, err) + } + } + if opens != 1 || known["session"] != state.Generation || knownRoutes["session"] != route { + t.Fatalf("cold load state: opens=%d known=%#v routes=%#v", opens, known, knownRoutes) + } + acknowledgement, err := os.ReadFile(filepath.Join(storeDir, "fs", "sessions", "session", retirementAcknowledgementFilename)) + if err != nil || !bytes.Contains(acknowledgement, []byte(`"token"`)) { + t.Fatalf("retirement acknowledgement = %q err=%v", acknowledgement, err) + } + if got := readMountedFilesystemFile(t, filesystem, route, len(original)); !bytes.Equal(got, original) { + t.Fatalf("native-preferred bytes = %q", got) + } + if err := os.Remove(nativeTargetPath); err != nil { + t.Fatal(err) + } + if got := readMountedFilesystemFile(t, filesystem, route, len(original)); !bytes.Equal(got, original) { + t.Fatalf("managed fallback bytes = %q", got) + } +} + +func TestCanonicalRetirementDaemonRestartRejectsStaleNativeAcknowledgement(t *testing.T) { + home, storeDir, originalPath := fsFixture(t, true) + original, err := os.ReadFile(originalPath) + if err != nil { + t.Fatal(err) + } + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + retainedPath := filepath.Join(storeDir, "fs", "snapshots", "session", "native.jsonl") + if err := os.MkdirAll(filepath.Dir(retainedPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(retainedPath, original, 0o600); err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + retained, err := hashPath(retainedPath) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: retained, + }) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + tail := []byte("{\"pending_retirement\":true}\n") + writer, err := managed.OpenWriter() + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + if _, err := writer.Append(context.Background(), tail); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + if err := writer.Close(); err != nil { + _ = resolver.Close() + t.Fatal(err) + } + if err := resolver.Close(); err != nil { + t.Fatal(err) + } + current := append(append([]byte(nil), original...), tail...) + + route := "/archived_sessions/rollout-session.jsonl" + nativeRoot := filepath.Join(home, "fold-native") + nativeTargetPath := filepath.Join(nativeRoot, "archived_sessions", "rollout-session.jsonl") + if err := os.MkdirAll(filepath.Dir(nativeTargetPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(nativeTargetPath, current, 0o600); err != nil { + t.Fatal(err) + } + nativeTarget, err := hashPath(nativeTargetPath) + if err != nil { + t.Fatal(err) + } + state, err := managedState(storeDir, "session") + if err != nil { + t.Fatal(err) + } + retirement, err := createRetirementRequest(storeDir, "session", state.Generation, route, nativeTarget) + if err != nil { + t.Fatal(err) + } + + var closers []io.Closer + t.Cleanup(func() { + for _, closer := range closers { + _ = closer.Close() + } + }) + opens := 0 + openState := func(state vfs.SessionState) (*vfs.Session, error) { + opens++ + current, nextResolver, err := openManagedSession(context.Background(), storeDir, state) + if err == nil { + closers = append(closers, nextResolver) + } + return current, err + } + + firstDaemon := mountfs.NewCanonical() + firstDaemon.SetNativeRoot(nativeRoot) + firstKnown := make(map[string]uint64) + firstRoutes := make(map[string]string) + handled, err := syncCanonicalRetirement(storeDir, home, nativeRoot, firstDaemon, state, route, true, firstKnown, firstRoutes, openState) + if err != nil || !handled { + t.Fatalf("initial retirement sync: handled=%t err=%v", handled, err) + } + if got := readMountedFilesystemFile(t, firstDaemon, route, len(current)); !bytes.Equal(got, current) { + t.Fatalf("native-preferred bytes = %q, want %q", got, current) + } + + if err := os.Remove(nativeTargetPath); err != nil { + t.Fatal(err) + } + restartedDaemon := mountfs.NewCanonical() + restartedDaemon.SetNativeRoot(nativeRoot) + restartedKnown := make(map[string]uint64) + restartedRoutes := make(map[string]string) + handled, err = syncCanonicalRetirement(storeDir, home, nativeRoot, restartedDaemon, state, route, true, restartedKnown, restartedRoutes, openState) + if err != nil || !handled { + t.Fatalf("restart retirement sync: handled=%t err=%v", handled, err) + } + if opens != 2 { + t.Fatalf("daemon restarts opened managed state %d times, want 2", opens) + } + acknowledgement, err := os.ReadFile(filepath.Join(storeDir, "fs", "sessions", "session", retirementAcknowledgementFilename)) + if err != nil { + t.Fatal(err) + } + var acknowledged retirementControl + if err := json.Unmarshal(acknowledgement, &acknowledged); err != nil { + t.Fatal(err) + } + if acknowledged.Token != retirement.Token || acknowledged.Error == "" { + t.Fatalf("stale acknowledgement was not rejected: %#v", acknowledged) + } + if got := readMountedFilesystemFile(t, restartedDaemon, route, len(current)); !bytes.Equal(got, current) { + t.Fatalf("restart managed fallback bytes = %q, want %q", got, current) + } +} + func TestFSRollbackCanonicalFailureWaitsForManagedRouteRestoration(t *testing.T) { allowFixtureMount(t) home, storeDir, originalPath := fsFixture(t, true) @@ -1053,24 +1395,61 @@ func TestFSRollbackCanonicalFailureWaitsForManagedRouteRestoration(t *testing.T) t.Fatal(err) } stateDirectory := filepath.Join(storeDir, "fs", "sessions", "session") + retirementRequest := filepath.Join(stateDirectory, "retire.request.json") + canonicalRoute := "/sessions/2026/07/12/" + filename restored := make(chan error, 1) go func() { deadline := time.Now().Add(5 * time.Second) - retired := false for time.Now().Before(deadline) { - _, statErr := os.Stat(stateDirectory) - if !retired && os.IsNotExist(statErr) { - retired = true - _ = os.Remove(mountedTarget) + if _, err := os.Stat(stateDirectory); err != nil { + restored <- fmt.Errorf("managed state moved before retirement request: %w", err) + return } - if retired && statErr == nil { - time.Sleep(50 * time.Millisecond) - restored <- os.WriteFile(mountedTarget, want, 0o600) + request, err := os.ReadFile(retirementRequest) + if err == nil { + var rejection retirementControl + if err := json.Unmarshal(request, &rejection); err != nil { + restored <- err + return + } + rejection.Error = "native rollback target is unavailable or changed" + if err := writeRetirementAcknowledgement(storeDir, "session", rejection); err != nil { + restored <- err + return + } + break + } + if !errors.Is(err, os.ErrNotExist) { + restored <- err return } time.Sleep(5 * time.Millisecond) } - restored <- errors.New("managed state was not restored") + for time.Now().Before(deadline) { + if _, err := os.Stat(stateDirectory); err != nil { + restored <- fmt.Errorf("managed state moved during retirement cancellation: %w", err) + return + } + if _, err := os.Stat(retirementRequest); errors.Is(err, os.ErrNotExist) { + state, stateErr := managedState(storeDir, "session") + if stateErr != nil { + restored <- stateErr + return + } + if state.Generation < 2 { + time.Sleep(5 * time.Millisecond) + continue + } + if err := os.WriteFile(mountedTarget, want, 0o600); err != nil { + restored <- err + return + } + restored <- writeMountAcknowledgement(storeDir, "session", state.Generation, canonicalRoute) + return + } + time.Sleep(5 * time.Millisecond) + } + restored <- errors.New("managed route was not restored") }() root := NewRootCommand() @@ -1082,8 +1461,8 @@ func TestFSRollbackCanonicalFailureWaitsForManagedRouteRestoration(t *testing.T) "--mount-wait", "200ms", }) err = root.Execute() - if err == nil || !strings.Contains(err.Error(), "verify canonical native rollback") { - t.Fatalf("rollback error = %v, want canonical verification failure", err) + if err == nil || !strings.Contains(err.Error(), "retirement rejected") { + t.Fatalf("rollback error = %v, want retirement rejection", err) } select { case restoreErr := <-restored: @@ -1186,23 +1565,7 @@ func TestFSRollbackCanonicalRetiresHiddenSnapshot(t *testing.T) { if err := os.WriteFile(mountedTarget, original, 0o600); err != nil { t.Fatal(err) } - stateDirectory := filepath.Join(storeDir, "fs", "sessions", "session") - copyDone := make(chan error, 1) - go func() { - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - if _, err := os.Stat(stateDirectory); os.IsNotExist(err) { - data, readErr := os.ReadFile(targetNativePath) - if readErr == nil { - readErr = os.WriteFile(mountedTarget, data, 0o600) - } - copyDone <- readErr - return - } - time.Sleep(10 * time.Millisecond) - } - copyDone <- errors.New("managed state was not retired") - }() + copyDone := emulateCanonicalRetirement(storeDir, "session", mountedTarget, targetNativePath) // The mounted target is only used as the FUSE visibility probe. The // canonical rollback writes the latest bytes to the retained native route. executeFS(t, []string{ @@ -1489,6 +1852,122 @@ func executeFS(t *testing.T, args []string) { } } +func emulateCanonicalRetirement(storeDir string, sessionID string, mountedTarget string, nativeTarget string) <-chan error { + done := make(chan error, 1) + go func() { + stateDirectory := filepath.Join(storeDir, "fs", "sessions", sessionID) + requestPath := filepath.Join(stateDirectory, retirementRequestFilename) + acknowledgementPath := filepath.Join(stateDirectory, retirementAcknowledgementFilename) + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(stateDirectory); err != nil { + done <- fmt.Errorf("managed state moved before retirement request: %w", err) + return + } + request, err := os.ReadFile(requestPath) + if err == nil { + data, readErr := os.ReadFile(nativeTarget) + if readErr == nil { + readErr = os.WriteFile(mountedTarget, data, 0o600) + } + if readErr == nil { + readErr = os.WriteFile(acknowledgementPath, request, 0o600) + } + done <- readErr + return + } + if !errors.Is(err, os.ErrNotExist) { + done <- err + return + } + time.Sleep(5 * time.Millisecond) + } + done <- errors.New("retirement request was not created") + }() + return done +} + +func emulateCanonicalRetirementGeneration(t *testing.T, storeDir string, sessionID string, mountedTarget string, nativeTarget string, expectedGeneration uint64) <-chan error { + t.Helper() + done := make(chan error, 1) + go func() { + stateDirectory := filepath.Join(storeDir, "fs", "sessions", sessionID) + requestPath := filepath.Join(stateDirectory, retirementRequestFilename) + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(requestPath) + if err == nil { + var request retirementControl + if err := json.Unmarshal(data, &request); err != nil { + done <- err + return + } + if request.Generation != expectedGeneration { + rejection := request + rejection.Error = fmt.Sprintf("retirement generation = %d, want recovered %d", request.Generation, expectedGeneration) + if err := writeRetirementAcknowledgement(storeDir, sessionID, rejection); err != nil { + done <- err + return + } + for time.Now().Before(deadline) { + if _, err := os.Stat(requestPath); !errors.Is(err, os.ErrNotExist) { + time.Sleep(5 * time.Millisecond) + continue + } + state, err := managedState(storeDir, sessionID) + if err != nil || state.Generation <= expectedGeneration { + time.Sleep(5 * time.Millisecond) + continue + } + native, err := os.ReadFile(nativeTarget) + if err == nil { + err = os.WriteFile(mountedTarget, native, 0o600) + } + if err == nil { + err = writeMountAcknowledgement(storeDir, sessionID, state.Generation, request.Route) + } + done <- err + return + } + done <- errors.New("managed route was not republished after stale retirement request") + return + } + native, err := os.ReadFile(nativeTarget) + if err == nil { + err = os.WriteFile(mountedTarget, native, 0o600) + } + if err == nil { + err = writeRetirementAcknowledgement(storeDir, sessionID, request) + } + done <- err + return + } + if !errors.Is(err, os.ErrNotExist) { + done <- err + return + } + time.Sleep(5 * time.Millisecond) + } + done <- errors.New("retirement request was not created") + }() + return done +} + +func readMountedFilesystemFile(t *testing.T, filesystem *mountfs.Filesystem, route string, size int) []byte { + t.Helper() + handle, errno := filesystem.Open(route, os.O_RDONLY) + if errno != 0 { + t.Fatalf("open mounted route %s: %v", route, errno) + } + defer filesystem.Release(handle) + data := make([]byte, size) + n, errno := filesystem.Read(handle, data, 0) + if errno != 0 { + t.Fatalf("read mounted route %s: %v", route, errno) + } + return data[:n] +} + func allowFixtureMount(t *testing.T) { t.Helper() previous := mountHealthProbe diff --git a/internal/mountfs/filesystem.go b/internal/mountfs/filesystem.go index 5793feb..fa28d57 100644 --- a/internal/mountfs/filesystem.go +++ b/internal/mountfs/filesystem.go @@ -39,6 +39,7 @@ type Filesystem struct { sessions map[string]*vfs.Session paths map[string]string retained map[string]string + nativeFirst map[string]struct{} directories map[string]struct{} handles map[uint64]*fileHandle next uint64 @@ -54,7 +55,7 @@ func New() *Filesystem { func NewCanonical() *Filesystem { return &Filesystem{ sessions: make(map[string]*vfs.Session), paths: make(map[string]string), - retained: make(map[string]string), + retained: make(map[string]string), nativeFirst: make(map[string]struct{}), directories: map[string]struct{}{`/`: {}, `/sessions`: {}, `/archived_sessions`: {}}, handles: make(map[uint64]*fileHandle), next: 1, canonical: true, } @@ -115,6 +116,7 @@ func (f *Filesystem) AddSessionAt(sessionID string, name string, session *vfs.Se f.ensureDirectoryChainLocked(path.Dir(cleaned)) f.sessions[sessionID] = session f.paths[cleaned] = sessionID + delete(f.nativeFirst, sessionID) f.registerRetainedPathLocked(sessionID, session) return nil } @@ -142,6 +144,7 @@ func (f *Filesystem) UpsertSessionAt(sessionID string, name string, session *vfs } f.sessions[sessionID] = session f.paths[cleaned] = sessionID + delete(f.nativeFirst, sessionID) f.registerRetainedPathLocked(sessionID, session) return nil } @@ -174,6 +177,19 @@ func (f *Filesystem) MoveSessionAt(sessionID string, name string) error { return nil } +func (f *Filesystem) PreferNativeSession(sessionID string) error { + if !f.canonical || !safeSessionID(sessionID) { + return errors.New("canonical filesystem and safe session ID are required") + } + f.mu.Lock() + defer f.mu.Unlock() + if _, exists := f.sessions[sessionID]; !exists { + return os.ErrNotExist + } + f.nativeFirst[sessionID] = struct{}{} + return nil +} + func (f *Filesystem) RemoveSession(sessionID string) error { if !safeSessionID(sessionID) { return errors.New("safe session ID is required") @@ -184,6 +200,7 @@ func (f *Filesystem) RemoveSession(sessionID string) error { return os.ErrNotExist } delete(f.sessions, sessionID) + delete(f.nativeFirst, sessionID) for route, currentID := range f.paths { if currentID == sessionID { delete(f.paths, route) @@ -676,10 +693,18 @@ func (f *Filesystem) sessionForPath(name string) (*vfs.Session, syscall.Errno) { f.mu.RLock() sessionID := f.paths[cleaned] session := f.sessions[sessionID] + _, nativeFirst := f.nativeFirst[sessionID] + root := f.nativeRoot + _, retained := f.retained[cleaned] f.mu.RUnlock() if session == nil { return nil, syscall.ENOENT } + if nativeFirst && root != "" && !retained { + if info, err := os.Stat(nativePathFromRoot(root, cleaned)); err == nil && !info.IsDir() { + return nil, syscall.ENOENT + } + } return session, 0 } if cleaned == "/" || strings.Count(cleaned, "/") != 1 || !strings.HasSuffix(cleaned, ".jsonl") { diff --git a/internal/mountfs/filesystem_test.go b/internal/mountfs/filesystem_test.go index b384cec..753f2b7 100644 --- a/internal/mountfs/filesystem_test.go +++ b/internal/mountfs/filesystem_test.go @@ -264,6 +264,66 @@ func TestCanonicalFilesystemManagedSessionMasksRetainedSnapshotAtCurrentRoute(t } } +func TestCanonicalFilesystemNativePreferenceFallsBackToManagedWithoutPathLoss(t *testing.T) { + root := t.TempDir() + route := "/sessions/2026/07/14/rollout-retirement.jsonl" + nativePath := nativePathFromRoot(root, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + managedBytes := []byte("managed-current\n") + nativeBytes := []byte("native-current\n") + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + if err := filesystem.AddSessionAt("session", route, mountSessionFixture(t, "session", managedBytes)); err != nil { + t.Fatal(err) + } + read := func() []byte { + t.Helper() + handle, errno := filesystem.Open(route, os.O_RDONLY) + if errno != 0 { + t.Fatalf("Open errno=%v", errno) + } + defer filesystem.Release(handle) + attribute, errno := filesystem.Getattr(route) + if errno != 0 { + t.Fatalf("Getattr errno=%v", errno) + } + data := make([]byte, attribute.Size) + n, errno := filesystem.Read(handle, data, 0) + if errno != 0 || n != len(data) { + t.Fatalf("Read n=%d errno=%v size=%d", n, errno, len(data)) + } + return data + } + + if got := read(); !bytes.Equal(got, managedBytes) { + t.Fatalf("initial bytes = %q, want managed %q", got, managedBytes) + } + if err := filesystem.PreferNativeSession("session"); err != nil { + t.Fatal(err) + } + if got := read(); !bytes.Equal(got, nativeBytes) { + t.Fatalf("preferred bytes = %q, want native %q", got, nativeBytes) + } + if err := os.Remove(nativePath); err != nil { + t.Fatal(err) + } + if got := read(); !bytes.Equal(got, managedBytes) { + t.Fatalf("fallback bytes = %q, want managed %q", got, managedBytes) + } + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + if got := read(); !bytes.Equal(got, nativeBytes) { + t.Fatalf("restored native bytes = %q, want %q", got, nativeBytes) + } +} + func TestCanonicalFilesystemHidesRetainedSnapshotAfterManagedRouteMoves(t *testing.T) { root := t.TempDir() filename := "rollout-2026-07-12T14-28-28-session.jsonl" diff --git a/internal/mountfs/fuse_integration_test.go b/internal/mountfs/fuse_integration_test.go index 12cd9c6..2e3734e 100644 --- a/internal/mountfs/fuse_integration_test.go +++ b/internal/mountfs/fuse_integration_test.go @@ -230,6 +230,154 @@ func TestRealFuseCanonicalManagedRemovalRevealsNative(t *testing.T) { waitForRealUnmount(t, mountPoint) } +func TestRealFuseCanonicalManagedRemovalCanBeReaddedAtSamePath(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := filepath.Join("sessions", "2026", "07", "14", "rollout-republish.jsonl") + nativePath := filepath.Join(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + firstManagedBytes := []byte("{\"generation\":1}\n") + nativeBytes := []byte("{\"native\":true}\n") + digest := sha256.Sum256(firstManagedBytes) + digestHex := hex.EncodeToString(digest[:]) + managedRoot := filepath.Join(root, "managed") + managedNativePath := filepath.Join(root, "managed-native.jsonl") + if err := os.WriteFile(managedNativePath, firstManagedBytes, 0o600); err != nil { + t.Fatal(err) + } + manifest := fold.Manifest{ + Version: fold.ManifestVersion, Kind: fold.ManifestKind, + Session: fold.ManifestSession{ID: "republish", RolloutPath: managedNativePath}, + Source: fold.ManifestSource{Bytes: int64(len(firstManagedBytes)), SHA256: digestHex}, + Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: digestHex, RawBytes: int64(len(firstManagedBytes))}}}, + } + managedOptions := vfs.SessionOptions{ + Root: managedRoot, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, + Reader: fuseFixtureReader{digestHex: firstManagedBytes}, + NativeSnapshot: vfs.NativeFile{ + Path: managedNativePath, Bytes: int64(len(firstManagedBytes)), SHA256: digestHex, + }, + } + firstManaged, err := vfs.OpenSession(context.Background(), managedOptions) + if err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + if err := filesystem.AddSessionAt("republish", "/"+filepath.ToSlash(route), firstManaged); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + canonicalHome := filepath.Join(root, "home") + if err := os.MkdirAll(canonicalHome, 0o700); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(mountPoint, "sessions"), filepath.Join(canonicalHome, "sessions")); err != nil { + t.Fatal(err) + } + stopMount := startRealMount(t, mountPoint, filesystem) + target := filepath.Join(canonicalHome, route) + waitForRealFile(t, target, firstManagedBytes) + + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + stateDirectory := filepath.Dir(firstManaged.State().DeltaPath) + retiredStateDirectory := stateDirectory + ".retired" + if err := os.Rename(stateDirectory, retiredStateDirectory); err != nil { + t.Fatal(err) + } + if _, err := os.ReadFile(target); err == nil { + t.Fatal("managed read unexpectedly succeeded after its state directory was retired") + } + if err := filesystem.RemoveSession("republish"); err != nil { + t.Fatal(err) + } + waitForRealFileTransition(t, target, nativeBytes) + if err := os.Remove(nativePath); err != nil { + t.Fatal(err) + } + waitForRealFileMissing(t, target) + if err := os.Rename(retiredStateDirectory, stateDirectory); err != nil { + t.Fatal(err) + } + republished, err := vfs.RepublishSessionState(filepath.Join(stateDirectory, "state.json")) + if err != nil { + t.Fatal(err) + } + if republished.Generation != 2 { + t.Fatalf("republished generation = %d, want 2", republished.Generation) + } + secondManaged, err := vfs.OpenSession(context.Background(), managedOptions) + if err != nil { + t.Fatal(err) + } + + if err := filesystem.UpsertSessionAt("republish", "/"+filepath.ToSlash(route), secondManaged); err != nil { + t.Fatal(err) + } + waitForRealFile(t, target, firstManagedBytes) + + stopMount() + waitForRealUnmount(t, mountPoint) +} + +func TestRealFuseCanonicalNativePreferenceNeverLosesPath(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := filepath.Join("sessions", "2026", "07", "14", "rollout-native-preference.jsonl") + nativePath := filepath.Join(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + managedBytes := []byte("{\"managed\":true}\n") + nativeBytes := []byte("{\"native\":true}\n") + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + if err := filesystem.AddSessionAt("native-preference", "/"+filepath.ToSlash(route), mountSessionFixture(t, "native-preference", managedBytes)); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealMount(t, mountPoint, filesystem) + target := filepath.Join(mountPoint, route) + waitForRealFile(t, target, managedBytes) + + if err := filesystem.PreferNativeSession("native-preference"); err != nil { + t.Fatal(err) + } + waitForRealFileTransition(t, target, nativeBytes) + if err := os.Remove(nativePath); err != nil { + t.Fatal(err) + } + waitForRealFileTransition(t, target, managedBytes) + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + waitForRealFileTransition(t, target, nativeBytes) + + stopMount() + waitForRealUnmount(t, mountPoint) +} + func TestRealFuseMountCanonicalManagedRename(t *testing.T) { if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") @@ -417,6 +565,20 @@ func waitForRealFileTransition(t *testing.T, path string, want []byte) { t.Fatal("managed file did not transition to native bytes") } +func waitForRealFileMissing(t *testing.T, path string) { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(path); os.IsNotExist(err) { + return + } else if err != nil { + t.Fatalf("stat transitioned file: %v", err) + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("canonical file did not become absent") +} + type fuseFixtureReader map[string][]byte func (r fuseFixtureReader) ReadAt(_ context.Context, ref fold.ObjectRef, destination []byte, offset int64) (int, error) { From cc1c5160270682c09f7d79d9aee6a633bcd12d30 Mon Sep 17 00:00:00 2001 From: King Star Date: Tue, 14 Jul 2026 23:08:01 +0800 Subject: [PATCH 26/33] docs: align transparent filesystem plan with implementation --- ...arent-session-filesystem-implementation.md | 238 ++++++++++++------ ...1-transparent-session-filesystem-design.md | 56 ++++- ...-session-filesystem-traceability-review.md | 14 +- ...ion-filesystem-implementation-alignment.md | 98 ++++++++ docs/validation-macos-canary.md | 8 +- 5 files changed, 333 insertions(+), 81 deletions(-) create mode 100644 docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md diff --git a/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md b/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md index ad4821e..4991f2c 100644 --- a/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md +++ b/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md @@ -1,6 +1,6 @@ # Transparent Codex Session Filesystem Implementation Plan -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +> Execute this plan task-by-task. Checkboxes describe current repository completion, not intended future work; mark a step complete only when code and fresh evidence support it. **Goal:** Make unmodified Codex Desktop and Codex CLI open, resume, fork, and continue managed JSONL sessions directly while exact duplicate bytes are stored once and current session writes remain durable and independently recoverable. @@ -8,12 +8,30 @@ **Tech Stack:** Go 1.26, zstd, Cobra, modernc SQLite, `cgofuse` v1.6.0 behind platform/build tags, and FUSE-T 1.2.7 as the validated macOS host. +## Alignment Snapshot + +Current public status remains `fs-engine-preview`. Tasks 1 through 7 and 9 through 10 are implemented. Task 8 lacks bounded automatic enrollment. Task 11 has substantial isolated macOS evidence but has not passed managed-session host restart, sleep/wake, current-client compatibility, real-home retained-source canaries, or seven-day retention. + +| Task | Status | Current evidence | Remaining work | +| --- | --- | --- | --- | +| 1 | Complete | Commit `17564e9`; `internal/pack` tests | None in this task | +| 2 | Complete | Commit `039e6b9`; exact and 10,000-range view tests | None in this task | +| 3 | Complete | Commit `9a7f1e8`; append and COW tests | None in this task | +| 4 | Complete | Commit `076d772`; journal, compaction, and fallback tests | None in this task | +| 5 | Complete | Commit `35a53fc`; status, shadow, doctor, and benchmark tests | Platform evidence remains outside this task | +| 6 | Complete | Commit `5be10d2`; exact-version compatibility and optimistic route tests | New installed client versions still require fresh contracts | +| 7 | Complete | Commit `3f51aa5`; neutral operations and real FUSE-T adapter tests | Linux and Windows real adapters remain separate product gates | +| 8 | Partial | Commit `3352b87`; standalone CLI and guarded lifecycle commands | Implement bounded automatic discovery and enrollment | +| 9 | Complete | Commit `4589ffa`; launchd lifecycle and update preflight tests | Production update promotion remains gated by platform readiness | +| 10 | Complete | Commit `a1ac76e`; synthetic, crash, race, cross-compile, and 758 MiB evidence | This task proves only the shared engine preview | +| 11 | Partial | Real macOS CLI/Desktop, FUSE-T, rollback, restart, and quarantine evidence | Complete the remaining disruptive and retention gates | + ## Global Constraints - `TF-001`: normal open and resume require no manual `materialize` or preparation command. - `TF-002`: Codex Desktop and CLI remain unmodified and access normal regular-file JSONL paths. - `TF-003`: exact bytes and every operation observed in native Codex traces must have native-equivalent behavior. -- `TF-004`: identical bytes across sessions and forks are stored once while histories remain independently writable. +- `TF-004`: exact repeated fields, records, and content-defined chunks at arbitrary positions are stored once across sessions and forks while histories remain independently writable; strict prefix sharing is not required. - `TF-005`: append writes persist to a delta without complete base hydration. - `TF-006`: truncate, random write, and other representable mutations transition to complete copy-on-write before success. - `TF-007`: packed reads perform neither per-part loose-object opens nor per-object persistent-index queries. @@ -27,6 +45,11 @@ - `TF-015`: unknown client versions enter compatibility quarantine and cannot write a virtual route before current bytes are automatically routed to verified native backing. - `TF-016`: FUSE-T or another privileged prerequisite is not installed without explicit user authorization. - `TF-017`: canonical namespace activation requires a verified CodexFold mount identity, write-sealed unmounted backing, route normalization, and a watcher that tolerates canonical and mount-alias spellings. +- `TF-018`: branch classification and archival are conservative, proof-first, explicitly selected, and recoverable. +- `TF-019`: fully contained archived-session deletion requires exact containment and recovery proof. +- `TF-020`: byte-preserving optimization never invokes content-changing repair, reconciliation, or prompt cleanup implicitly. +- `TF-021`: full-size copies, retained generations, temporary artifacts, and savings claims obey hard physical-space budgets and accounting. +- `TF-022`: the public product has no private control-plane runtime or documentation dependency. - Mock and fixture evidence never satisfies a gate that names real Codex, a real adapter, client upgrade, host restart, or canary retention. - Public code and documentation contain no private paths, domains, credentials, real session IDs, or private control-plane dependency. @@ -51,6 +74,11 @@ | `TF-015` | 4, 6 | 10, 11 | | `TF-016` | 7, 9 | 11 | | `TF-017` | 7, 9 | 10, 11 | +| `TF-018` | 13 | 13 | +| `TF-019` | Storage-engine baseline, 13 | `internal/contain`, `internal/prune`, and Task 13 boundary tests | +| `TF-020` | Storage-engine baseline, 13 | `internal/reconcile` and CLI boundary tests | +| `TF-021` | 12, 14 | 14, 15 | +| `TF-022` | All public tasks | 10 and public sanitization checks | --- @@ -69,17 +97,17 @@ - Consumes: Fold V1 `fold.Manifest`, `fold.ObjectRef`, and loose zstd objects. - Produces: `pack.Build(ctx, storeDir, BuildOptions) (BuildResult, error)`, `pack.Open(storeDir, OpenOptions) (*Resolver, error)`, and `(*Resolver).ReadAt(ctx, ref, dst, offset) (int, error)`. -- [ ] **Step 1: Write failing pack round-trip, corruption, transaction, and random-read tests** +- [x] **Step 1: Write failing pack round-trip, corruption, transaction, and random-read tests** Tests construct repeated small objects and one object larger than two 256 KiB blocks, build a generation with a 1 MiB pack limit, remove a copied loose-object fixture, and assert all offset/length reads match source bytes. They also corrupt a block, interrupt before `CURRENT` publication, and assert the previous generation remains readable. -- [ ] **Step 2: Run the focused tests and confirm missing package/API failures** +- [x] **Step 2: Run the focused tests and confirm missing package/API failures** Run: `go test ./internal/pack -count=1` Expected: FAIL because `internal/pack` and its exported API do not exist. -- [ ] **Step 3: Implement the candidate packed format and transactional builder** +- [x] **Step 3: Implement the candidate packed format and transactional builder** Use this public shape: @@ -108,15 +136,15 @@ type Object struct { Decode each loose object as a stream, split decoded bytes into independently compressed 256 KiB blocks, hash both object and blocks, and write bounded immutable pack files. Publish a verified generation directory and atomically replace `packs/CURRENT`; never encode physical locations into Fold V1 manifests. -- [ ] **Step 4: Implement in-memory resolver lookup and bounded block cache** +- [x] **Step 4: Implement in-memory resolver lookup and bounded block cache** Load the candidate JSON index once at `Open`, keep `map[string]Object`, use `ReadAt` on pack files, verify block checksum and decoded size, and cache decompressed blocks under a byte budget. A runtime read must not open a loose object or query SQLite. -- [ ] **Step 5: Implement pack doctor and loose-object streaming helper** +- [x] **Step 5: Implement pack doctor and loose-object streaming helper** Export a streaming loose-object reader from `internal/fold` without changing Fold V1 paths. Doctor verifies `CURRENT`, every indexed extent, block digest, raw length, object digest, and every manifest reference. -- [ ] **Step 6: Run focused, race, and complete tests** +- [x] **Step 6: Run focused, race, and complete tests** Run: @@ -128,7 +156,7 @@ go test ./... -count=1 Expected: all PASS. -- [ ] **Step 7: Commit** +- [x] **Step 7: Commit** ```bash git add internal/pack internal/fold/store.go @@ -146,17 +174,17 @@ git commit -m "feat: add transactional packed object resolver" - Consumes: `fold.Manifest` and an `ObjectReader` implemented by `pack.Resolver`. - Produces: `vfs.NewView(manifest, reader) (*View, error)`, `(*View).Size() int64`, and `(*View).ReadAt(ctx, dst, offset) (int, error)`. -- [ ] **Step 1: Write failing exact-read tests** +- [x] **Step 1: Write failing exact-read tests** Cover empty reads, EOF, final partial reads, random offsets, cross-part boundaries, a read spanning more than two parts, and 10,000 deterministic random comparisons against a native byte slice. -- [ ] **Step 2: Run the focused test and verify the API is absent** +- [x] **Step 2: Run the focused test and verify the API is absent** Run: `go test ./internal/vfs -run 'TestView' -count=1` Expected: FAIL because `View` is undefined. -- [ ] **Step 3: Implement cumulative offsets and binary-search reads** +- [x] **Step 3: Implement cumulative offsets and binary-search reads** ```go type ObjectReader interface { @@ -172,7 +200,7 @@ type View struct { Validate total part bytes against `manifest.Source.Bytes`, locate the first part with `sort.Search`, and fill the destination across parts without materializing the session. Match `io.ReaderAt` EOF semantics exactly. -- [ ] **Step 4: Run focused, race, and complete tests** +- [x] **Step 4: Run focused, race, and complete tests** Run: @@ -184,7 +212,7 @@ go test ./... -count=1 Expected: all PASS. -- [ ] **Step 5: Commit** +- [x] **Step 5: Commit** ```bash git add internal/vfs @@ -203,17 +231,17 @@ git commit -m "feat: add exact virtual rollout byte view" - Consumes: immutable `View`, session state directory, and native snapshot metadata. - Produces: `vfs.OpenSession(ctx, Options) (*Session, error)`, reader/writer handles, `Append`, `WriteAt`, `Truncate`, `Sync`, and `MaterializeCurrent`. -- [ ] **Step 1: Write failing append and COW state-machine tests** +- [x] **Step 1: Write failing append and COW state-machine tests** Assert append immediately extends visible bytes, `fsync` persists after reopen, one writer lease is enforced, readers retain generation snapshots, random write and truncate create byte-verified native backing before mutation, and interrupted COW leaves the old generation readable. -- [ ] **Step 2: Run focused tests and verify expected missing API failures** +- [x] **Step 2: Run focused tests and verify expected missing API failures** Run: `go test ./internal/vfs -run 'TestSession|TestAppend|TestCopyOnWrite' -count=1` Expected: FAIL because writable session APIs are undefined. -- [ ] **Step 3: Implement atomic session state and generation leases** +- [x] **Step 3: Implement atomic session state and generation leases** ```go type SessionState struct { @@ -230,15 +258,15 @@ type SessionState struct { Commit state through a synchronized temporary file and atomic replacement. Readers pin a generation. Writers acquire a process-local and on-disk lease and never trigger compaction from `Release`. -- [ ] **Step 4: Implement append fast path** +- [x] **Step 4: Implement append fast path** Append accepted bytes to a normal delta opened with `O_APPEND`, return success only for written bytes, synchronize on `Sync`, and compose reads as `base || delta`. -- [ ] **Step 5: Implement safe COW transition** +- [x] **Step 5: Implement safe COW transition** Freeze new writers, stream current visible bytes to a temporary native backing, verify byte count and SHA-256, atomically publish backing state, then apply `WriteAt` or `Truncate`. Never mutate packs or manifests in place. -- [ ] **Step 6: Run focused, crash-reopen, race, and complete tests** +- [x] **Step 6: Run focused, crash-reopen, race, and complete tests** Run: @@ -250,7 +278,7 @@ go test ./... -count=1 Expected: all PASS. -- [ ] **Step 7: Commit** +- [x] **Step 7: Commit** ```bash git add internal/vfs @@ -270,29 +298,29 @@ git commit -m "feat: add durable append and copy-on-write sessions" - Consumes: writable `Session`, Fold V1 writer, and atomic state commits. - Produces: `Recover`, `Compact`, `CreateCurrentNativeBacking`, and deterministic journal phase records. -- [ ] **Step 1: Write failing phase-interruption tests** +- [x] **Step 1: Write failing phase-interruption tests** Inject termination-equivalent errors before and after every prepare, data sync, state publish, route-ready, and cleanup phase. Reopen and assert exact bytes, one active generation, retained old readers, and idempotent recovery. -- [ ] **Step 2: Verify the recovery tests fail for missing APIs** +- [x] **Step 2: Verify the recovery tests fail for missing APIs** Run: `go test ./internal/vfs -run 'TestRecover|TestCompact|TestFallback' -count=1` Expected: FAIL because journal operations do not exist. -- [ ] **Step 3: Implement append-only journal and recovery dispatcher** +- [x] **Step 3: Implement append-only journal and recovery dispatcher** Journal records contain operation ID, session ID, operation kind, phase, source generation, candidate generation, paths, byte counts, and digests, but never session content. Synchronize phase records before the represented state change. -- [ ] **Step 4: Implement idle compaction with optimistic revalidation** +- [x] **Step 4: Implement idle compaction with optimistic revalidation** Require zero writers, no writer lease, elapsed idle window, and unchanged delta size, mtime, and digest. Fold a current native stream into a new immutable manifest generation, verify the complete virtual SHA-256, atomically switch state, and retain old files until generation leases close. -- [ ] **Step 5: Implement latest-byte native fallback** +- [x] **Step 5: Implement latest-byte native fallback** `CreateCurrentNativeBacking` freezes writers, streams the latest committed virtual bytes, verifies exact digest, publishes a normal JSONL, and records it separately from the stale migration snapshot. It may reuse the snapshot only when digest and size still match. -- [ ] **Step 6: Run phase, race, and complete tests** +- [x] **Step 6: Run phase, race, and complete tests** Run: @@ -304,7 +332,7 @@ go test ./... -count=1 Expected: all PASS. -- [ ] **Step 7: Commit** +- [x] **Step 7: Commit** ```bash git add internal/vfs @@ -324,17 +352,17 @@ git commit -m "feat: add journaled recovery compaction and fallback" - Consumes: pack doctor, session engine, route metadata, and native source. - Produces: typed status, shadow evidence, doctor report, and JSON benchmark report. -- [ ] **Step 1: Write failing status, shadow, and doctor tests** +- [x] **Step 1: Write failing status, shadow, and doctor tests** Assert only canonical status terms are emitted; block-by-block and random shadow comparison catches a one-byte mismatch; doctor distinguishes daemon and mount health and checks pack, manifest, delta, backing, route, fallback, journal, and client compatibility independently. -- [ ] **Step 2: Run focused tests and verify missing package failures** +- [x] **Step 2: Run focused tests and verify missing package failures** Run: `go test ./internal/fsctl -count=1` Expected: FAIL because `internal/fsctl` does not exist. -- [ ] **Step 3: Implement canonical status and complete doctor aggregation** +- [x] **Step 3: Implement canonical status and complete doctor aggregation** ```go type Capability string @@ -347,11 +375,11 @@ const ( Return structured issues with component, severity, session ID, generation, and remediation; never include rollout contents. -- [ ] **Step 4: Implement shadow and benchmark runners** +- [x] **Step 4: Implement shadow and benchmark runners** Shadow compares full SHA-256 plus deterministic random offset/length reads. Benchmark measures native and virtual cold/warm sequential reads, 4 KiB random p50/p95/p99, stat/open, append, append+fsync, CPU time, RSS, and bytes read under the 128 MiB test budget. -- [ ] **Step 5: Run focused and complete tests** +- [x] **Step 5: Run focused and complete tests** Run: @@ -363,7 +391,7 @@ go test ./... -count=1 Expected: all PASS. -- [ ] **Step 6: Commit** +- [x] **Step 6: Commit** ```bash git add internal/fsctl @@ -385,17 +413,17 @@ git commit -m "feat: add shadow doctor benchmark and fs status" - Consumes: sanitized native operation traces, installed client versions, Codex SQLite state, and current-byte fallback. - Produces: compatibility results and optimistic `RouteSession`/`RestoreSession` transactions. -- [ ] **Step 1: Write failing trace, version, and SQLite transaction tests** +- [x] **Step 1: Write failing trace, version, and SQLite transaction tests** Cover sanitized `fs_usage` parsing, unknown-version quarantine, exact approved-version matching, optimistic route update, concurrent route change rejection, and rollback to a current backing rather than a stale migration snapshot. -- [ ] **Step 2: Run focused tests and verify missing API failures** +- [x] **Step 2: Run focused tests and verify missing API failures** Run: `go test ./internal/compat ./internal/codex -count=1` Expected: FAIL because compatibility and route APIs are absent. -- [ ] **Step 3: Implement machine-readable compatibility contracts** +- [x] **Step 3: Implement machine-readable compatibility contracts** ```go type Contract struct { @@ -410,15 +438,15 @@ type Contract struct { Store operation names, flags, and observed semantics without paths or contents. A version mismatch returns quarantine, never inferred compatibility. -- [ ] **Step 4: Implement optimistic Codex state routing** +- [x] **Step 4: Implement optimistic Codex state routing** Use `BEGIN IMMEDIATE`, `busy_timeout`, and `update threads set rollout_path=? where id=? and rollout_path=?`. Verify exactly one row changes, commit, then re-read. Route only after shadow succeeds and current-byte fallback metadata is durable. -- [ ] **Step 5: Implement upgrade quarantine transaction** +- [x] **Step 5: Implement upgrade quarantine transaction** When a new client version is detected, pause enrollment and destructive operations, create and verify current native backing for routed sessions, then atomically route those sessions to native files before marking the version safe to launch writes. -- [ ] **Step 6: Run focused, race, and complete tests** +- [x] **Step 6: Run focused, race, and complete tests** Run: @@ -430,7 +458,7 @@ go test ./... -count=1 Expected: all PASS. -- [ ] **Step 7: Commit** +- [x] **Step 7: Commit** ```bash git add internal/compat internal/codex @@ -453,25 +481,25 @@ git commit -m "feat: add Codex compatibility and route transactions" - Consumes: session manager and the native-operation contract. - Produces: a testable platform-neutral filesystem and `Mount(ctx, Options) error` adapter boundary. -- [ ] **Step 1: Write failing filesystem operation tests** +- [x] **Step 1: Write failing filesystem operation tests** Exercise root listing, regular-file stat, open flags, sequential/random read, append, fsync, flush/release, truncate, random write, rename/unlink policy, stable handles, concurrent readers, and single writer. -- [ ] **Step 2: Run focused tests and verify missing API failures** +- [x] **Step 2: Run focused tests and verify missing API failures** Run: `go test ./internal/mountfs -count=1` Expected: FAIL because the package is absent. -- [ ] **Step 3: Implement platform-neutral operation methods** +- [x] **Step 3: Implement platform-neutral operation methods** Keep all path validation, handle ownership, session lookups, and error mapping independent of FUSE. Expose operations through Go methods returning `syscall.Errno`; do not put storage logic in the adapter. -- [ ] **Step 4: Add `cgofuse` v1.6.0 behind explicit build constraints** +- [x] **Step 4: Add `cgofuse` v1.6.0 behind explicit build constraints** `host_cgofuse.go` uses `//go:build fuse && cgo` and translates cgofuse callbacks to the neutral filesystem. `host_stub.go` uses `//go:build !fuse || !cgo` and returns a typed prerequisite error. Default `go test ./...` and cross-compilation must not require an installed FUSE host. -- [ ] **Step 5: Run default, race, and cross-platform compile tests** +- [x] **Step 5: Run default, race, and cross-platform compile tests** Run: @@ -485,7 +513,7 @@ go test ./... -count=1 Expected: all PASS without an installed FUSE host. -- [ ] **Step 6: Commit** +- [x] **Step 6: Commit** ```bash git add internal/mountfs go.mod go.sum @@ -505,30 +533,34 @@ git commit -m "feat: add platform filesystem and tagged fuse host" - Consumes: pack, fsctl, compat, codex route, vfs, and mountfs packages. - Produces: the public `codexfold pack` and `codexfold fs` command contracts. -- [ ] **Step 1: Write failing command-surface and dry-run tests** +- [x] **Step 1: Write failing command-surface and dry-run tests** Assert the complete public command tree exists, status/doctor/compatibility/benchmark are read-only, migrate/rollback/compact require `--apply`, and bulk enrollment excludes active or changing sessions until their promotion policy allows them. -- [ ] **Step 2: Run CLI tests and verify command failures** +- [x] **Step 2: Run CLI tests and verify command failures** Run: `go test ./internal/cli -run 'TestPack|TestFS|TestRoot' -count=1` Expected: FAIL because `pack` and `fs` commands are missing. -- [ ] **Step 3: Implement `pack` and read-only `fs` commands** +- [x] **Step 3: Implement `pack` and read-only `fs` commands** Expose `pack build`, `pack doctor`, `fs status`, `fs doctor`, `fs compatibility`, and `fs benchmark` with JSON output. Status remains `storage-engine` until preview gates are actually met. -- [ ] **Step 4: Implement guarded lifecycle commands** +- [x] **Step 4: Implement guarded lifecycle commands** Expose `fs serve`, `fs migrate`, `fs rollback`, `fs compact`, and `fs recover`. Mutation commands are dry-run by default and require `--apply`. Migration requires clean doctor, passing shadow evidence, approved client version, and eligible session state. - [ ] **Step 5: Implement bounded automatic discovery** +Current state: missing. Session discovery primitives exist, but no stable-session policy loop or bounded automatic enrollment transaction exists. + Discover existing, new, and forked Codex sessions from state; keep active native paths directly openable; enqueue only stable eligible sessions for shadow. Never require per-session production setup. - [ ] **Step 6: Run CLI, race, and complete tests** +Current state: the existing CLI, race, build, and complete tests pass, but this step remains open until automatic-enrollment tests are present and passing. + Run: ```bash @@ -541,7 +573,7 @@ go build ./cmd/codexfold Expected: all PASS. -- [ ] **Step 7: Commit** +- [x] **Step 7: Commit the implemented standalone CLI surface** ```bash git add internal/cli @@ -561,25 +593,25 @@ git commit -m "feat: expose transparent filesystem command surface" - Consumes: built tagged binary, mount path, installed-client compatibility result, and doctor status. - Produces: deterministic service definition, start/stop/status, and update preflight. -- [ ] **Step 1: Write failing service-render and update-guard tests** +- [x] **Step 1: Write failing service-render and update-guard tests** Assert launchd arguments are absolute, logs contain no session content, daemon and mount health are separate, preview auto-update is rejected, and a client version change enters quarantine before restart. -- [ ] **Step 2: Run focused tests and verify missing service API** +- [x] **Step 2: Run focused tests and verify missing service API** Run: `go test ./internal/service ./internal/cli -run 'TestService|TestFSService' -count=1` Expected: FAIL because service APIs are absent. -- [ ] **Step 3: Implement service lifecycle without self-elevation** +- [x] **Step 3: Implement service lifecycle without self-elevation** Render a per-user launchd plist and use `launchctl bootstrap/bootout/kickstart` only after an explicit apply command. Detect prerequisites but never install FUSE-T or request elevation from library code. -- [ ] **Step 4: Implement update compatibility guard** +- [x] **Step 4: Implement update compatibility guard** Before service binary promotion, run doctor and compatibility against installed clients. Preview/canary versions require explicit promotion. Unknown client versions trigger Task 6 quarantine and current-byte native routing. -- [ ] **Step 5: Run focused and complete tests** +- [x] **Step 5: Run focused and complete tests** Run: @@ -590,7 +622,7 @@ go test ./... -count=1 Expected: all PASS. -- [ ] **Step 6: Commit** +- [x] **Step 6: Commit** ```bash git add internal/service internal/cli/fs.go @@ -610,15 +642,15 @@ git commit -m "feat: add guarded filesystem service lifecycle" - Consumes: all platform-neutral packages and tagged stubs. - Produces: reproducible gate report for `fs-engine-preview`; does not claim real adapter or Codex readiness. -- [ ] **Step 1: Add a deterministic fork/session corpus generator** +- [x] **Step 1: Add a deterministic fork/session corpus generator** Generate exact repeated fields at arbitrary positions, repeated JSONL records, forked histories, large multi-block fields, independent append tails, random writes, truncation, invalid JSONL, and empty sessions without using real user content. -- [ ] **Step 2: Add fault injection and stress tests** +- [x] **Step 2: Add fault injection and stress tests** Run 10,000 random reads, 100,000 appends, concurrent reader/single-writer race tests, every journal phase interruption, resolver corruption, daemon-equivalent reopen, route transaction races, and current-byte fallback validation. -- [ ] **Step 3: Run complete correctness and race gates** +- [x] **Step 3: Run complete correctness and race gates** Run: @@ -633,15 +665,15 @@ CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build ./cmd/codexfold Expected: all PASS. -- [ ] **Step 4: Run packed and virtual benchmarks** +- [x] **Step 4: Run packed and virtual benchmarks** Compare the same generated 758 MiB rollout through native and virtual reads, record warm/cold throughput, p50/p95/p99, CPU, RSS, cache budget, and loose-object open count. Failure leaves status below `fs-engine-preview`. -- [ ] **Step 5: Document exact evidence and limitations** +- [x] **Step 5: Document exact evidence and limitations** Record commands, hardware, versions, results, and unmet real-adapter gates. Do not call fixture results transparent, canary, or production-ready. -- [ ] **Step 6: Commit** +- [x] **Step 6: Commit** ```bash git add internal/testfs scripts/test-cross-platform.sh docs/validation-fs-preview.md @@ -652,38 +684,42 @@ git commit -m "test: validate transparent filesystem engine preview" **Files:** - Create: `docs/validation-macos-canary.md` -- Modify only after approval: local FUSE-T prerequisite and user launch service outside the public repository. +- Modify only after approval: local FUSE-T prerequisite and the standalone per-user launch service state. - Modify only after all gates pass: selected Codex state rows through `codexfold fs migrate --apply`. **Interfaces:** - Consumes: completed Tasks 1 through 10, explicit system-extension authorization, real Codex versions, selected archived sessions, and retained native snapshots. - Produces: actual `platform-canary` evidence or an explicit blocked result; no stronger status without seven-day retention. -- [ ] **Step 1: Capture native Codex Desktop and CLI operations** +- [x] **Step 1: Capture native Codex Desktop and CLI operations** With explicit elevation approval, run sanitized `fs_usage` tracing for list/open/read/pread/append/fsync/truncate/rename/unlink/lock/watcher behavior during history display, resume, message send, tool use, fork, archive, unarchive, repair, restart, and upgrade. Import the trace into a machine-readable compatibility contract. -- [ ] **Step 2: Reconcile the adapter with every observed operation** +- [x] **Step 2: Reconcile the adapter with every observed operation** Add or correct platform operation tests before changing adapter code. Any unsupported observed operation blocks installation and migration. -- [ ] **Step 3: Request and apply the selected FUSE host authorization** +- [x] **Step 3: Request and apply the selected FUSE host authorization** Install the selected prerequisite only after explicit approval, build with `-tags fuse`, mount a temporary fixture namespace, and run fstest/fsx-equivalent plus the project operation suite. A mount alone is not success. -- [ ] **Step 4: Run real-session shadow without changing Codex routes** +- [x] **Step 4: Run real-session shadow without changing Codex routes** Select 5–10 archived sessions, fold and pack without source removal, compare every byte and 10,000 random ranges, run doctor and benchmark, then keep Codex on native routes. Any mismatch stops the task. - [ ] **Step 5: Route retained-source canaries** +Current state: isolated retained-source CLI and Desktop canaries passed direct open, resume, append, tool use, fork, archive/unarchive, daemon restart, rollback, re-migration, and quarantine. This step remains open because managed-session sleep/wake, full host restart, current installed-client compatibility, and real-home retained-source canaries are not complete. + After clean shadow and compatibility, migrate only the selected archived sessions. Verify Desktop direct click, CLI resume, history, message send, tool use, fork, archive, unarchive, daemon termination, mount restart, sleep/wake, host restart, rollback, and compatibility quarantine. Never delete native snapshots. - [ ] **Step 6: Start seven-day canary retention** +Current state: not started because the project has not reached `platform-canary`. + Record daemon/mount health, exact-byte doctor, recovery incidents, client versions, and performance. Status remains `platform-canary` during retention; `production-ready:macos` requires the full period with zero unresolved incidents. -- [ ] **Step 7: Commit only public sanitized evidence** +- [x] **Step 7: Commit only public sanitized evidence** ```bash git add docs/validation-macos-canary.md @@ -692,11 +728,69 @@ git commit -m "docs: record macOS transparent filesystem canary" No private path, session ID, trace content, credential, or control-plane name may enter the public evidence. +### Task 12: Bounded Automatic Discovery And Enrollment + +**Status:** Missing. This completes Task 8 Step 5 and is required before production operation can be called automatic. + +**Requirements:** `TF-001`, `TF-010`, `TF-011`, `TF-014`, `TF-015`, `TF-021`. + +**Exact next work:** + +- [ ] Add policy tests for existing sessions, newly created sessions, forks, active writers, changing files, archived eligibility, unknown client versions, failed doctor state, insufficient disk budget, bounded batches, restart idempotency, and failed cutover. +- [ ] Implement a read-only enrollment planner that consumes Codex state, rollout stability evidence, compatibility, doctor, writer state, promotion stage, and storage-budget preflight, and emits explicit eligible/ineligible reasons without changing routes. +- [ ] Implement bounded apply transactions that fold, pack, shadow, stage at most one retained native snapshot, wait for exact mount acknowledgement, and only then update routing. A failure leaves the original native route and source unchanged. +- [ ] Integrate the planner into the standalone service with a bounded interval and batch size. Newly created sessions and forks remain native while active and need no per-session command when they later become eligible. +- [ ] Validate in an isolated Codex home across daemon restart and client-version quarantine before any real-home enrollment is allowed. + +### Task 13: Conservative Branch Lifecycle And Content-Change Boundary + +**Status:** Partial. Exact containment deletion and separate repair/reconciliation outputs exist; conservative family classification and guarded archive execution are missing. + +**Requirements:** `TF-018`, `TF-019`, `TF-020`. + +**Exact next work:** + +- [ ] Add read-only fork-family reports that distinguish shared exact content, independent tails, complete containment, active/archived state, and unknown relationships. Never label a branch useless from ancestry, age, title, or size alone. +- [ ] Trace and test the current official Codex archive operation, then add a dry-run-first archive mutation that requires an explicit session selection, revalidates database route and source digest, preserves the rollout, and updates file and state atomically. +- [ ] Keep `remove-contained` as a separate archived-only operation and add integration coverage proving that family classification or archive never triggers deletion automatically. +- [ ] Add CLI regression tests proving `repair-rollout` and `reconcile-rollout` require explicit separate outputs and cannot be called by fold, migrate, compact, enrollment, rollback, or GC paths. + +### Task 14: Hard Storage Budgets, Retention, Cleanup, And Reclamation Accounting + +**Status:** Missing. Current correctness paths create verified snapshots and retirement state, but no product-wide hard budget or bounded retirement cleanup policy exists. + +**Requirements:** `TF-009`, `TF-014`, `TF-021`. + +**Exact next work:** + +- [ ] Add a platform-neutral storage inventory that accounts separately for logical session bytes, unique loose objects, packs, native sources, retained snapshots, current fallbacks, writable backings, old generations, retirement state, journal-owned recovery files, and unowned temporary files. +- [ ] Add preflight APIs that calculate projected peak bytes and reject fold, pack, migrate, rollback, compact, enrollment, and content-changing output before writing when the hard temporary budget or free-space reserve would be exceeded. +- [ ] Enforce one immutable migration snapshot and one current writable fallback per managed session, one full-session scratch file per transaction, and current-plus-previous pack-generation retention until leases close. +- [ ] Add startup and explicit GC for abandoned temporary files, expired unleased generations, and retired state whose journal and retention proofs allow removal. Never remove the sole recoverable generation. +- [ ] Extend status, doctor, and mutation results with projected versus actual physical reclamation. Add low-space, interrupted-cleanup, retained-fallback, and repeated-enrollment tests that prove disk use remains bounded. + +### Task 15: Remaining Platform And Retention Gates + +**Status:** Missing as release evidence; it does not block continued engine development but blocks stronger capability claims. + +**Requirements:** `TF-003`, `TF-008`, `TF-009`, `TF-011`, `TF-012`, `TF-014`, `TF-015`, `TF-017`, `TF-021`. + +**Exact next work:** + +- [ ] Import exact compatibility contracts for the currently installed Codex Desktop and CLI versions and return `fs doctor` to a clean client state. +- [ ] Run retained-source managed macOS canaries through sleep/wake and real host restart, including interrupted append, compaction, migration, and rollback cases required by the contract. +- [ ] Activate only bounded real-home canaries after Tasks 12 and 14 pass, then complete seven incident-free days before any `platform-canary` promotion decision. +- [ ] Implement and execute real Linux FUSE3 and Windows WinFsp adapters and their independent operation, crash, performance, upgrade, and rollback gates. + ## Plan Self-Review -- `TF-001` through `TF-017` each map to implementation and verification tasks. -- Real-client, real-adapter, restart, upgrade, and retention gates remain in Task 11 and cannot be satisfied by Task 10 fixtures. +- `TF-001` through `TF-022` each map to implementation and verification tasks or an explicitly identified storage-engine baseline. +- Real-client, real-adapter, restart, upgrade, and retention gates remain in Tasks 11 and 15 and cannot be satisfied by Task 10 fixtures. - FUSE-T is the validated macOS host and remains an explicit authorization boundary; Linux and Windows adapters are certified independently. - The stale migration snapshot is never used as current fallback after virtual writes diverge. - The default build remains portable and does not require installed FUSE headers. - No task changes a real Codex route before shadow, compatibility, doctor, and explicit apply gates pass. +- Automatic enrollment remains blocked until Task 12 and Task 14 are complete. +- Branch archival, exact-contained deletion, and content-changing repair remain separate operations. +- No logical deduplication result is presented as physical reclamation without storage accounting. +- Public product behavior remains independent of any private control plane. diff --git a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md index 2f1db89..0c94334 100644 --- a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md +++ b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md @@ -17,7 +17,7 @@ Every implementation plan, task, test report, release note, and control-plane st | `TF-001` | Opening or resuming a managed session requires no manual materialization or preparation command. | | `TF-002` | Unmodified Codex Desktop and Codex CLI access a normal regular-file JSONL path and do not know that storage is virtual. | | `TF-003` | Every byte and every file operation Codex actually uses has native-equivalent observable behavior. Platform readiness is blocked by any unsupported operation used by Codex. | -| `TF-004` | Identical content across sessions and forks is stored once in the shared object store; session histories remain independently writable. | +| `TF-004` | Identical byte content across sessions and forks is stored once in the shared object store; session histories remain independently writable. Reuse applies to exact repeated fields, records, and content-defined chunks at arbitrary positions and is not limited to a shared file prefix. | | `TF-005` | Normal append writes go to a durable delta without complete base materialization. | | `TF-006` | Truncate, random write, and every non-append mutation that can be represented safely must transition to copy-on-write before success. A mutating operation used by Codex may not be silently rejected in a production-ready adapter. | | `TF-007` | Packed runtime reads do not open loose object files per manifest part and do not perform a persistent-index lookup per object. The concrete durable and runtime index formats are selected by implementation evidence and must satisfy the performance and recovery gates. | @@ -31,6 +31,11 @@ Every implementation plan, task, test report, release note, and control-plane st | `TF-015` | A Codex client version without passing compatibility evidence enters compatibility quarantine: enrollment and destructive automation pause, and an already-routed session is automatically switched to a byte-verified current native writable backing before that client version may write. Routine client upgrades require no manual session preparation. | | `TF-016` | Platform filesystem prerequisites requiring elevated or system-extension approval are installed only after explicit user authorization. | | `TF-017` | A canonical mount may never degrade into a writable ordinary directory or expose stale session files. The unmounted backing directory is empty and write-sealed, activation requires a live CodexFold mount identity, and service start succeeds only after the daemon and operational mount probe are both healthy. Desktop realpath rewrites from `CODEX_HOME/sessions` or `archived_sessions` into the mount alias are synchronously normalized in the Codex state database, and the route watcher accepts either spelling without exiting. | +| `TF-018` | Fork-family classification and branch archival are conservative, evidence-driven, and dry-run-first. Fork ancestry, age, size, or title alone never proves that a branch is useless. An archive mutation requires explicit user selection or an explicit policy, revalidates current Codex state and rollout bytes, and preserves a recoverable session. | +| `TF-019` | Deleting a fully duplicated branch is limited to an archived session whose applicable JSONL record sequence is proven exactly and completely contained in another retained session. Apply additionally requires current-source and temporary-unfold recovery proof, transactional Codex state cleanup, and a retained tombstone and manifest. Similarity and fork ancestry are not deletion evidence. | +| `TF-020` | Byte-preserving storage optimization never changes rollout bytes. Repair, reconciliation, prompt cleanup, message removal, or any other content-changing operation is a separate explicit workflow that writes a separately verified output and is never run implicitly by fold, migration, compaction, enrollment, GC, or rollback. | +| `TF-021` | Full-size transaction files, retained native snapshots, writable fallbacks, old pack generations, recovery artifacts, and temporary files are governed by hard preflight and retention budgets. Successful and abandoned temporary artifacts are cleaned automatically. Status and completion reports separate logical bytes, physical store bytes, retained source/fallback bytes, temporary/recovery bytes, projected reclamation, and actual reclaimed bytes; no physical-saving claim is allowed while equivalent full copies still occupy disk. | +| `TF-022` | CodexFold is a standalone public product. Its public code, CLI, daemon, service definitions, configuration, storage formats, doctor, GC, rollback, and enrollment contain no dependency on or reference to a private control plane. External package managers or operators may install and supervise CodexFold only from outside the product boundary. | ### Decision Hierarchy @@ -47,7 +52,7 @@ An implementation change is therefore acceptable only when its requirement cover ### Drift Control - Each implementation-plan task lists the requirement IDs it implements or verifies. -- Each plan starts with a complete requirement-to-task coverage table for `TF-001` through `TF-017`. +- Each plan starts with a complete requirement-to-task coverage table for `TF-001` through `TF-022`. - A requirement with no implementation or verification task blocks plan approval. - Completion reports list fresh evidence by requirement ID and state any unmet ID explicitly. - Mock, fixture, or synthetic evidence cannot satisfy a requirement that names real Codex, a real platform adapter, a client upgrade, a host restart, or a canary period. @@ -92,6 +97,11 @@ Passing unit tests, successful materialization, a mounted filesystem, one succes - Claiming identical APFS, ext4, or NTFS metadata that Codex does not observe. - Migrating real user sessions during engine development. - Deleting native fallbacks before canary and rollback gates pass. +- Treating strict byte-prefix ancestry as the only reusable-content shape. +- Automatically deciding that a branch is useless from age, title, size, or fork ancestry. +- Running repair, reconciliation, prompt cleanup, or other content-changing transforms as part of byte-preserving optimization. +- Claiming physical disk savings from logical deduplication while retained sources, fallbacks, recovery copies, or temporary files still occupy the same bytes. +- Requiring a private deployment or control-plane product at runtime. ## Architecture @@ -288,7 +298,7 @@ Platform readiness is independent. Passing macOS gates does not imply Linux or W The platform-neutral core defines byte layout and transaction behavior, not a lowest-common-denominator filesystem API. Each adapter must implement the strongest native semantics Codex uses on that platform; macOS behavior may not be weakened to match Windows or Linux limitations. -macFUSE, FUSE3, and WinFsp are initial candidates rather than product promises. If native-operation traces or platform gates disqualify a candidate, it must be replaced without weakening `TF-001` through `TF-016`. +FUSE-T, FUSE3, and WinFsp are current candidates rather than product promises. If native-operation traces or platform gates disqualify a candidate, it must be replaced without weakening `TF-001` through `TF-022`. ## Migration And Rollback @@ -326,6 +336,43 @@ After platform production readiness, enrollment is policy-driven rather than man - No user action is required to enroll, open, resume, fork, compact, or re-enroll a normal session. - Enrollment failure leaves the native database route and source file unchanged. +## Branch Cleanup And Content-Change Boundary + +Storage sharing, branch archival, exact-contained deletion, and content-changing repair are four separate operations: + +1. **Storage sharing** preserves every byte and may reuse exact fields, records, or content-defined chunks found anywhere in any session. +2. **Branch classification and archival** reports evidence first. It may recommend an archive candidate, but it never mutates from ancestry, age, title, or size alone and never removes recovery ability. +3. **Exact-contained deletion** applies only to an already archived session after complete direct containment and recovery proof. It is not a side effect of folding, packing, enrollment, compaction, or GC. +4. **Repair, reconciliation, and prompt cleanup** change content and therefore write a separate verified output. They never replace either source implicitly and never participate in byte-identical savings claims. + +## Storage Budget And Reclamation Accounting + +Before any operation that can create a full-size session copy or a new store generation, CodexFold calculates its projected peak physical bytes and checks the configured hard budget and required free-space reserve. Automatic enrollment is disabled until this preflight is implemented and passes. + +The default retention model is cardinality-bounded: + +- A managed session has at most one immutable migration snapshot and at most one current native writable fallback. A current fallback must replace or reuse stale current-fallback state rather than accumulate another full copy. +- One transaction may create at most one full-session scratch file for the affected session. Named historical copies such as `native-before`, `fold-before`, `merged`, and `repaired` are not implicit recovery generations. +- Pack publication retains the current generation and only the immediately previous verified generation while a lease or rollback window requires it. Older unleased generations are GC candidates. +- Startup recovery removes abandoned temporary artifacts only after journal analysis proves that they are not the sole committed or recoverable generation. + +Every mutating command reports, before and after apply: + +```text +logical session bytes +unique object bytes +pack bytes +native source bytes +retained snapshot bytes +current fallback bytes +temporary and recovery bytes +projected peak bytes +projected reclaimable bytes +actual reclaimed bytes +``` + +Logical duplicate savings and physical disk reclamation are distinct metrics. A fold or migration may report logical reuse while reporting zero actual reclamation when source or fallback copies are still retained. + ## Failure Semantics - Pack, manifest, delta, backing, and journal commits use temporary files, synchronization, and atomic replacement. @@ -339,6 +386,8 @@ After platform production readiness, enrollment is policy-driven rather than man - A store has one filesystem-host process lock. Service installation and restart return success only after launchd reports a running process and the mount identity is readable. - Database and global-state changes use optimistic revalidation and rollback. - A session with an active writer is never folded, removed, migrated, or rolled back. +- A branch is never archived or removed solely from inferred fork ancestry, age, title, or size. +- Budget preflight failure blocks the mutating operation before any full-size temporary file is created. - A detected Codex Desktop or CLI version change immediately enters compatibility quarantine and schedules the native-operation compatibility suite. - Quarantine pauses enrollment, migration, compaction that removes fallback state, fallback deletion, and GC. - Before an unapproved client version may write an already-routed session, the service automatically switches it to a verified current native writable backing. It never routes the stale migration snapshot as current data. @@ -446,6 +495,7 @@ Platform adapters translate native filesystem calls only. They do not fork or re - Pack and delta files use user-only permissions. - Mount access is restricted to the owning user. - Management operations require explicit apply flags for migration, rollback, fallback deletion, and GC. +- Public runtime behavior and configuration have no private control-plane dependency. - Crash reports and benchmark output must not include raw rollout data. ## CLI Contract diff --git a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-traceability-review.md b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-traceability-review.md index 8c8ec54..271c728 100644 --- a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-traceability-review.md +++ b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-traceability-review.md @@ -13,6 +13,7 @@ The purpose is to prevent implementation plans from replacing the requested outc | Codex opens a normal JSONL path without manual materialization | `TF-001`, `TF-002`, Product Promise | Aligned | | The Codex client is not patched and does not know storage is virtual | `TF-002`, Scope | Aligned | | Duplicate content across sessions and forks is stored once | `TF-004`, Reference Packfile Design | Aligned | +| Exact repeated fields, records, and chunks can be shared even when they are not a strict file prefix | `TF-004`, Branch Cleanup And Content-Change Boundary | Corrected: the contract now states explicitly that strict prefix ancestry is not the only reusable shape | | A session can be read at arbitrary offsets without complete materialization | `TF-003`, Virtual File Model, File Operation Contract | Aligned | | Append writes use a durable delta and do not hydrate the complete base | `TF-005`, Append And Copy-On-Write | Aligned | | Truncate and random writes automatically move to a complete writable backing | `TF-006`, Append And Copy-On-Write | Corrected: the first spec allowed fail-closed as an equal outcome; production behavior now requires copy-on-write for safely representable mutations | @@ -29,20 +30,25 @@ The purpose is to prevent implementation plans from replacing the requested outc | The stable workflow is automatic rather than one manual migrate per session | `TF-011`, Migration And Rollback | Corrected: automatic discovery and enrollment of existing sessions, new sessions, and forks was missing from the first spec | | Codex upgrades must not silently change supported file behavior | `TF-015`, Native Behavior Discovery, Behavioral Gates | Aligned | | Every Codex Desktop or CLI version change quarantines virtual writes until current bytes are automatically routed to verified native backing or compatibility passes | `TF-015`, Failure Semantics | Corrected: routine upgrades remain automatic without exposing unknown write behavior to virtual storage | -| macOS, Linux, and Windows use independently validated platform adapters | `TF-012`, Platform Adapters | Aligned; macFUSE, FUSE3, and WinFsp are current candidates, not product promises | +| macOS, Linux, and Windows use independently validated platform adapters | `TF-012`, Platform Adapters | Aligned; FUSE-T, FUSE3, and WinFsp are current candidates, not product promises | | Shared storage, read, write, generation, doctor, and recovery behavior stays in the platform-neutral core | `TF-012`, Platform-Neutral Core Contract | Aligned as a responsibility boundary; internal formats and algorithms remain replaceable | | Each platform is certified independently | `TF-012`, Canonical Status Terms, Platform Adapters | Aligned | | Windows handles share mode, oplock, replace, Defender, case-insensitive paths, service restart, and mount naming | `TF-003`, `TF-012`, Windows adapter | Corrected: mount namespace or drive-letter behavior was added | | iOS and Android access a desktop host and do not host this local filesystem | Scope, Mobile | Corrected: the mobile boundary was missing | -| macFUSE or other elevated prerequisites require explicit user approval | `TF-016`, Platform Adapters | Aligned | +| FUSE-T or other privileged prerequisites require explicit user approval | `TF-016`, Platform Adapters | Aligned | | Status language must not call the storage engine transparent or production-ready | `TF-013`, Canonical Status Terms | Aligned | +| Useless or closed fork branches may be classified and archived conservatively, but ancestry or age cannot decide the mutation | `TF-018`, Branch Cleanup And Content-Change Boundary | Added: the original cleanup goal was not represented as a non-negotiable requirement | +| An archived branch that is exactly and completely contained in another retained session can be removed only after recovery proof | `TF-019`, Branch Cleanup And Content-Change Boundary | Added: the existing containment implementation is now protected by the product contract | +| Prompt cleanup, repair, and reconciliation are content-changing workflows and must not be confused with byte-preserving folding | `TF-020`, Branch Cleanup And Content-Change Boundary | Added: the implementation already separates outputs, but the contract did not prevent future drift | +| Temporary copies, recovery generations, retained snapshots, and claimed savings require bounded physical-space accounting | `TF-021`, Storage Budget And Reclamation Accounting | Added: logical deduplication was previously specified without a hard physical-space contract | +| CodexFold remains an independent public product even when an external operator installs or supervises it | `TF-022`, Security And Privacy | Added: private deployment policy cannot enter the public runtime architecture | ## Open Engineering Questions That Do Not Change The Goal These questions require evidence during implementation. They are not permission to weaken a requirement: - The exact native Codex operation trace on each client version. -- Whether the current macFUSE candidate can satisfy the observed `mmap`, lock, watcher, and cache behavior. +- Whether the current FUSE-T adapter continues to satisfy every operation introduced by future Codex versions. - The optimal immutable pack size and decompressed-cache admission policy within `TF-008` limits. - The idle window that prevents compaction from racing a resumed writer. - The maximum mount-recovery time that remains acceptable during canary. @@ -65,4 +71,4 @@ The corrected contract matches the approved outcome. The intent-level review fou 9. Client upgrades enter compatibility quarantine and route current bytes to verified native writable backing before unknown writes. 10. Adapter products and cache/index algorithms are reference choices rather than product promises. -After these corrections, no known goal-level drift remains in the design contract. Implementation details may still improve, but they must preserve the fixed outcome, invariants, and gates. This is a contract conclusion, not a claim that transparent filesystem implementation or production validation is complete. +The 2026-07-14 alignment added the cleanup, physical-space, and standalone-product commitments that were present in the original product discussion but absent from the first review. After those additions, no known goal-level drift remains in the design contract. Implementation gaps remain, especially automatic enrollment, conservative branch classification and archive execution, hard disk-budget enforcement, real Linux and Windows adapters, managed-session host-restart and sleep/wake validation, and canary retention. This is a contract conclusion, not a claim that transparent filesystem implementation or production validation is complete. diff --git a/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md b/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md new file mode 100644 index 0000000..56747e9 --- /dev/null +++ b/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md @@ -0,0 +1,98 @@ +# Transparent Session Filesystem Implementation Alignment + +## Purpose + +This document aligns the original product commitments, the canonical transparent-filesystem contract, the implementation plan, the current repository, and the available validation evidence. + +It does not redesign CodexFold, authorize real-session enrollment, promote the capability above `fs-engine-preview`, or treat fixtures as production evidence. CodexFold remains a standalone public product. External installation or supervision is outside its runtime architecture. + +Baseline reviewed: commit `045eea1` on 2026-07-14. + +## Original Commitment To Requirement Mapping + +| Original product commitment | Canonical requirement | Alignment result | +| --- | --- | --- | +| Codex Desktop and CLI open and resume a normal JSONL path without a materialization step | `TF-001`, `TF-002`, `TF-003` | Preserved | +| The client remains unmodified and cannot distinguish managed storage from a supported native file | `TF-002`, `TF-003` | Preserved | +| Exact duplicate content is stored once across sessions and forks | `TF-004` | Preserved | +| Reuse is not limited to a strict shared prefix; repeated fields, records, and chunks at arbitrary positions are shareable | `TF-004` | Clarified in the contract | +| Forks remain independently writable after sharing common content | `TF-004`, `TF-005`, `TF-006` | Preserved | +| Normal writes append to a durable delta; non-append mutations use safe copy-on-write | `TF-005`, `TF-006` | Preserved | +| Runtime reads use packed storage rather than tens of thousands of loose-object opens | `TF-007`, `TF-008` | Preserved | +| Correctness includes performance, bounded memory, crash recovery, restart recovery, and exact rollback | `TF-008`, `TF-009`, `TF-010` | Preserved | +| Stable production operation discovers existing sessions, new sessions, and forks automatically | `TF-011` | Preserved; implementation missing | +| macOS, Linux, and Windows share one storage engine but have independent adapters and readiness gates | `TF-012`, `TF-016`, `TF-017` | Preserved | +| Capability language cannot overstate a storage engine, preview, or one successful canary | `TF-013` | Preserved | +| Native sources and current recoverable bytes remain available until the relevant gates pass | `TF-010`, `TF-014`, `TF-015` | Preserved | +| Useless or closed branches can be identified and archived, but the tool must not guess destructively | `TF-018` | Added to the contract; implementation missing | +| A branch that is exactly 100% contained in another retained session can be deleted only after exact recovery proof | `TF-019` | Added to the contract; implementation exists | +| Prompt cleanup, repair, and reconciliation are separate content-changing workflows, not storage folding | `TF-020` | Added to the contract; implementation boundary exists, regression coverage is incomplete | +| Temporary files, recovery generations, retained snapshots, and repeated operations must not consume unbounded disk | `TF-021` | Added to the contract; implementation missing | +| Reported savings distinguish logical reuse from actual physical bytes reclaimed | `TF-021` | Added to the contract; implementation missing | +| CodexFold is an independent open-source product with no private control-plane dependency | `TF-022` | Added to the contract; current repository is aligned | + +## Requirement To Implementation And Evidence + +| Requirement | Current implementation | Tests or evidence | Status | +| --- | --- | --- | --- | +| `TF-001` | `internal/cli/fs.go`, `internal/mountfs`, canonical migration and routing | Isolated CLI/Desktop direct-open and resume canaries | Partial: verified for isolated macOS canaries; automatic general enrollment is missing | +| `TF-002` | `internal/mountfs`, `internal/sessionns`, `internal/mountid` | Real FUSE-T operation tests and isolated unmodified clients | Implemented for the validated macOS client versions | +| `TF-003` | Neutral operation layer plus exact compatibility contracts in `internal/compat` | Real macOS traces and adapter canaries | Partial: current installed clients need fresh contracts; Linux and Windows are not validated | +| `TF-004` | `internal/scan`, `internal/cdc`, `internal/fold`, `internal/pack` | Repeated field, record, CDC, fork, and non-prefix corpus tests | Implemented | +| `TF-005` | `internal/vfs` append delta and writer leases | Append-without-hydration tests and real CLI/Desktop append evidence | Implemented | +| `TF-006` | `internal/vfs` copy-on-write backing and neutral write operations | Random-write, truncate, interruption, and real FUSE-T mutation tests | Implemented | +| `TF-007` | Immutable packs, in-memory index, bounded cache, random-read resolver | Pack round-trip/corruption tests and 758 MiB packed-read benchmark | Implemented | +| `TF-008` | `internal/fsctl` benchmark and `internal/testfs` stress harness | `docs/validation-fs-preview.md` | Partial: shared-core gates pass; full real-adapter metrics and other platforms remain open | +| `TF-009` | Journal recovery, generation recovery, service keep-alive, restart-safe retirement | Recovery tests, daemon restart canaries, actual host boot of service and mount | Partial: no managed-session sleep/wake or full-host restart gate | +| `TF-010` | Shadow compare, optimistic routes, retained snapshots, current-byte fallback | 90,000 real random-range comparisons, rollback and failure-containment canaries | Implemented for isolated canaries | +| `TF-011` | Codex state discovery primitives exist in `internal/codex` | Discovery unit tests | Missing: no stability policy, batch planner, or automatic enrollment loop | +| `TF-012` | Shared Go core and macOS FUSE-T adapter | macOS real adapter tests; Linux and Windows non-CGO compile checks | Partial: Linux and Windows real adapters are missing | +| `TF-013` | Canonical capability type in `internal/fsctl/status.go` | Status rejection tests and CLI status tests | Implemented; current status is `fs-engine-preview` | +| `TF-014` | Snapshot retention and destructive-action guards | Migration, rollback, and quarantine tests | Implemented as a safety rule; retention promotion gates remain open | +| `TF-015` | Exact-version compatibility and update preflight quarantine | Unknown-version fallback and isolated canary tests | Implemented; the currently installed clients are presently uncovered | +| `TF-016` | Tagged adapter prerequisite errors and non-elevating service lifecycle | Stub, service, and authorization-gated FUSE-T evidence | Implemented | +| `TF-017` | Canonical namespace, write-sealed backing, mount identity, route normalization, process lock | Neutral, real FUSE-T, launchd, Desktop restart, and rollback tests | Implemented for macOS canaries | +| `TF-018` | Canonical archive/unarchive file operations exist, but no conservative family classifier or guarded archive product workflow exists | No qualifying end-to-end tests | Missing | +| `TF-019` | `internal/contain` and `internal/prune`; public `contains` and `remove-contained` commands | Exact containment, archived-only apply, transaction rollback, and recovery-manifest tests | Implemented | +| `TF-020` | Exact fold/migrate paths are byte-preserving; `repair-rollout` and `reconcile-rollout` write separate explicit outputs | `internal/reconcile` and repair tests | Partial: add direct CLI boundary and non-invocation regression tests | +| `TF-021` | Individual temporary files are transactional, but no global inventory, hard preflight budget, bounded retired-state cleanup, or truthful reclamation report exists | No qualifying product-wide tests | Missing | +| `TF-022` | Standalone CLI, daemon, launchd renderer, configuration, storage, doctor, GC, rollback, and enrollment code | Public coupling scan and sanitization test | Implemented | + +## Implementation Plan Task Status + +| Task | Status | Evidence | Exact remaining scope | +| --- | --- | --- | --- | +| Task 1: packed object generation and resolver | Complete | Commit `17564e9`; `internal/pack` tests pass | None in Task 1 | +| Task 2: exact immutable virtual byte view | Complete | Commit `039e6b9`; exact and 10,000 random-read tests pass | None in Task 2 | +| Task 3: append and copy-on-write engine | Complete | Commit `9a7f1e8`; append, COW, writer, reopen, and interruption tests pass | None in Task 3 | +| Task 4: journal, compaction, and fallback | Complete | Commit `076d772`; recovery, compaction, and latest-byte fallback tests pass | None in Task 4 | +| Task 5: shadow, doctor, benchmark, and status | Complete | Commit `35a53fc`; focused and shared-core evidence exists | Real-platform promotion remains outside Task 5 | +| Task 6: compatibility and route transactions | Complete | Commit `5be10d2`; route race and exact-version tests pass | New client versions require new contracts, not a redesign | +| Task 7: neutral filesystem and tagged FUSE host | Complete | Commit `3f51aa5`; neutral and real macOS FUSE-T tests pass | Linux and Windows real adapters remain platform work | +| Task 8: standalone CLI and automatic enrollment | Partial | Commit `3352b87`; command surface and guarded lifecycle exist | Task 8 Step 5, bounded automatic enrollment, is missing | +| Task 9: service lifecycle and update guard | Complete | Commit `4589ffa`; launchd and preflight tests pass | Stronger automatic update claims remain release-gated | +| Task 10: synthetic, crash, performance, and compile gates | Complete for the shared engine | Commit `a1ac76e`; preview validation report | It cannot satisfy real-adapter or retention gates | +| Task 11: real macOS trace, adapter, shadow, and canary | Partial | Sanitized real CLI/Desktop/FUSE-T evidence is public | Managed-session sleep/wake and host restart, current-client compatibility, real-home canaries, and seven-day retention remain | + +## Missing Product Behavior And Exact Next Work + +| Missing behavior | Exact implementation work | Required verification | +| --- | --- | --- | +| Bounded automatic discovery and enrollment | Add a read-only policy planner over Codex state, stability evidence, writer state, doctor, compatibility, promotion stage, and storage budget; then add an idempotent bounded apply loop that does not change a route before fold, pack, shadow, snapshot, and mount acknowledgement succeed | Existing/new/forked sessions, active and changing files, unknown clients, failed doctor, low disk, batch limits, restart idempotency, and failed cutover in an isolated home | +| Conservative fork-family classification | Add evidence-only reports for exact shared content, independent tails, complete containment, active/archive state, and unknown relationships; never infer uselessness from ancestry, age, title, or size | Diverse fork and non-fork fixtures plus real sanitized families; zero automatic mutation | +| Guarded branch archival | Trace the current official Codex archive behavior, then implement a dry-run-first explicit archive transaction that revalidates the selected route and digest and preserves the rollout | Concurrent route change, active writer, source mutation, archive/unarchive round trip, daemon restart, and recovery | +| Content-changing boundary regression | Add CLI tests proving repair and reconciliation require a separate output and cannot be invoked by fold, migration, compaction, enrollment, rollback, or GC | Command-tree tests, call-boundary tests, unchanged source hashes, and verified output hashes | +| Hard disk budgets and bounded retention | Add a storage inventory and preflight budget used by every operation that can create a full copy or generation; enforce one migration snapshot, one current fallback, one transaction scratch file, and bounded old generations | Low-space refusal before write, repeated migration/rollback/enrollment, interrupted cleanup, live lease retention, and no unbounded retired-state growth | +| Actual physical reclamation reporting | Extend status, doctor, and mutating results with logical, unique, pack, source, snapshot, fallback, temporary/recovery, projected peak, projected reclaimable, and actual reclaimed bytes | Fixture accounting checked against filesystem allocation before and after GC/removal; zero reclaimed bytes while full copies remain | +| Current macOS compatibility and disruptive gates | Import exact contracts for installed clients; run a retained-source managed canary through sleep/wake and host restart; then start bounded real-home canaries only after automatic enrollment and disk budgets pass | Clean doctor, exact SHA after restart cases, rollback, no route loss, and seven incident-free days | +| Linux and Windows readiness | Implement FUSE3 and WinFsp adapters without moving shared behavior out of the core | Native operation traces, crash/restart, performance, upgrade quarantine, rollback, and retention on each platform | + +## Current Capability Decision + +The shared storage and virtual-file engines are implemented and validated strongly enough for `fs-engine-preview`. The repository does not yet satisfy automatic stable enrollment, bounded physical-space governance, complete macOS disruptive gates and retention, or real Linux and Windows adapter gates. Therefore: + +- Keep the capability at `fs-engine-preview`. +- Keep real user sessions native unless they are explicitly selected for a retained-source canary. +- Do not claim physical disk reclamation from logical deduplication alone. +- Do not enable automatic enrollment until Tasks 12 and 14 pass. +- Do not introduce a private control-plane dependency into any public surface. diff --git a/docs/validation-macos-canary.md b/docs/validation-macos-canary.md index 6cda3c4..d71daa2 100644 --- a/docs/validation-macos-canary.md +++ b/docs/validation-macos-canary.md @@ -2,10 +2,12 @@ ## Current Status -The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have passed for read, append, resume, fork, child-session enrollment, canonical archive/unarchive moves, launchd restart, rollback, namespace deactivation, and unknown-version quarantine. The project remains at `fs-engine-preview` because the user Codex home is intentionally not enrolled and sleep or full host restart has not been exercised. +The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have passed for read, append, resume, fork, child-session enrollment, canonical archive/unarchive moves, launchd restart, rollback, namespace deactivation, and unknown-version quarantine. An actual host reboot has now recovered the standalone launchd service and healthy empty mount, but no managed session was present during that reboot. The project remains at `fs-engine-preview` because the user Codex home is intentionally not enrolled, managed-session sleep/wake and full-host restart have not been exercised, the currently installed client versions are not covered by exact contracts, and canary retention has not started. Additional failure-containment evidence on 2026-07-14: +- After an actual host reboot, launchd started a fresh CodexFold process and the FUSE-T mount identity was healthy. The store contained zero managed session states, ordinary user sessions remained native, and status reported `fs-engine-preview`. This proves service and mount boot recovery only; it does not satisfy managed-session host-restart recovery. +- The post-reboot `fs doctor` check found daemon, mount, backing, delta, fallback, journal, manifest, pack, and route components healthy. It remained unhealthy overall because the currently installed Codex clients do not yet have exact compatibility contracts. - Canonical rollback now uses a two-stage retirement request and acknowledgement. The daemon keeps the managed session loaded while preferring a verified native target, so removing or changing that target falls back to managed bytes instead of creating an `ENOENT` window. - A live pending-retirement restart loaded the managed fallback into a fresh daemon, acknowledged the exact generation and route, and preserved the complete SHA-256. Toggling the native target 100 times while opening the mounted route 2,000 times produced zero read failures. - A second live restart began with an earlier successful acknowledgement after the native target had disappeared. The fresh daemon replaced it with `native rollback target is unavailable or changed`, remained running, and exposed the complete managed JSONL with the same SHA-256. @@ -128,7 +130,9 @@ A direct `SIGTERM` stopped the foreground service and removed the mount cleanly. The following gates are still open: -- Sleep/wake and full host-restart recovery. These disruptive checks were not run against the user's active machine. +- Managed-session sleep/wake recovery. +- Managed-session full host-restart recovery, including the append, compaction, migration, and rollback interruption cases required by the product contract. The successful empty-mount host boot above does not satisfy this gate. +- Exact compatibility contracts for the currently installed Codex Desktop and CLI versions. - Retained-source canary routes in the real Codex home. - Seven incident-free days after reaching `platform-canary`. From 1d043aa2988f46da9f450321569b29b1bc4e875e Mon Sep 17 00:00:00 2001 From: King Star Date: Wed, 15 Jul 2026 02:34:59 +0800 Subject: [PATCH 27/33] docs: record managed-session host reboot canary --- ...arent-session-filesystem-implementation.md | 6 +++--- docs/validation-macos-canary.md | 21 +++++++++++++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md b/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md index 4991f2c..28d761f 100644 --- a/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md +++ b/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md @@ -10,7 +10,7 @@ ## Alignment Snapshot -Current public status remains `fs-engine-preview`. Tasks 1 through 7 and 9 through 10 are implemented. Task 8 lacks bounded automatic enrollment. Task 11 has substantial isolated macOS evidence but has not passed managed-session host restart, sleep/wake, current-client compatibility, real-home retained-source canaries, or seven-day retention. +Current public status remains `fs-engine-preview`. Tasks 1 through 7 and 9 through 10 are implemented. Task 8 lacks bounded automatic enrollment. Task 11 has substantial isolated macOS evidence, including one idle retained-source managed CLI session surviving an actual host reboot, but has not passed managed-session sleep/wake, transaction-interruption restart cases, current Desktop compatibility, real-home retained-source canaries, or seven-day retention. | Task | Status | Current evidence | Remaining work | | --- | --- | --- | --- | @@ -24,7 +24,7 @@ Current public status remains `fs-engine-preview`. Tasks 1 through 7 and 9 throu | 8 | Partial | Commit `3352b87`; standalone CLI and guarded lifecycle commands | Implement bounded automatic discovery and enrollment | | 9 | Complete | Commit `4589ffa`; launchd lifecycle and update preflight tests | Production update promotion remains gated by platform readiness | | 10 | Complete | Commit `a1ac76e`; synthetic, crash, race, cross-compile, and 758 MiB evidence | This task proves only the shared engine preview | -| 11 | Partial | Real macOS CLI/Desktop, FUSE-T, rollback, restart, and quarantine evidence | Complete the remaining disruptive and retention gates | +| 11 | Partial | Real macOS CLI/Desktop, FUSE-T, rollback, daemon restart, idle managed-session host reboot, and quarantine evidence | Complete the remaining disruptive and retention gates | ## Global Constraints @@ -709,7 +709,7 @@ Select 5–10 archived sessions, fold and pack without source removal, compare e - [ ] **Step 5: Route retained-source canaries** -Current state: isolated retained-source CLI and Desktop canaries passed direct open, resume, append, tool use, fork, archive/unarchive, daemon restart, rollback, re-migration, and quarantine. This step remains open because managed-session sleep/wake, full host restart, current installed-client compatibility, and real-home retained-source canaries are not complete. +Current state: isolated retained-source CLI and Desktop canaries passed direct open, resume, append, tool use, fork, archive/unarchive, daemon restart, rollback, re-migration, and quarantine. One idle retained-source managed CLI session also passed an actual host reboot, post-boot managed resume, exact rollback, and native resume. This step remains open because managed-session sleep/wake, host interruption during append/compaction/migration/rollback, current Desktop compatibility, and real-home retained-source canaries are not complete. After clean shadow and compatibility, migrate only the selected archived sessions. Verify Desktop direct click, CLI resume, history, message send, tool use, fork, archive, unarchive, daemon termination, mount restart, sleep/wake, host restart, rollback, and compatibility quarantine. Never delete native snapshots. diff --git a/docs/validation-macos-canary.md b/docs/validation-macos-canary.md index d71daa2..e319569 100644 --- a/docs/validation-macos-canary.md +++ b/docs/validation-macos-canary.md @@ -2,7 +2,20 @@ ## Current Status -The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have passed for read, append, resume, fork, child-session enrollment, canonical archive/unarchive moves, launchd restart, rollback, namespace deactivation, and unknown-version quarantine. An actual host reboot has now recovered the standalone launchd service and healthy empty mount, but no managed session was present during that reboot. The project remains at `fs-engine-preview` because the user Codex home is intentionally not enrolled, managed-session sleep/wake and full-host restart have not been exercised, the currently installed client versions are not covered by exact contracts, and canary retention has not started. +The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have passed for read, append, resume, fork, child-session enrollment, canonical archive/unarchive moves, launchd restart, rollback, namespace deactivation, and unknown-version quarantine. A retained-source CLI canary also survived an actual host reboot while its parent session was managed, then resumed through the recovered mount and rolled back to an exact native JSONL. The project remains at `fs-engine-preview` because the user Codex home is intentionally not enrolled, managed-session sleep/wake and interruption during append, compaction, migration, or rollback have not been exercised, the currently installed Desktop version is not covered by an exact contract, and canary retention has not started. + +Additional host-restart evidence on 2026-07-15: + +- The isolated canary used PATH `codex-cli 0.144.3` with a real Responses model turn. Its current version contract was exact and approved before migration. The installed Desktop `26.707.72221+5307` was not used in this run and remains outside this evidence. +- A native parent was created and resumed before folding. Its 85,776-byte, 49-record baseline was folded, packed, and reconstructed with an exact SHA-256 plus 10,000 successful random-range comparisons while the source was retained. +- The managed parent resumed and appended through `append.delta` without a writable backing. After a standalone daemon restart, another real turn recalled the prior managed turn and appended again. The complete base prefix remained byte-identical. +- A real CLI fork created an ordinary native child while the parent remained managed. The child and parent then resumed independently; marker checks and complete-file hashes showed no cross-branch writes. +- The first rollback restored the exact 107,091-byte managed view to an ordinary JSONL. A native resume recalled the managed history and appended successfully, producing 111,271 bytes and 97 valid JSONL records. +- The updated parent was folded and migrated again before an actual macOS reboot. After login, launchd recreated both the standalone process and FUSE-T mount. Before any new turn, the recovered managed view was exactly 111,271 bytes, 97 records, and SHA-256 `98369563df4766c50d4d8886c8dff8163471e19d670325a2ef1190537064cbba`, with an empty delta and no writable backing. +- A real post-reboot resume recalled the latest native turn and appended 4,255 bytes through the managed delta. The 111,271-byte base prefix kept the same SHA-256, the complete visible file became 115,526 bytes and 106 valid records, and no writable backing appeared. +- Post-reboot rollback materialized the exact 115,526-byte visible view. A final native resume recalled the managed post-reboot turn and appended successfully. The parent ended at 119,678 bytes and 115 valid records with SHA-256 `e891263cbdfcbe9f149138eb94bfb4371afee0424b20d4afa1d261954dde5144`; the child remained unchanged at SHA-256 `76b19d4c5b0d1b41b312dadfee3b9b871165da2dac58ba47b1e60c1e1cb102bd`. +- Namespace deactivation restored ordinary `sessions` and `archived_sessions` directories. Both SQLite routes point to native JSONL files, every record parses, all expected parent markers remain in order, and parent/child marker isolation still passes. +- This proves idle retained-source managed-session recovery across one actual host reboot. It does not cover a reboot or power loss during an active append, compaction, migration, or rollback transaction, and it does not authorize enrollment of the real Codex home. Additional failure-containment evidence on 2026-07-14: @@ -72,7 +85,7 @@ These results validate exact reconstruction and random reads. They do not valida ## Isolated Real Codex Canary -The canary used an isolated Codex home and state database. It did not modify the user's real Codex routes. The final full-flow run used the desktop-bundled `codex-cli 0.144.0-alpha.4` and a clean isolated root; the later focused route-guard run used Desktop `26.707.61608+5200` and PATH `codex-cli 0.144.1`. `scripts/prepare-isolated-codex-home.sh` copies the current `config.toml`, `auth.json`, and optional `models_cache.json` byte-for-byte and uses APFS clones for static plugin assets, so the canary uses the current provider configuration without allowing canary writes to modify the source home. +The canary used an isolated Codex home and state database. It did not modify the user's real Codex routes. The final full-flow run used the desktop-bundled `codex-cli 0.144.0-alpha.4` and a clean isolated root; the later focused route-guard run used Desktop `26.707.61608+5200` and PATH `codex-cli 0.144.1`; the host-restart run used PATH `codex-cli 0.144.3`. `scripts/prepare-isolated-codex-home.sh` copies the current `config.toml`, `auth.json`, and optional `models_cache.json` byte-for-byte and uses APFS clones for static plugin assets, so the canary uses the current provider configuration without allowing canary writes to modify the source home. The validated sequence was: @@ -131,8 +144,8 @@ A direct `SIGTERM` stopped the foreground service and removed the mount cleanly. The following gates are still open: - Managed-session sleep/wake recovery. -- Managed-session full host-restart recovery, including the append, compaction, migration, and rollback interruption cases required by the product contract. The successful empty-mount host boot above does not satisfy this gate. -- Exact compatibility contracts for the currently installed Codex Desktop and CLI versions. +- Managed-session host-interruption recovery during append, compaction, migration, and rollback. One idle retained-source managed session has passed a full host restart, but that result does not cover interruption inside those transactions. +- Exact compatibility contract and retained-source canary for the currently installed Codex Desktop `26.707.72221+5307`; PATH `codex-cli 0.144.3` passed this run. - Retained-source canary routes in the real Codex home. - Seven incident-free days after reaching `platform-canary`. From 7f909cd6d430f8e6ff18a76a213a12cacf5d860c Mon Sep 17 00:00:00 2001 From: King Star Date: Wed, 15 Jul 2026 04:15:40 +0800 Subject: [PATCH 28/33] fix: recover interrupted filesystem transactions --- docs/validation-macos-canary.md | 16 ++- internal/cli/fs.go | 99 ++++++++++++++-- internal/cli/fs_service.go | 10 +- internal/cli/fs_test.go | 198 ++++++++++++++++++++++++++++++++ internal/vfs/compact.go | 26 ++++- internal/vfs/recover.go | 70 +++++++++-- internal/vfs/recovery_test.go | 101 ++++++++++++++++ internal/vfs/state.go | 35 +++++- 8 files changed, 519 insertions(+), 36 deletions(-) diff --git a/docs/validation-macos-canary.md b/docs/validation-macos-canary.md index e319569..d7f1bcb 100644 --- a/docs/validation-macos-canary.md +++ b/docs/validation-macos-canary.md @@ -2,7 +2,17 @@ ## Current Status -The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have passed for read, append, resume, fork, child-session enrollment, canonical archive/unarchive moves, launchd restart, rollback, namespace deactivation, and unknown-version quarantine. A retained-source CLI canary also survived an actual host reboot while its parent session was managed, then resumed through the recovered mount and rolled back to an exact native JSONL. The project remains at `fs-engine-preview` because the user Codex home is intentionally not enrolled, managed-session sleep/wake and interruption during append, compaction, migration, or rollback have not been exercised, the currently installed Desktop version is not covered by an exact contract, and canary retention has not started. +The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have passed for read, append, resume, fork, child-session enrollment, canonical archive/unarchive moves, launchd restart, rollback, namespace deactivation, and unknown-version quarantine. A retained-source CLI canary survived an actual host reboot while managed, then resumed through the recovered mount and rolled back to an exact native JSONL. The currently installed CLI and Desktop versions also passed exact compatibility and isolated retained-source canaries. Process-level interruption recovery now covers append, compaction, migration, and rollback, and a managed session passed an actual Deep Idle sleep/wake cycle followed by a real model turn. The project remains at `fs-engine-preview` because the user Codex home is intentionally not enrolled, in-flight transaction evidence does not claim an actual power-loss test, and canary retention has not started. + +Additional current-client, interruption, and sleep/wake evidence on 2026-07-15: + +- Exact contracts were imported and approved for PATH `codex-cli 0.144.3`, Desktop `26.707.72221+5307`, and the Desktop-bundled app server `0.144.2`. The current Desktop opened an isolated managed task, displayed the complete native and managed history, completed a real model turn, survived forced termination and restart, rolled back exactly, and resumed natively. +- A canonical migration process and the FUSE daemon were both terminated immediately after managed state became durable but before cutover. Launchd started a fresh daemon, startup recovery retired the incomplete state, and both the retained native file and mounted native view retained the same complete SHA-256. A separate regression proves that a noncanonical migration failure also retires state created by that failed attempt. +- During a real CLI append, the FUSE daemon was terminated after a 9,291-byte delta prefix was durable. Codex observed one `EIO`, reopened its rollout writer, retried, and completed the turn. The durable prefix remained byte-identical, the complete 86-record JSONL parsed, no writable backing appeared, and a later real resume recalled the interrupted turn. +- Compaction now acquires the cross-process writer lease in addition to the in-process writer state. A termination before state publication recovered by rolling back the candidate generation and removing the journal-owned candidate delta, scratch file, and state temporary. A second termination after atomic state publication recovered by completing the candidate generation. Both sides preserved the same exact 132,510-byte visible SHA-256, and a later real resume recalled pre-compaction history and appended through the new delta. +- Canonical rollback was paused after the daemon acknowledged a verified native target, then both rollback and daemon processes were terminated. A fresh daemon preserved a complete readable route and the pending request. Re-running the same rollback reused the exact token only because generation, route, byte count, and SHA-256 still matched; it then retired managed state and cleared the control files. The resulting 136,514-byte, 104-record native JSONL resumed successfully. +- macOS power logs recorded entry into Software Sleep and wake from Deep Idle. Across that cycle, the same daemon PID and FUSE mount remained healthy, and the managed view stayed exactly 144,580 bytes, 122 valid records, a 4,018-byte delta, and the same SHA-256. A real post-wake resume recalled the pre-sleep turn and produced a 148,547-byte, 131-record view with a 7,985-byte delta and no writable backing. +- These interruption runs used real FUSE-T, real launchd restarts, and unmodified Codex clients. They validate deterministic recovery from simultaneous client/control-process and daemon termination. They do not represent an actual power loss during an in-flight transaction. Additional host-restart evidence on 2026-07-15: @@ -143,10 +153,8 @@ A direct `SIGTERM` stopped the foreground service and removed the mount cleanly. The following gates are still open: -- Managed-session sleep/wake recovery. -- Managed-session host-interruption recovery during append, compaction, migration, and rollback. One idle retained-source managed session has passed a full host restart, but that result does not cover interruption inside those transactions. -- Exact compatibility contract and retained-source canary for the currently installed Codex Desktop `26.707.72221+5307`; PATH `codex-cli 0.144.3` passed this run. - Retained-source canary routes in the real Codex home. +- An actual power-loss or host-restart interruption while a transaction is in flight; simultaneous process termination and a separate idle managed-session host reboot have passed, but they are recorded as distinct evidence. - Seven incident-free days after reaching `platform-canary`. Until every applicable gate passes, the project must keep the capability at `fs-engine-preview`, retain original JSONL files, and avoid changing real Codex routes. diff --git a/internal/cli/fs.go b/internal/cli/fs.go index 6648e52..d146172 100644 --- a/internal/cli/fs.go +++ b/internal/cli/fs.go @@ -273,6 +273,18 @@ func newFSServeCommand() *cobra.Command { return err } defer processLock.Close() + if canonicalNamespace { + for _, state := range states { + if _, err := recoverInterruptedCanonicalMigration(home, store, nativeRoot, state); err != nil { + return err + } + } + states, err = vfs.DiscoverSessionStates(store) + if err != nil { + return err + } + result.ManagedSessions = len(states) + } if err := os.MkdirAll(mount, 0o700); err != nil { return err } @@ -618,8 +630,16 @@ func newFSMigrateCommand() *cobra.Command { native = retained result.Native = retained } - rollbackCanonicalMigration := func(cause error) error { + rollbackMigration := func(cause error) error { if !canonicalNamespace { + if _, err := os.Stat(filepath.Join(store, "fs", "sessions", session.ID)); errors.Is(err, os.ErrNotExist) { + return cause + } else if err != nil { + return errors.Join(cause, err) + } + if _, err := retireManagedState(store, session.ID); err != nil { + return errors.Join(cause, err) + } return cause } if _, err := os.Stat(filepath.Join(store, "fs", "sessions", session.ID)); err == nil { @@ -634,34 +654,34 @@ func newFSMigrateCommand() *cobra.Command { } managed, migrationLease, err := vfs.OpenSessionWithWriter(command.Context(), vfs.SessionOptions{Root: store, ManifestPath: fold.ManifestPath(store, session.ID), Manifest: manifest, Reader: resolver, NativeSnapshot: native}) if err != nil { - return rollbackCanonicalMigration(err) + return rollbackMigration(err) } defer migrationLease.Close() if canonicalNamespace { if err := waitForMountAcknowledgement(command.Context(), store, session.ID, managed.State().Generation, canonicalRoute, mountWait); err != nil { - return rollbackCanonicalMigration(fmt.Errorf("wait for canonical mount acknowledgement: %w", err)) + return rollbackMigration(fmt.Errorf("wait for canonical mount acknowledgement: %w", err)) } sessions, err := codex.LoadSessions(home) if err != nil { - return rollbackCanonicalMigration(err) + return rollbackMigration(err) } current, err := findSession(sessions, session.ID) if err != nil || filepath.Clean(current.RolloutPath) != filepath.Clean(session.RolloutPath) { - return rollbackCanonicalMigration(errors.New("canonical Codex route changed during migration")) + return rollbackMigration(errors.New("canonical Codex route changed during migration")) } if _, err := waitForTargetMatch(command.Context(), target, vfs.NativeFile{Bytes: shadow.Bytes, SHA256: shadow.SHA256}, mountWait); err != nil { - return rollbackCanonicalMigration(fmt.Errorf("verify managed target before canonical cutover: %w", err)) + return rollbackMigration(fmt.Errorf("verify managed target before canonical cutover: %w", err)) } if err := finalizeCanonicalSnapshotSource(canonicalSource, native); err != nil { - return rollbackCanonicalMigration(err) + return rollbackMigration(err) } } targetFile, err := waitForTarget(command.Context(), target, mountWait) if err != nil { - return rollbackCanonicalMigration(fmt.Errorf("verify mounted target: %w", err)) + return rollbackMigration(fmt.Errorf("verify mounted target: %w", err)) } if targetFile.Bytes != shadow.Bytes || targetFile.SHA256 != shadow.SHA256 { - return rollbackCanonicalMigration(errors.New("mounted target differs from the shadow-verified native session")) + return rollbackMigration(errors.New("mounted target differs from the shadow-verified native session")) } if !canonicalNamespace { if _, err := codex.RouteSession(command.Context(), codex.RouteOptions{CodexHome: home, SessionID: session.ID, ExpectedPath: session.RolloutPath, Target: codex.RouteTarget{Path: target, Bytes: targetFile.Bytes, SHA256: targetFile.SHA256}}); err != nil { @@ -674,7 +694,7 @@ func newFSMigrateCommand() *cobra.Command { } current, err := findSession(sessions, session.ID) if err != nil || filepath.Clean(current.RolloutPath) != filepath.Clean(session.RolloutPath) { - return rollbackCanonicalMigration(errors.New("canonical Codex route changed during migration")) + return rollbackMigration(errors.New("canonical Codex route changed during migration")) } } result.Routed = true @@ -1037,7 +1057,11 @@ func newFSRecoverCommand() *cobra.Command { _ = resolver.Close() return err } + recoveredState := managed.State() _ = resolver.Close() + if _, err := recoverInterruptedCanonicalMigration(home, store, filepath.Join(home, "fold-native"), recoveredState); err != nil { + return err + } result.Recovered++ } if jsonOutput { @@ -1055,6 +1079,61 @@ func newFSRecoverCommand() *cobra.Command { return command } +func recoverInterruptedCanonicalMigration(home string, store string, nativeRoot string, state vfs.SessionState) (bool, error) { + retainedPath := filepath.Join(store, "fs", "snapshots", state.SessionID, "native.jsonl") + if filepath.Clean(state.NativeSnapshot.Path) != filepath.Clean(retainedPath) { + return false, nil + } + if _, pending, err := readRetirementRequest(store, state.SessionID); err != nil { + return false, err + } else if pending { + return false, nil + } + sessions, err := codex.LoadSessions(home) + if err != nil { + return false, err + } + current, err := findSession(sessions, state.SessionID) + if err != nil { + return false, err + } + sourcePath, err := canonicalNativeRoute(home, nativeRoot, current.RolloutPath) + if err != nil { + return false, err + } + source, err := hashPath(sourcePath) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + if state.BackingPath != "" { + return false, nil + } + delta, err := os.Stat(state.DeltaPath) + if err != nil { + return false, err + } + if delta.Size() != 0 { + return false, nil + } + if source.Bytes != state.BaseBytes || source.SHA256 != state.BaseSHA256 || source.Bytes != state.NativeSnapshot.Bytes || source.SHA256 != state.NativeSnapshot.SHA256 { + return false, errors.New("interrupted canonical migration source no longer matches the managed base") + } + retiredState, err := retireManagedState(store, state.SessionID) + if err != nil { + return false, err + } + if _, err := retireCanonicalNativeSnapshot(store, nativeRoot, state.SessionID, state.NativeSnapshot.Path, sourcePath, retiredState); err != nil { + if restoreErr := restoreManagedState(store, state.SessionID, retiredState); restoreErr != nil { + return false, errors.Join(err, restoreErr) + } + return false, err + } + return true, nil +} + func addCompatibilityFlags(command *cobra.Command, flags *compatibilityFlags) { defaults := defaultCompatibilityFlags() command.Flags().StringVar(&flags.contractsPath, "contracts", "", "Compatibility contract directory; defaults to /compatibility") diff --git a/internal/cli/fs_service.go b/internal/cli/fs_service.go index 346dc6a..4e1c9f6 100644 --- a/internal/cli/fs_service.go +++ b/internal/cli/fs_service.go @@ -566,11 +566,13 @@ func createRetirementRequest(store string, sessionID string, generation uint64, return retirementControl{}, errors.New("complete retirement request metadata is required") } directory := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) - requestPath := filepath.Join(directory, retirementRequestFilename) - if _, err := os.Lstat(requestPath); err == nil { - return retirementControl{}, errors.New("session retirement is already pending") - } else if !errors.Is(err, os.ErrNotExist) { + if pending, exists, err := readRetirementRequest(store, sessionID); err != nil { return retirementControl{}, err + } else if exists { + if pending.Generation != generation || pending.Route != route || pending.Bytes != target.Bytes || pending.SHA256 != target.SHA256 { + return retirementControl{}, errors.New("pending session retirement does not match the requested generation and target") + } + return pending, nil } if err := removeIfExists(filepath.Join(directory, retirementAcknowledgementFilename)); err != nil { return retirementControl{}, err diff --git a/internal/cli/fs_test.go b/internal/cli/fs_test.go index fc50062..18e2065 100644 --- a/internal/cli/fs_test.go +++ b/internal/cli/fs_test.go @@ -459,6 +459,43 @@ func TestFSMigrateApplyFailsClosedWithoutMountedTarget(t *testing.T) { } } +func TestFSMigrateApplyRetiresManagedStateWhenMountedTargetNeverAppears(t *testing.T) { + allowFixtureMount(t) + home, storeDir, nativePath := fsFixture(t, true) + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + mount := filepath.Join(home, "mount") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + original, err := os.ReadFile(nativePath) + if err != nil { + t.Fatal(err) + } + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "migrate", "session", "--codex-home", home, "--store", storeDir, + "--mount", mount, "--mount-wait", "50ms", "--cli", cliPath, + "--desktop-app", "none", "--apply", + }) + if err := root.Execute(); err == nil { + t.Fatal("fs migrate --apply should fail when the mounted target never appears") + } + sessions, err := codex.LoadSessions(home) + if err != nil || sessions[0].RolloutPath != nativePath { + t.Fatalf("failed apply changed route: sessions=%#v err=%v", sessions, err) + } + got, err := os.ReadFile(nativePath) + if err != nil || !bytes.Equal(got, original) { + t.Fatalf("failed apply changed source: got=%q err=%v", got, err) + } + states, err := vfs.DiscoverSessionStates(storeDir) + if err != nil || len(states) != 0 { + t.Fatalf("failed apply left managed state: states=%#v err=%v", states, err) + } +} + func TestFSMigrateApplyRejectsPlainDirectoryThatOnlyLooksLikeMount(t *testing.T) { home, storeDir, nativePath := fsFixture(t, true) cliPath := approvedCLIContract(t, storeDir, "1.2.3") @@ -673,6 +710,167 @@ func TestFSMigrateCanonicalKeepsCodexRouteAndHidesRetainedSnapshot(t *testing.T) } } +func TestFSRecoverRetiresInterruptedCanonicalMigrationWithUnchangedSource(t *testing.T) { + fixture := interruptedCanonicalMigrationFixture(t) + _ = fixture.resolver.Close() + + executeFS(t, []string{"fs", "recover", "session", "--apply", "--codex-home", fixture.home, "--store", fixture.store}) + if _, err := os.Stat(filepath.Join(fixture.store, "fs", "sessions", "session")); !os.IsNotExist(err) { + t.Fatalf("interrupted migration state remained: %v", err) + } + got, err := os.ReadFile(fixture.nativePath) + if err != nil || !bytes.Equal(got, fixture.source) { + t.Fatalf("recovery changed canonical source: got=%q err=%v", got, err) + } +} + +func TestFSRecoverLeavesPendingCanonicalRollbackManaged(t *testing.T) { + fixture := interruptedCanonicalMigrationFixture(t) + defer fixture.resolver.Close() + tail := []byte("{\"pending_rollback\":true}\n") + writer, err := fixture.managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + if _, err := writer.Append(context.Background(), tail); err != nil { + _ = writer.Close() + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + _ = writer.Close() + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + target, err := fixture.managed.MaterializeCurrent(context.Background(), fixture.nativePath, true) + if err != nil { + t.Fatal(err) + } + state := fixture.managed.State() + if _, err := createRetirementRequest(fixture.store, "session", state.Generation, "/archived_sessions/rollout-session.jsonl", target); err != nil { + t.Fatal(err) + } + recovered, err := recoverInterruptedCanonicalMigration(fixture.home, fixture.store, fixture.nativeRoot, state) + if err != nil || recovered { + t.Fatalf("pending rollback recovery = %t, %v", recovered, err) + } + if _, err := managedState(fixture.store, "session"); err != nil { + t.Fatalf("pending rollback state was retired: %v", err) + } + if err := clearRetirementControl(filepath.Join(fixture.store, "fs", "sessions", "session")); err != nil { + t.Fatal(err) + } + recovered, err = recoverInterruptedCanonicalMigration(fixture.home, fixture.store, fixture.nativeRoot, state) + if err != nil || recovered { + t.Fatalf("pre-request rollback recovery = %t, %v", recovered, err) + } + want := append(append([]byte(nil), fixture.source...), tail...) + if got, err := os.ReadFile(fixture.nativePath); err != nil || !bytes.Equal(got, want) { + t.Fatalf("pending rollback target changed: got=%q err=%v", got, err) + } +} + +func TestCreateRetirementRequestResumesOnlyExactPendingRequest(t *testing.T) { + storeDir := t.TempDir() + directory := filepath.Join(storeDir, "fs", "sessions", "session") + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + target := vfs.NativeFile{Path: filepath.Join(t.TempDir(), "current.jsonl"), Bytes: 42, SHA256: strings.Repeat("a", 64)} + first, err := createRetirementRequest(storeDir, "session", 3, "/archived_sessions/rollout.jsonl", target) + if err != nil { + t.Fatal(err) + } + resumed, err := createRetirementRequest(storeDir, "session", 3, "/archived_sessions/rollout.jsonl", target) + if err != nil { + t.Fatalf("resume exact retirement request: %v", err) + } + if resumed != first { + t.Fatalf("resumed request changed token or metadata: first=%#v resumed=%#v", first, resumed) + } + target.SHA256 = strings.Repeat("b", 64) + if _, err := createRetirementRequest(storeDir, "session", 3, "/archived_sessions/rollout.jsonl", target); err == nil { + t.Fatal("mismatched pending retirement request should fail closed") + } +} + +type interruptedCanonicalFixture struct { + home string + store string + nativeRoot string + nativePath string + source []byte + managed *vfs.Session + resolver *pack.Resolver +} + +func interruptedCanonicalMigrationFixture(t *testing.T) interruptedCanonicalFixture { + t.Helper() + home := t.TempDir() + storeDir := filepath.Join(home, "fold-store") + route := filepath.Join(home, "archived_sessions", "rollout-session.jsonl") + if err := os.MkdirAll(filepath.Dir(route), 0o700); err != nil { + t.Fatal(err) + } + source := []byte("{\"type\":\"session_meta\"}\n{\"interrupted_migration\":true}\n") + if err := os.WriteFile(route, source, 0o600); err != nil { + t.Fatal(err) + } + writeStateFixture(t, home, route) + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`update threads set archived = 1, id = 'session' where id = 'fixture'`); err != nil { + _ = db.Close() + t.Fatal(err) + } + _ = db.Close() + if _, err := fold.Fold(context.Background(), codex.Session{ID: "session", RolloutPath: route, Archived: true}, fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { + t.Fatal(err) + } + if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { + t.Fatal(err) + } + nativeRoot := filepath.Join(home, "fold-native") + nativePath := filepath.Join(nativeRoot, "archived_sessions", filepath.Base(route)) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(route, nativePath); err != nil { + t.Fatal(err) + } + native, err := hashPath(nativePath) + if err != nil { + t.Fatal(err) + } + retained, err := retainCanonicalSnapshot(storeDir, "session", native) + if err != nil { + t.Fatal(err) + } + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: retained, + }) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + return interruptedCanonicalFixture{ + home: home, store: storeDir, nativeRoot: nativeRoot, nativePath: nativePath, + source: source, managed: managed, resolver: resolver, + } +} + func TestFSMigrateCanonicalReservesWriterDuringCutover(t *testing.T) { allowFixtureMount(t) home := t.TempDir() diff --git a/internal/vfs/compact.go b/internal/vfs/compact.go index f9e056a..3ae1e03 100644 --- a/internal/vfs/compact.go +++ b/internal/vfs/compact.go @@ -41,8 +41,23 @@ func (s *Session) Compact(ctx context.Context, options CompactOptions) (CompactR s.mu.Unlock() return CompactResult{}, errors.New("cannot compact while a writer lease is held") } + s.writerOpen = true state := s.state s.mu.Unlock() + lease, err := acquireWriterLease(filepath.Join(s.directory, "writer.lease")) + if err != nil { + s.mu.Lock() + s.writerOpen = false + s.mu.Unlock() + return CompactResult{}, err + } + defer func() { + _ = unlockWriterFile(lease) + _ = lease.Close() + s.mu.Lock() + s.writerOpen = false + s.mu.Unlock() + }() activePath := state.DeltaPath if state.BackingPath != "" { activePath = state.BackingPath @@ -99,7 +114,8 @@ func (s *Session) Compact(ctx context.Context, options CompactOptions) (CompactR next.DeltaPath = newDelta next.BackingPath = "" operationID := fmt.Sprintf("compact-%020d", state.Generation) - if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "prepared", Candidate: next, FinalPath: newDelta, Native: current}); err != nil { + stateTemporary := filepath.Join(s.directory, fmt.Sprintf(".state-compact-%020d.tmp", next.Generation)) + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "prepared", Candidate: next, TempPath: stateTemporary, FinalPath: newDelta, Native: current}); err != nil { return CompactResult{}, err } if options.BeforePhase != nil { @@ -107,7 +123,7 @@ func (s *Session) Compact(ctx context.Context, options CompactOptions) (CompactR return CompactResult{}, err } } - if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "state-publishing", Candidate: next, FinalPath: newDelta, Native: current}); err != nil { + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "state-publishing", Candidate: next, TempPath: stateTemporary, FinalPath: newDelta, Native: current}); err != nil { return CompactResult{}, err } if options.BeforePhase != nil { @@ -115,14 +131,14 @@ func (s *Session) Compact(ctx context.Context, options CompactOptions) (CompactR return CompactResult{}, err } } - if err := writeSessionState(s.statePath, next); err != nil { + if err := writeSessionStateWithTemporary(s.statePath, stateTemporary, next); err != nil { return CompactResult{}, err } s.mu.Lock() s.state = next s.view = prepared.View s.mu.Unlock() - if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "state-published", Candidate: next, FinalPath: newDelta, Native: current}); err != nil { + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "state-published", Candidate: next, TempPath: stateTemporary, FinalPath: newDelta, Native: current}); err != nil { return CompactResult{}, err } if options.BeforePhase != nil { @@ -130,7 +146,7 @@ func (s *Session) Compact(ctx context.Context, options CompactOptions) (CompactR return CompactResult{}, err } } - if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "complete", Candidate: next, FinalPath: newDelta, Native: current}); err != nil { + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "complete", Candidate: next, TempPath: stateTemporary, FinalPath: newDelta, Native: current}); err != nil { return CompactResult{}, err } return CompactResult{Generation: next.Generation, Bytes: current.Bytes, SHA256: current.SHA256}, nil diff --git a/internal/vfs/recover.go b/internal/vfs/recover.go index b79f47c..e3b5432 100644 --- a/internal/vfs/recover.go +++ b/internal/vfs/recover.go @@ -5,7 +5,9 @@ import ( "errors" "fmt" "os" + "path/filepath" "sort" + "strings" ) func (s *Session) recover(ctx context.Context) error { @@ -35,6 +37,11 @@ func (s *Session) recover(ctx context.Context) error { for _, record := range ordered { switch record.Phase { case "complete", "rolled-back": + if record.Kind == "compact" { + if err := s.cleanupCompactArtifacts(record, record.Phase == "rolled-back"); err != nil { + return err + } + } continue case "after-file-publish", "state-publishing", "state-published": if record.Candidate.SessionID != s.state.SessionID || record.Candidate.Generation == 0 || !pathWithin(s.directory, record.Candidate.DeltaPath) || (record.Candidate.BackingPath != "" && !pathWithin(s.directory, record.Candidate.BackingPath)) { @@ -46,8 +53,12 @@ func (s *Session) recover(ctx context.Context) error { } if record.Kind == "compact" { if state.Generation < record.Candidate.Generation { - _ = os.Remove(record.Candidate.DeltaPath) - if err := appendJournal(s.directory, JournalRecord{OperationID: record.OperationID, SessionID: record.SessionID, Kind: record.Kind, Phase: "rolled-back", Candidate: record.Candidate, FinalPath: record.FinalPath}); err != nil { + resolved := record + resolved.Phase = "rolled-back" + if err := appendJournal(s.directory, resolved); err != nil { + return err + } + if err := s.cleanupCompactArtifacts(resolved, true); err != nil { return err } continue @@ -71,19 +82,29 @@ func (s *Session) recover(ctx context.Context) error { s.state = record.Candidate } } - if err := appendJournal(s.directory, JournalRecord{OperationID: record.OperationID, SessionID: record.SessionID, Kind: record.Kind, Phase: "complete", Candidate: record.Candidate, FinalPath: record.FinalPath, Native: record.Native}); err != nil { + resolved := record + resolved.Phase = "complete" + if err := appendJournal(s.directory, resolved); err != nil { return err } - case "prepared", "data-synced": - if record.TempPath != "" { - _ = os.Remove(record.TempPath) - } - if record.Kind == "compact" && record.FinalPath != "" { - _ = os.Remove(record.FinalPath) + if record.Kind == "compact" { + if err := s.cleanupCompactArtifacts(resolved, false); err != nil { + return err + } } - if err := appendJournal(s.directory, JournalRecord{OperationID: record.OperationID, SessionID: record.SessionID, Kind: record.Kind, Phase: "rolled-back", Candidate: record.Candidate, TempPath: record.TempPath}); err != nil { + case "prepared", "data-synced": + resolved := record + resolved.Phase = "rolled-back" + if err := appendJournal(s.directory, resolved); err != nil { return err } + if record.Kind == "compact" { + if err := s.cleanupCompactArtifacts(resolved, true); err != nil { + return err + } + } else if record.TempPath != "" { + _ = os.Remove(record.TempPath) + } default: return fmt.Errorf("journal operation %s has unknown phase %q", record.OperationID, record.Phase) } @@ -91,4 +112,33 @@ func (s *Session) recover(ctx context.Context) error { return nil } +func (s *Session) cleanupCompactArtifacts(record JournalRecord, rollback bool) error { + removeOwned := func(candidate string, prefix string, suffix string) error { + if candidate == "" { + return nil + } + candidate = filepath.Clean(candidate) + name := filepath.Base(candidate) + if filepath.Dir(candidate) != filepath.Clean(s.directory) || !strings.HasPrefix(name, prefix) || !strings.HasSuffix(name, suffix) { + return fmt.Errorf("journal operation %s has unsafe compact artifact %q", record.OperationID, candidate) + } + if err := os.Remove(candidate); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil + } + if err := removeOwned(record.Native.Path, ".compact-", ".jsonl"); err != nil { + return err + } + if err := removeOwned(record.TempPath, ".state-compact-", ".tmp"); err != nil { + return err + } + if rollback { + if err := removeOwned(record.FinalPath, "delta-", ".jsonl"); err != nil { + return err + } + } + return nil +} + func (s *Session) Recover(ctx context.Context) error { return s.recover(ctx) } diff --git a/internal/vfs/recovery_test.go b/internal/vfs/recovery_test.go index ba6dcd8..9284500 100644 --- a/internal/vfs/recovery_test.go +++ b/internal/vfs/recovery_test.go @@ -166,6 +166,107 @@ func TestCompactRejectsDeltaChangedDuringPreparation(t *testing.T) { } } +func TestCompactRejectsWriterLeaseHeldByAnotherSession(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + serving := openFixtureSession(t, root, manifest, reader, nil) + writer, err := serving.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + defer writer.Close() + + maintenance := openFixtureSession(t, root, manifest, reader, nil) + _, err = maintenance.Compact(context.Background(), CompactOptions{ + Prepare: func(context.Context, NativeFile, uint64) (PreparedGeneration, error) { + t.Fatal("compact preparation ran while another process held the writer lease") + return PreparedGeneration{}, nil + }, + }) + if !errors.Is(err, ErrWriterBusy) { + t.Fatalf("Compact error = %v, want %v", err, ErrWriterBusy) + } +} + +func TestCompactHoldsAndReleasesInProcessWriterState(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + stop := errors.New("stop during preparation") + _, err := session.Compact(context.Background(), CompactOptions{ + Prepare: func(context.Context, NativeFile, uint64) (PreparedGeneration, error) { + session.mu.Lock() + held := session.writerOpen + session.mu.Unlock() + if !held { + t.Fatal("compact did not publish its in-process writer state") + } + if writer, writerErr := session.OpenWriter(); !errors.Is(writerErr, ErrWriterBusy) { + if writer != nil { + _ = writer.Close() + } + t.Fatalf("OpenWriter during compact = %v, want %v", writerErr, ErrWriterBusy) + } + return PreparedGeneration{}, stop + }, + }) + if !errors.Is(err, stop) { + t.Fatalf("Compact error = %v, want %v", err, stop) + } + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter after compact failure: %v", err) + } + _ = writer.Close() +} + +func TestRecoverInterruptedCompactRemovesJournalOwnedScratch(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + state := session.State() + next := state + next.Generation++ + next.DeltaPath = filepath.Join(session.directory, "delta-00000000000000000002.jsonl") + scratch := filepath.Join(session.directory, ".compact-00000000000000000001.jsonl") + stateTemporary := filepath.Join(session.directory, ".state-compact-00000000000000000002.tmp") + for path, data := range map[string][]byte{ + next.DeltaPath: nil, + scratch: source, + stateTemporary: []byte("partial state"), + } { + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write interrupted compact artifact %s: %v", path, err) + } + } + if err := appendJournal(session.directory, JournalRecord{ + OperationID: "compact-00000000000000000001", SessionID: state.SessionID, + Kind: "compact", Phase: "state-publishing", Candidate: next, + TempPath: stateTemporary, FinalPath: next.DeltaPath, + Native: NativeFile{Path: scratch, Bytes: int64(len(source)), SHA256: digestBytes(source)}, + }); err != nil { + t.Fatalf("append interrupted compact journal: %v", err) + } + + reopened := openFixtureSession(t, root, manifest, reader, nil) + if reopened.State().Generation != state.Generation || reopened.State().DeltaPath != state.DeltaPath { + t.Fatalf("recovery changed committed state: %#v", reopened.State()) + } + for _, path := range []string{next.DeltaPath, scratch, stateTemporary} { + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("recovery left interrupted compact artifact %s: %v", path, err) + } + } + records, err := readJournal(session.directory) + if err != nil { + t.Fatalf("read recovered journal: %v", err) + } + latest := records[len(records)-1] + if latest.Phase != "rolled-back" || latest.TempPath != stateTemporary || latest.Native.Path != scratch { + t.Fatalf("recovery did not preserve cleanup ownership: %#v", latest) + } +} + func TestOpenSessionCleansUnlockedStaleWriterLease(t *testing.T) { root := t.TempDir() manifest, reader, _ := sessionFixture(t, root) diff --git a/internal/vfs/state.go b/internal/vfs/state.go index 841f197..4ade7d9 100644 --- a/internal/vfs/state.go +++ b/internal/vfs/state.go @@ -103,16 +103,45 @@ func DiscoverSessionStates(root string) ([]SessionState, error) { } func writeSessionState(path string, state SessionState) error { - data, err := json.MarshalIndent(state, "", " ") + data, err := encodeSessionState(state) if err != nil { - return fmt.Errorf("encode session state: %w", err) + return err } - data = append(data, '\n') directory := filepath.Dir(path) temporary, err := os.CreateTemp(directory, ".state-*.tmp") if err != nil { return fmt.Errorf("create temporary session state: %w", err) } + return commitSessionState(path, data, temporary) +} + +func writeSessionStateWithTemporary(path string, temporaryPath string, state SessionState) error { + directory := filepath.Clean(filepath.Dir(path)) + temporaryPath = filepath.Clean(temporaryPath) + if filepath.Dir(temporaryPath) != directory { + return errors.New("temporary session state must be in the state directory") + } + data, err := encodeSessionState(state) + if err != nil { + return err + } + temporary, err := os.OpenFile(temporaryPath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600) + if err != nil { + return fmt.Errorf("create temporary session state: %w", err) + } + return commitSessionState(path, data, temporary) +} + +func encodeSessionState(state SessionState) ([]byte, error) { + data, err := json.MarshalIndent(state, "", " ") + if err != nil { + return nil, fmt.Errorf("encode session state: %w", err) + } + return append(data, '\n'), nil +} + +func commitSessionState(path string, data []byte, temporary *os.File) error { + directory := filepath.Dir(path) temporaryPath := temporary.Name() defer func() { _ = os.Remove(temporaryPath) }() if err := temporary.Chmod(0o600); err != nil { From e366ae3995f1840d005c36b4d062a20c49c7223b Mon Sep 17 00:00:00 2001 From: King Star Date: Thu, 16 Jul 2026 02:55:24 +0800 Subject: [PATCH 29/33] fix: harden macOS transparent session writes --- ...1-transparent-session-filesystem-design.md | 5 +- ...ion-filesystem-implementation-alignment.md | 14 +- docs/validation-macos-canary.md | 15 +- internal/mountfs/filesystem.go | 52 ++++++- internal/mountfs/filesystem_test.go | 59 ++++++++ internal/mountfs/fuse_integration_test.go | 128 ++++++++++++++++++ internal/mountfs/host_cgofuse.go | 89 ++++++++++-- internal/mountfs/mount_policy_darwin.go | 51 +++++++ internal/mountfs/mount_policy_other.go | 7 + .../activate-canonical-after-codex-exit.sh | 82 +++++------ ...est-activate-canonical-symlink-snapshot.sh | 43 +++++- 11 files changed, 474 insertions(+), 71 deletions(-) create mode 100644 internal/mountfs/mount_policy_darwin.go create mode 100644 internal/mountfs/mount_policy_other.go diff --git a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md index 0c94334..39239f5 100644 --- a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md +++ b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md @@ -30,7 +30,7 @@ Every implementation plan, task, test report, release note, and control-plane st | `TF-014` | Native fallback deletion is disabled until platform production readiness and per-session retention gates pass. | | `TF-015` | A Codex client version without passing compatibility evidence enters compatibility quarantine: enrollment and destructive automation pause, and an already-routed session is automatically switched to a byte-verified current native writable backing before that client version may write. Routine client upgrades require no manual session preparation. | | `TF-016` | Platform filesystem prerequisites requiring elevated or system-extension approval are installed only after explicit user authorization. | -| `TF-017` | A canonical mount may never degrade into a writable ordinary directory or expose stale session files. The unmounted backing directory is empty and write-sealed, activation requires a live CodexFold mount identity, and service start succeeds only after the daemon and operational mount probe are both healthy. Desktop realpath rewrites from `CODEX_HOME/sessions` or `archived_sessions` into the mount alias are synchronously normalized in the Codex state database, and the route watcher accepts either spelling without exiting. | +| `TF-017` | A canonical mount may never degrade into a writable ordinary directory or expose stale session files. The unmounted backing directory is empty and write-sealed, activation requires a live CodexFold mount identity, and service start succeeds only after the daemon, required platform mount policy, and operational mount probe are healthy. Desktop realpath rewrites from `CODEX_HOME/sessions` or `archived_sessions` into the mount alias are synchronously normalized in the Codex state database, and the route watcher accepts either spelling without exiting. | | `TF-018` | Fork-family classification and branch archival are conservative, evidence-driven, and dry-run-first. Fork ancestry, age, size, or title alone never proves that a branch is useless. An archive mutation requires explicit user selection or an explicit policy, revalidates current Codex state and rollout bytes, and preserves a recoverable session. | | `TF-019` | Deleting a fully duplicated branch is limited to an archived session whose applicable JSONL record sequence is proven exactly and completely contained in another retained session. Apply additionally requires current-source and temporary-unfold recovery proof, transactional Codex state cleanup, and a retained tombstone and manifest. Similarity and fork ancestry are not deletion evidence. | | `TF-020` | Byte-preserving storage optimization never changes rollout bytes. Repair, reconciliation, prompt cleanup, message removal, or any other content-changing operation is a separate explicit workflow that writes a separately verified output and is never run implicitly by fold, migration, compaction, enrollment, GC, or rollback. | @@ -274,7 +274,8 @@ The trace suite covers listing, opening, scrolling old history, resume, sending - Service: user launch service with keep-alive and mount health monitoring. - Required tests: APFS native baseline, Apple Silicon, Codex Desktop, Codex CLI, canonical `sessions` and `archived_sessions` namespace moves, sleep/wake, network changes, user logout/login, daemon kill, mount restart, and Codex upgrade. - FUSE-T is the validated userspace host for this project; macFUSE is not a prerequisite for the current macOS route. -- The current flat mount is insufficient for production because Codex moves archived rollouts between canonical directories. Platform readiness requires a directory-level namespace or an equivalent mechanism that keeps those moves native-compatible. +- The FUSE-T NFS mount must use synchronous write requests before its health identity becomes readable. This is verified through the live `MNT_SYNCHRONOUS` mount flag and a real same-offset JSONL write regression; libfuse `direct_io` or disabled attribute caching alone is not accepted as evidence. +- Platform readiness requires a directory-level canonical namespace or an equivalent mechanism that keeps Codex archive and unarchive moves native-compatible. ### Linux diff --git a/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md b/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md index 56747e9..2d72f49 100644 --- a/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md +++ b/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md @@ -37,21 +37,21 @@ Baseline reviewed: commit `045eea1` on 2026-07-14. | --- | --- | --- | --- | | `TF-001` | `internal/cli/fs.go`, `internal/mountfs`, canonical migration and routing | Isolated CLI/Desktop direct-open and resume canaries | Partial: verified for isolated macOS canaries; automatic general enrollment is missing | | `TF-002` | `internal/mountfs`, `internal/sessionns`, `internal/mountid` | Real FUSE-T operation tests and isolated unmodified clients | Implemented for the validated macOS client versions | -| `TF-003` | Neutral operation layer plus exact compatibility contracts in `internal/compat` | Real macOS traces and adapter canaries | Partial: current installed clients need fresh contracts; Linux and Windows are not validated | +| `TF-003` | Neutral operation layer plus exact compatibility contracts in `internal/compat` | Real macOS traces and adapter canaries | Partial: current installed macOS clients are covered; Linux and Windows are not validated | | `TF-004` | `internal/scan`, `internal/cdc`, `internal/fold`, `internal/pack` | Repeated field, record, CDC, fork, and non-prefix corpus tests | Implemented | | `TF-005` | `internal/vfs` append delta and writer leases | Append-without-hydration tests and real CLI/Desktop append evidence | Implemented | | `TF-006` | `internal/vfs` copy-on-write backing and neutral write operations | Random-write, truncate, interruption, and real FUSE-T mutation tests | Implemented | | `TF-007` | Immutable packs, in-memory index, bounded cache, random-read resolver | Pack round-trip/corruption tests and 758 MiB packed-read benchmark | Implemented | -| `TF-008` | `internal/fsctl` benchmark and `internal/testfs` stress harness | `docs/validation-fs-preview.md` | Partial: shared-core gates pass; full real-adapter metrics and other platforms remain open | -| `TF-009` | Journal recovery, generation recovery, service keep-alive, restart-safe retirement | Recovery tests, daemon restart canaries, actual host boot of service and mount | Partial: no managed-session sleep/wake or full-host restart gate | +| `TF-008` | `internal/fsctl` benchmark and `internal/testfs` stress harness | `docs/validation-fs-preview.md` and synchronous FUSE-T read/write measurements | Partial: shared-core and measured warm macOS gates pass; cold/full-distribution metrics and other platforms remain open | +| `TF-009` | Journal recovery, generation recovery, service keep-alive, restart-safe retirement | Recovery tests, daemon restart canaries, managed Deep Idle sleep/wake, and actual retained-source host reboot | Partial: no actual power loss during an in-flight transaction | | `TF-010` | Shadow compare, optimistic routes, retained snapshots, current-byte fallback | 90,000 real random-range comparisons, rollback and failure-containment canaries | Implemented for isolated canaries | | `TF-011` | Codex state discovery primitives exist in `internal/codex` | Discovery unit tests | Missing: no stability policy, batch planner, or automatic enrollment loop | | `TF-012` | Shared Go core and macOS FUSE-T adapter | macOS real adapter tests; Linux and Windows non-CGO compile checks | Partial: Linux and Windows real adapters are missing | | `TF-013` | Canonical capability type in `internal/fsctl/status.go` | Status rejection tests and CLI status tests | Implemented; current status is `fs-engine-preview` | | `TF-014` | Snapshot retention and destructive-action guards | Migration, rollback, and quarantine tests | Implemented as a safety rule; retention promotion gates remain open | -| `TF-015` | Exact-version compatibility and update preflight quarantine | Unknown-version fallback and isolated canary tests | Implemented; the currently installed clients are presently uncovered | +| `TF-015` | Exact-version compatibility and update preflight quarantine | Unknown-version fallback and isolated canary tests | Implemented; the currently installed macOS CLI and Desktop are covered | | `TF-016` | Tagged adapter prerequisite errors and non-elevating service lifecycle | Stub, service, and authorization-gated FUSE-T evidence | Implemented | -| `TF-017` | Canonical namespace, write-sealed backing, mount identity, route normalization, process lock | Neutral, real FUSE-T, launchd, Desktop restart, and rollback tests | Implemented for macOS canaries | +| `TF-017` | Canonical namespace, write-sealed backing, mount identity, synchronous Darwin mount policy, route normalization, process lock | Neutral, real FUSE-T, launchd, Desktop restart, stale-offset write, and rollback tests | Implemented for macOS canaries | | `TF-018` | Canonical archive/unarchive file operations exist, but no conservative family classifier or guarded archive product workflow exists | No qualifying end-to-end tests | Missing | | `TF-019` | `internal/contain` and `internal/prune`; public `contains` and `remove-contained` commands | Exact containment, archived-only apply, transaction rollback, and recovery-manifest tests | Implemented | | `TF-020` | Exact fold/migrate paths are byte-preserving; `repair-rollout` and `reconcile-rollout` write separate explicit outputs | `internal/reconcile` and repair tests | Partial: add direct CLI boundary and non-invocation regression tests | @@ -72,7 +72,7 @@ Baseline reviewed: commit `045eea1` on 2026-07-14. | Task 8: standalone CLI and automatic enrollment | Partial | Commit `3352b87`; command surface and guarded lifecycle exist | Task 8 Step 5, bounded automatic enrollment, is missing | | Task 9: service lifecycle and update guard | Complete | Commit `4589ffa`; launchd and preflight tests pass | Stronger automatic update claims remain release-gated | | Task 10: synthetic, crash, performance, and compile gates | Complete for the shared engine | Commit `a1ac76e`; preview validation report | It cannot satisfy real-adapter or retention gates | -| Task 11: real macOS trace, adapter, shadow, and canary | Partial | Sanitized real CLI/Desktop/FUSE-T evidence is public | Managed-session sleep/wake and host restart, current-client compatibility, real-home canaries, and seven-day retention remain | +| Task 11: real macOS trace, adapter, shadow, and canary | Partial | Sanitized real CLI/Desktop/FUSE-T evidence, managed sleep/wake, retained-source host reboot, current-client contracts, canonical user-home activation, and a synchronous isolated real CLI canary are public | Dedicated user-home canary retention, actual in-flight power loss, and seven-day retention remain | ## Missing Product Behavior And Exact Next Work @@ -84,7 +84,7 @@ Baseline reviewed: commit `045eea1` on 2026-07-14. | Content-changing boundary regression | Add CLI tests proving repair and reconciliation require a separate output and cannot be invoked by fold, migration, compaction, enrollment, rollback, or GC | Command-tree tests, call-boundary tests, unchanged source hashes, and verified output hashes | | Hard disk budgets and bounded retention | Add a storage inventory and preflight budget used by every operation that can create a full copy or generation; enforce one migration snapshot, one current fallback, one transaction scratch file, and bounded old generations | Low-space refusal before write, repeated migration/rollback/enrollment, interrupted cleanup, live lease retention, and no unbounded retired-state growth | | Actual physical reclamation reporting | Extend status, doctor, and mutating results with logical, unique, pack, source, snapshot, fallback, temporary/recovery, projected peak, projected reclaimable, and actual reclaimed bytes | Fixture accounting checked against filesystem allocation before and after GC/removal; zero reclaimed bytes while full copies remain | -| Current macOS compatibility and disruptive gates | Import exact contracts for installed clients; run a retained-source managed canary through sleep/wake and host restart; then start bounded real-home canaries only after automatic enrollment and disk budgets pass | Clean doctor, exact SHA after restart cases, rollback, no route loss, and seven incident-free days | +| Remaining macOS disruptive and retention gates | Keep the dedicated retained-source user-home canary bounded to one explicitly selected session; perform an actual in-flight power-loss test only in a disposable host or VM; then complete the incident-free retention window | Clean doctor, exact SHA after restart and recovery cases, rollback, no route loss, and seven incident-free days | | Linux and Windows readiness | Implement FUSE3 and WinFsp adapters without moving shared behavior out of the core | Native operation traces, crash/restart, performance, upgrade quarantine, rollback, and retention on each platform | ## Current Capability Decision diff --git a/docs/validation-macos-canary.md b/docs/validation-macos-canary.md index d7f1bcb..d7e2ee2 100644 --- a/docs/validation-macos-canary.md +++ b/docs/validation-macos-canary.md @@ -2,7 +2,18 @@ ## Current Status -The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have passed for read, append, resume, fork, child-session enrollment, canonical archive/unarchive moves, launchd restart, rollback, namespace deactivation, and unknown-version quarantine. A retained-source CLI canary survived an actual host reboot while managed, then resumed through the recovered mount and rolled back to an exact native JSONL. The currently installed CLI and Desktop versions also passed exact compatibility and isolated retained-source canaries. Process-level interruption recovery now covers append, compaction, migration, and rollback, and a managed session passed an actual Deep Idle sleep/wake cycle followed by a real model turn. The project remains at `fs-engine-preview` because the user Codex home is intentionally not enrolled, in-flight transaction evidence does not claim an actual power-loss test, and canary retention has not started. +The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have passed for read, append, resume, fork, child-session enrollment, canonical archive/unarchive moves, launchd restart, rollback, namespace deactivation, and unknown-version quarantine. A retained-source CLI canary survived an actual host reboot while managed, then resumed through the recovered mount and rolled back to an exact native JSONL. The currently installed CLI and Desktop versions also passed exact compatibility and isolated retained-source canaries. Process-level interruption recovery now covers append, compaction, migration, and rollback, and a managed session passed an actual Deep Idle sleep/wake cycle followed by a real model turn. The user Codex home now uses the canonical namespace with ordinary sessions remaining native passthrough and zero managed sessions at activation. The project remains at `fs-engine-preview` because its dedicated retained-source user-home canary has not completed retention, in-flight transaction evidence does not claim an actual power-loss test, and the seven-day incident-free gate has not started. + +Additional synchronous-write and canonical-activation evidence on 2026-07-16: + +- Canonical activation preserved all 2,313 native rollouts. The mounted and native path/size/mtime inventories matched exactly, and the pre/post underlying native inventory also matched path, size, mtime, inode, mode, owner, and group. Explicit critical canaries retained full SHA-256 checks. Ordinary sessions remained native passthrough and the managed-session count stayed zero. +- The one-shot background activation job completed the namespace switch but failed before reopening Desktop because macOS denied that LaunchAgent a recursive traversal of the network-volume-backed mount. Foreground Codex processes could traverse the same mount. The activation script now inventories the underlying native tree instead of recursively hashing or traversing the mounted tree, rechecks Desktop, CLI, and app-server quiescence immediately before activation, and retries preflight if Codex reopens. The user reopened Desktop manually; no rollout content was changed by the failed reopen step. +- The first dedicated real-home migration passed exact shadow verification and 10,000 random reads, but a real CLI turn exposed a corruption bug: the original 97,388-byte prefix remained exact while the final JSONL contained a partially overwritten record. The canary was rolled back and restored byte-for-byte before further work. +- The real write trace showed that Codex opens rollout JSONL with `O_RDWR` and explicit offsets. A same-handle JSONL append guard was added, but the failing FUSE-T regression proved that the macOS NFS client could merge two same-offset `pwrite` calls before either reached CodexFold. Per-open and global libfuse `direct_io`, plus disabled NFS attribute caching, did not change that behavior. +- Updating only the mounted localhost NFS volume with `mount -u -o sync` made the previously deterministic stale-offset regression pass. The Darwin adapter now withholds its health identity until that update succeeds and `MNT_SYNCHRONOUS` is visible through `statfs`; a failure unmounts the host instead of advertising readiness. No global NFS configuration, patched FUSE-T binary, privileged helper, or system-wide mount change is used. +- A 64 MiB mounted read measured 7,045 MiB/s versus 7,435 MiB/s from the native APFS file, or 95% of native throughput. Across 200 JSONL append-plus-`fsync` operations, the synchronous mount averaged 3.99 ms with a 5.03 ms p95, versus 3.84 ms and 5.26 ms natively. +- A fresh isolated real CLI canary used the official unarchive flow and then resumed through the synchronous canonical mount. The complete view grew from 97,388 to 120,859 bytes and 33 valid JSONL records. The complete original prefix retained SHA-256 `4cd4bcc1807d875b70e04b3028441f330f9c7ee0cd41cbcff08c18c9ec44d416`, the 23,471-byte delta parsed independently, the expected historical and new markers were recalled, generation remained 1, and no writable backing appeared. +- Default, FUSE-tagged, race, vet, shell, cross-platform compile, and complete real FUSE-T suites passed after the fix. The real FUSE-T suite explicitly requires synchronous mount readiness before exercising the stale-offset regression. Additional current-client, interruption, and sleep/wake evidence on 2026-07-15: @@ -41,7 +52,7 @@ Additional failure-containment evidence on 2026-07-14: - Canonical migration now verifies the mounted managed target before removing the native directory entry. A clean first migration passed without a retry. - A real Desktop canary preserved the exact 79,067-byte, 16-record source prefix, appended an 8,414-byte, 12-record managed delta, and rolled back to their exact 87,481-byte concatenation. A subsequent native Desktop turn appended 5,590 bytes and 9 records. The final 93,071-byte, 37-record JSONL parsed completely and preserved byte order. - Rollback now holds an exclusive writer lease across materialization and state retirement. A live FUSE writer and a real Desktop app-server both caused rollback to fail closed; after every writer drained, rollback preserved the exact visible SHA-256 and a new Desktop turn persisted to the native JSONL. -- The canonical activation gate hashes every rollout before and after namespace activation. File size alone is no longer accepted as full-history evidence. +- The canonical activation gate compares every native rollout's path, size, modification time, inode, mode, owner, and group before and after the directory rename. Explicit critical sessions retain full SHA-256 comparisons. Mounted visibility is checked separately after activation because a background LaunchAgent can be denied recursive traversal of the FUSE-T mount even while the foreground Codex client can use it normally. This avoids holding Codex closed while reading the complete session corpus without falling back to file-size-only evidence. Additional failure-containment evidence on 2026-07-13: diff --git a/internal/mountfs/filesystem.go b/internal/mountfs/filesystem.go index fa28d57..a6360b1 100644 --- a/internal/mountfs/filesystem.go +++ b/internal/mountfs/filesystem.go @@ -1,9 +1,11 @@ package mountfs import ( + "bytes" "context" "crypto/sha256" "encoding/hex" + "encoding/json" "errors" "io" "os" @@ -25,12 +27,15 @@ type Attr struct { } type fileHandle struct { - mu sync.Mutex - session *vfs.Session - native *os.File - read *vfs.ReadHandle - write *vfs.WriteHandle - append bool + mu sync.Mutex + session *vfs.Session + native *os.File + read *vfs.ReadHandle + write *vfs.WriteHandle + append bool + appendStream bool + appendFloor int64 + appendOffset int64 } type Filesystem struct { @@ -445,7 +450,22 @@ func (f *Filesystem) Write(handleID uint64, data []byte, offset int64) (int, sys } if offset == info.Size { n, err = handle.write.Append(context.Background(), data) + if err == nil { + if !handle.appendStream { + handle.appendFloor = offset + } + handle.appendStream = true + handle.appendOffset = offset + int64(n) + } + } else if handle.appendStream && + offset >= handle.appendFloor && offset < handle.appendOffset && + info.Size == handle.appendOffset && completeJSONL(data) { + n, err = handle.write.Append(context.Background(), data) + if err == nil { + handle.appendOffset += int64(n) + } } else { + handle.appendStream = false n, err = handle.write.WriteAt(context.Background(), data, offset) } } @@ -476,6 +496,9 @@ func (f *Filesystem) Truncate(handleID uint64, size int64) syscall.Errno { if err := handle.write.Truncate(context.Background(), size); err != nil { return errnoFor(err) } + if handle.appendStream && size != handle.appendOffset { + handle.appendStream = false + } if handle.read != nil { return refreshReader(handle) } @@ -497,6 +520,9 @@ func (f *Filesystem) TruncatePath(name string, size int64) syscall.Errno { if err := handle.write.Truncate(context.Background(), size); err != nil { return errnoFor(err) } + if handle.appendStream && size != handle.appendOffset { + handle.appendStream = false + } if handle.read != nil { return refreshReader(handle) } @@ -514,6 +540,20 @@ func (f *Filesystem) TruncatePath(name string, size int64) syscall.Errno { return errnoFor(closeErr) } +func completeJSONL(data []byte) bool { + if len(data) == 0 || data[len(data)-1] != '\n' { + return false + } + for len(data) > 0 { + end := bytes.IndexByte(data, '\n') + if end <= 0 || !json.Valid(data[:end]) { + return false + } + data = data[end+1:] + } + return true +} + func (f *Filesystem) lockActiveWriter(session *vfs.Session) *fileHandle { f.mu.RLock() defer f.mu.RUnlock() diff --git a/internal/mountfs/filesystem_test.go b/internal/mountfs/filesystem_test.go index 753f2b7..66e161b 100644 --- a/internal/mountfs/filesystem_test.go +++ b/internal/mountfs/filesystem_test.go @@ -122,6 +122,65 @@ func TestFilesystemWriteAtVisibleEOFUsesDeltaWithoutCopyOnWrite(t *testing.T) { } } +func TestFilesystemStaleTailOffsetAppendsCompleteJSONLRecord(t *testing.T) { + source := []byte("{\"record\":0}\n") + session := mountSessionFixture(t, "session", source) + filesystem := New() + if err := filesystem.AddSession("session", session); err != nil { + t.Fatal(err) + } + handle, errno := filesystem.Open("/session.jsonl", os.O_RDWR) + if errno != 0 { + t.Fatalf("Open writer errno=%v", errno) + } + t.Cleanup(func() { _ = filesystem.Release(handle) }) + first := []byte("{\"record\":1}\n") + second := []byte("{\"record\":2}\n") + staleEOF := int64(len(source)) + if n, errno := filesystem.Write(handle, first, staleEOF); errno != 0 || n != len(first) { + t.Fatalf("first append = %d errno=%v", n, errno) + } + if n, errno := filesystem.Write(handle, second, staleEOF); errno != 0 || n != len(second) { + t.Fatalf("stale-offset append = %d errno=%v", n, errno) + } + if state := session.State(); state.BackingPath != "" { + t.Fatalf("stale JSONL tail offset created copy-on-write backing %q", state.BackingPath) + } + want := append(append(append([]byte(nil), source...), first...), second...) + current := make([]byte, len(want)) + if n, errno := filesystem.Read(handle, current, 0); errno != 0 || n != len(want) { + t.Fatalf("Read after stale-offset append = %d errno=%v", n, errno) + } + if !bytes.Equal(current, want) { + t.Fatalf("visible bytes differ: got=%q want=%q", current, want) + } +} + +func TestFilesystemStaleTailOffsetWithArbitraryBytesUsesCopyOnWrite(t *testing.T) { + source := []byte("{\"record\":0}\n") + session := mountSessionFixture(t, "session", source) + filesystem := New() + if err := filesystem.AddSession("session", session); err != nil { + t.Fatal(err) + } + handle, errno := filesystem.Open("/session.jsonl", os.O_RDWR) + if errno != 0 { + t.Fatalf("Open writer errno=%v", errno) + } + t.Cleanup(func() { _ = filesystem.Release(handle) }) + first := []byte("{\"record\":1}\n") + staleEOF := int64(len(source)) + if n, errno := filesystem.Write(handle, first, staleEOF); errno != 0 || n != len(first) { + t.Fatalf("first append = %d errno=%v", n, errno) + } + if n, errno := filesystem.Write(handle, []byte("PATCH"), staleEOF); errno != 0 || n != len("PATCH") { + t.Fatalf("random write = %d errno=%v", n, errno) + } + if state := session.State(); state.BackingPath == "" { + t.Fatal("arbitrary stale-offset write did not create copy-on-write backing") + } +} + func TestFilesystemPathTruncateUsesTheActiveWriter(t *testing.T) { filesystem, source := mountFixture(t) handle, errno := filesystem.Open("/session.jsonl", os.O_RDWR) diff --git a/internal/mountfs/fuse_integration_test.go b/internal/mountfs/fuse_integration_test.go index 2e3734e..d01e3ba 100644 --- a/internal/mountfs/fuse_integration_test.go +++ b/internal/mountfs/fuse_integration_test.go @@ -20,9 +20,65 @@ import ( "github.com/jstar0/codexfold/internal/fold" "github.com/jstar0/codexfold/internal/service" "github.com/jstar0/codexfold/internal/vfs" + "github.com/winfsp/cgofuse/fuse" "golang.org/x/sys/unix" ) +func TestOperationTraceRecordsWriteShapeWithoutPath(t *testing.T) { + var recorded []string + filesystem := &fuseFilesystem{recorder: func(operation string) { + recorded = append(recorded, operation) + }} + privatePath := "/sessions/private-session.jsonl" + filesystem.recordOpen("open", privatePath, 0x9, os.O_WRONLY|os.O_APPEND, 17, 0) + filesystem.recordIO("write", privatePath, 17, 1234, 89, 89) + + joined := strings.Join(recorded, "\n") + for _, field := range []string{ + "open kind=session flags=0x9 translated=0x9 handle=17 result=0", + "write kind=session handle=17 offset=1234 bytes=89 result=89", + } { + if !strings.Contains(joined, field) { + t.Fatalf("operation trace missing %q: %s", field, joined) + } + } + if strings.Contains(joined, privatePath) || strings.Contains(joined, "private-session") { + t.Fatalf("operation trace exposed a session path: %s", joined) + } +} + +func TestOpenExUsesDirectIOOnlyForWritableSessions(t *testing.T) { + source := []byte("{\"record\":0}\n") + managed := mountSessionFixture(t, "direct-io", source) + core := New() + if err := core.AddSession("direct-io", managed); err != nil { + t.Fatal(err) + } + filesystem := &fuseFilesystem{core: core} + + readOnly := fuse.FileInfo_t{Flags: fuse.O_RDONLY} + if result := filesystem.OpenEx("/direct-io.jsonl", &readOnly); result != 0 { + t.Fatalf("read-only OpenEx result=%d", result) + } + if readOnly.DirectIo { + t.Fatal("read-only session unexpectedly enabled direct I/O") + } + if errno := core.Release(readOnly.Fh); errno != 0 { + t.Fatalf("release read-only handle errno=%v", errno) + } + + writable := fuse.FileInfo_t{Flags: fuse.O_RDWR} + if result := filesystem.OpenEx("/direct-io.jsonl", &writable); result != 0 { + t.Fatalf("writable OpenEx result=%d", result) + } + if !writable.DirectIo { + t.Fatal("writable session did not enable direct I/O") + } + if errno := core.Release(writable.Fh); errno != 0 { + t.Fatalf("release writable handle errno=%v", errno) + } +} + func TestRealFuseMountNativeFileOperations(t *testing.T) { if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real macFUSE adapter test") @@ -152,6 +208,78 @@ func TestRealFuseMountNativeFileOperations(t *testing.T) { waitForRealUnmount(t, mountPoint) } +func TestRealFuseManagedStaleTailOffsetsPreserveJSONL(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + source := []byte("{\"record\":0}\n") + managed := mountSessionFixture(t, "stale-tail", source) + filesystem := New() + if err := filesystem.AddSession("stale-tail", managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + var traceMu sync.Mutex + var trace []string + options := HostOptions{ + MountPoint: mountPoint, + Filesystem: filesystem, + Foreground: true, + OperationRecorder: func(operation string) { + traceMu.Lock() + defer traceMu.Unlock() + trace = append(trace, operation) + }, + } + stopMount := startRealMountWithOptions(t, options) + var mountStat unix.Statfs_t + if err := unix.Statfs(mountPoint, &mountStat); err != nil { + t.Fatal(err) + } + if mountStat.Flags&unix.MNT_SYNCHRONOUS == 0 { + t.Fatal("real FUSE-T mount was reported healthy before synchronous I/O was enabled") + } + target := filepath.Join(mountPoint, "stale-tail.jsonl") + file, err := os.OpenFile(target, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + first := []byte("{\"record\":1}\n") + second := []byte("{\"record\":2}\n") + staleEOF := int64(len(source)) + if _, err := file.WriteAt(first, staleEOF); err != nil { + _ = file.Close() + t.Fatal(err) + } + if _, err := file.WriteAt(second, staleEOF); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + want := append(append(append([]byte(nil), source...), first...), second...) + got, err := os.ReadFile(target) + if err != nil || !bytes.Equal(got, want) { + traceMu.Lock() + defer traceMu.Unlock() + t.Fatalf("stale-tail visible bytes differ: got=%q want=%q err=%v trace=%q", got, want, err, trace) + } + if state := managed.State(); state.BackingPath != "" { + t.Fatalf("stale-tail writes created backing %q", state.BackingPath) + } + stopMount() + waitForRealUnmount(t, mountPoint) +} + func TestRealFuseCanonicalNativeToManagedCutover(t *testing.T) { if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") diff --git a/internal/mountfs/host_cgofuse.go b/internal/mountfs/host_cgofuse.go index b4ba78d..c8bfa07 100644 --- a/internal/mountfs/host_cgofuse.go +++ b/internal/mountfs/host_cgofuse.go @@ -11,6 +11,7 @@ import ( "path/filepath" "runtime" "strings" + "sync/atomic" "syscall" "github.com/jstar0/codexfold/internal/mountid" @@ -23,6 +24,7 @@ type fuseFilesystem struct { core *Filesystem recorder func(string) mountIdentity []byte + mountReady atomic.Bool } const healthHandle = ^uint64(0) - 1 @@ -32,6 +34,9 @@ func Available() bool { return true } func (f *fuseFilesystem) Getattr(name string, stat *fuse.Stat_t, _ uint64) int { f.record("getattr") if cleanPath(name) == "/"+mountid.Path { + if !f.mountReady.Load() { + return -int(syscall.ENOENT) + } stat.Mode = syscall.S_IFREG | 0o400 stat.Size = int64(len(f.mountIdentity)) stat.Nlink = 1 @@ -118,32 +123,57 @@ func (f *fuseFilesystem) Readdir(name string, fill func(string, *fuse.Stat_t, in func (f *fuseFilesystem) Open(name string, flags int) (int, uint64) { if cleanPath(name) == "/"+mountid.Path { + if !f.mountReady.Load() { + return -int(syscall.ENOENT), ^uint64(0) + } if flags&fuse.O_ACCMODE != fuse.O_RDONLY { return -int(syscall.EPERM), ^uint64(0) } return 0, healthHandle } - handle, errno := f.core.Open(name, translateOpenFlags(flags)) + translated := translateOpenFlags(flags) + handle, errno := f.core.Open(name, translated) if errno != 0 { result := -int(errno) - f.recordResult("open", name, result) + f.recordOpen("open", name, flags, translated, handle, result) return result, ^uint64(0) } - f.recordResult("open", name, 0) + f.recordOpen("open", name, flags, translated, handle, 0) return 0, handle } func (f *fuseFilesystem) Create(name string, flags int, _ uint32) (int, uint64) { - handle, errno := f.core.Open(name, translateOpenFlags(flags)|os.O_CREATE) + translated := translateOpenFlags(flags) | os.O_CREATE + handle, errno := f.core.Open(name, translated) if errno != 0 { result := -int(errno) - f.recordResult("create", name, result) + f.recordOpen("create", name, flags, translated, handle, result) return result, ^uint64(0) } - f.recordResult("create", name, 0) + f.recordOpen("create", name, flags, translated, handle, 0) return 0, handle } +func (f *fuseFilesystem) OpenEx(name string, info *fuse.FileInfo_t) int { + result, handle := f.Open(name, info.Flags) + if result == 0 { + info.Fh = handle + info.DirectIo = writableSession(name, info.Flags) + f.record(fmt.Sprintf("open_config kind=%s handle=%d direct_io=%t", operationKind(name), handle, info.DirectIo)) + } + return result +} + +func (f *fuseFilesystem) CreateEx(name string, _ uint32, info *fuse.FileInfo_t) int { + result, handle := f.Create(name, info.Flags, 0o600) + if result == 0 { + info.Fh = handle + info.DirectIo = writableSession(name, info.Flags) + f.record(fmt.Sprintf("create_config kind=%s handle=%d direct_io=%t", operationKind(name), handle, info.DirectIo)) + } + return result +} + func (f *fuseFilesystem) Read(_ string, destination []byte, offset int64, handle uint64) int { f.record("read") if handle == healthHandle { @@ -159,24 +189,27 @@ func (f *fuseFilesystem) Read(_ string, destination []byte, offset int64, handle return n } -func (f *fuseFilesystem) Write(_ string, data []byte, offset int64, handle uint64) int { - f.record("write") +func (f *fuseFilesystem) Write(name string, data []byte, offset int64, handle uint64) int { n, errno := f.core.Write(handle, data, offset) if errno != 0 { - return -int(errno) + result := -int(errno) + f.recordIO("write", name, handle, offset, len(data), result) + return result } + f.recordIO("write", name, handle, offset, len(data), n) return n } func (f *fuseFilesystem) Truncate(name string, size int64, handle uint64) int { - f.record("truncate") var errno syscall.Errno if handle == 0 || handle == ^uint64(0) { errno = f.core.TruncatePath(name, size) } else { errno = f.core.Truncate(handle, size) } - return -int(errno) + result := -int(errno) + f.record(fmt.Sprintf("truncate kind=%s handle=%d size=%d result=%d", operationKind(name), handle, size, result)) + return result } func (f *fuseFilesystem) Flush(_ string, handle uint64) int { @@ -419,6 +452,18 @@ func (f *fuseFilesystem) record(operation string) { } func (f *fuseFilesystem) recordResult(operation string, name string, result int) { + f.record(fmt.Sprintf("%s kind=%s result=%d", operation, operationKind(name), result)) +} + +func (f *fuseFilesystem) recordOpen(operation string, name string, flags int, translated int, handle uint64, result int) { + f.record(fmt.Sprintf("%s kind=%s flags=%#x translated=%#x handle=%d result=%d", operation, operationKind(name), flags, translated, handle, result)) +} + +func (f *fuseFilesystem) recordIO(operation string, name string, handle uint64, offset int64, bytes int, result int) { + f.record(fmt.Sprintf("%s kind=%s handle=%d offset=%d bytes=%d result=%d", operation, operationKind(name), handle, offset, bytes, result)) +} + +func operationKind(name string) string { kind := "other" base := filepath.Base(name) if strings.HasPrefix(base, "._") { @@ -426,7 +471,11 @@ func (f *fuseFilesystem) recordResult(operation string, name string, result int) } else if strings.HasSuffix(base, ".jsonl") { kind = "session" } - f.record(fmt.Sprintf("%s kind=%s result=%d", operation, kind, result)) + return kind +} + +func writableSession(name string, flags int) bool { + return operationKind(name) == "session" && flags&fuse.O_ACCMODE != fuse.O_RDONLY } func translateOpenFlags(flags int) int { @@ -479,7 +528,20 @@ func mountHost(ctx context.Context, options HostOptions) (result error) { case <-done: } }() + policyContext, cancelPolicy := context.WithCancel(ctx) + policyDone := make(chan error, 1) + go func() { + err := configureMountedFilesystem(policyContext, options.MountPoint) + if err == nil { + filesystem.mountReady.Store(true) + } else { + _ = host.Unmount() + } + policyDone <- err + }() mounted := host.Mount(options.MountPoint, arguments) + cancelPolicy() + policyErr := <-policyDone close(done) if err := ctx.Err(); err != nil { return err @@ -487,5 +549,8 @@ func mountHost(ctx context.Context, options HostOptions) (result error) { if !mounted { return errors.New("FUSE host exited without mounting") } + if policyErr != nil { + return fmt.Errorf("configure mounted filesystem: %w", policyErr) + } return ctx.Err() } diff --git a/internal/mountfs/mount_policy_darwin.go b/internal/mountfs/mount_policy_darwin.go new file mode 100644 index 0000000..cbc5359 --- /dev/null +++ b/internal/mountfs/mount_policy_darwin.go @@ -0,0 +1,51 @@ +//go:build darwin && fuse && cgo + +package mountfs + +import ( + "bytes" + "context" + "fmt" + "os/exec" + "time" + + "golang.org/x/sys/unix" +) + +func configureMountedFilesystem(ctx context.Context, mountPoint string) error { + deadline, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + + var lastErr error + for { + var stat unix.Statfs_t + if err := unix.Statfs(mountPoint, &stat); err == nil && statfsType(stat) == "nfs" { + output, err := exec.CommandContext(deadline, "/sbin/mount", "-u", "-o", "sync", mountPoint).CombinedOutput() + if err == nil { + if err := unix.Statfs(mountPoint, &stat); err == nil && stat.Flags&unix.MNT_SYNCHRONOUS != 0 { + return nil + } + lastErr = fmt.Errorf("NFS mount did not report synchronous I/O") + } else if deadline.Err() == nil { + lastErr = fmt.Errorf("update NFS mount: %w: %s", err, bytes.TrimSpace(output)) + } + } + + select { + case <-deadline.Done(): + if lastErr != nil { + return lastErr + } + return deadline.Err() + case <-time.After(25 * time.Millisecond): + } + } +} + +func statfsType(stat unix.Statfs_t) string { + length := bytes.IndexByte(stat.Fstypename[:], 0) + if length < 0 { + length = len(stat.Fstypename) + } + return string(stat.Fstypename[:length]) +} diff --git a/internal/mountfs/mount_policy_other.go b/internal/mountfs/mount_policy_other.go new file mode 100644 index 0000000..6f68c6a --- /dev/null +++ b/internal/mountfs/mount_policy_other.go @@ -0,0 +1,7 @@ +//go:build !darwin && fuse && cgo + +package mountfs + +import "context" + +func configureMountedFilesystem(context.Context, string) error { return nil } diff --git a/scripts/activate-canonical-after-codex-exit.sh b/scripts/activate-canonical-after-codex-exit.sh index bc58c96..4da41e9 100755 --- a/scripts/activate-canonical-after-codex-exit.sh +++ b/scripts/activate-canonical-after-codex-exit.sh @@ -55,30 +55,28 @@ app_servers_running() { return 1 } -echo "waiting for Codex Desktop and CLI to exit" -while codex_running; do - sleep 2 -done -for _ in 1 2 3; do - sleep 1 - if codex_running; then - while codex_running; do - sleep 2 - done - fi -done -drained=0 -for _ in {1..30}; do - if ! app_servers_running; then - drained=1 - break - fi - sleep 1 -done -if (( drained == 0 )); then +wait_for_codex_drain() { + echo "waiting for Codex Desktop and CLI to exit" + while codex_running; do + sleep 2 + done + for _ in 1 2 3; do + sleep 1 + if codex_running; then + while codex_running; do + sleep 2 + done + fi + done + for _ in {1..30}; do + if ! app_servers_running; then + return 0 + fi + sleep 1 + done echo "real-home Codex app servers did not drain" - exit 1 -fi + return 1 +} service_status="$(${BIN} fs service status --json)" jq -e '.daemon_running == true and .mount_healthy == true' <<<"${service_status}" >/dev/null @@ -90,35 +88,43 @@ managed_count="$(find "${STORE}/fs/sessions" -type f -name state.json 2>/dev/nul fold_route_count="$(sqlite3 "${CODEX_HOME}/state_5.sqlite" "select count(*) from threads where rollout_path like '${MOUNT}/%';")" [[ "${fold_route_count}" == "0" ]] -snapshot_tree() { - output="$1" +snapshot_native_tree() { + local root="$1" + local output="$2" ( - cd "${CODEX_HOME}" - while IFS= read -r -d '' rollout; do - size="$(stat -f '%z' "${rollout}")" - digest="$(shasum -a 256 "${rollout}" | awk '{print $1}')" - printf '%s\t%s\t%s\n' "${rollout}" "${size}" "${digest}" - done < <(find -H sessions archived_sessions -type f ! -name '._*' -print0 | sort -z) + cd "${root}" + find -H sessions archived_sessions -type f ! -name '._*' \ + -exec stat -f '%N|%z|%m|%i|%p|%u|%g' {} + | LC_ALL=C sort ) >"${output}" } snapshot_critical() { - output="$1" + local root="$1" + local output="$2" + local id rollout digest : >"${output}" [[ -z "${CRITICAL_IDS_FILE}" ]] && return 0 [[ -f "${CRITICAL_IDS_FILE}" ]] while IFS= read -r id || [[ -n "${id}" ]]; do [[ -z "${id}" || "${id}" == \#* ]] && continue grep -Eq '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' <<<"${id}" - rollout="$(find -H "${CODEX_HOME}/sessions" "${CODEX_HOME}/archived_sessions" -type f -name "rollout-*-${id}.jsonl" ! -name '._*' -print -quit)" + rollout="$(find -H "${root}/sessions" "${root}/archived_sessions" -type f -name "rollout-*-${id}.jsonl" ! -name '._*' -print -quit)" [[ -n "${rollout}" ]] digest="$(shasum -a 256 "${rollout}" | awk '{print $1}')" printf '%s\t%s\n' "${id}" "${digest}" >>"${output}" done <"${CRITICAL_IDS_FILE}" } -snapshot_tree "${RUN_ROOT}/tree.before" -snapshot_critical "${RUN_ROOT}/critical.before" +while true; do + wait_for_codex_drain + snapshot_native_tree "${CODEX_HOME}" "${RUN_ROOT}/tree.before" + snapshot_critical "${CODEX_HOME}" "${RUN_ROOT}/critical.before" + if codex_running || app_servers_running; then + echo "Codex restarted during activation preflight; waiting again" + continue + fi + break +done "${BIN}" fs namespace activate --apply \ --codex-home "${CODEX_HOME}" --mount "${MOUNT}" --native-root "${NATIVE_ROOT}" --json \ @@ -130,9 +136,9 @@ activated=1 trigger_count="$(sqlite3 "${CODEX_HOME}/state_5.sqlite" "select count(*) from sqlite_master where type='trigger' and name like 'codexfold_normalize_rollout_path_%';")" [[ "${trigger_count}" == "2" ]] -snapshot_tree "${RUN_ROOT}/tree.after" -snapshot_critical "${RUN_ROOT}/critical.after" -diff -u "${RUN_ROOT}/tree.before" "${RUN_ROOT}/tree.after" +snapshot_native_tree "${NATIVE_ROOT}" "${RUN_ROOT}/tree.native.after" +snapshot_critical "${NATIVE_ROOT}" "${RUN_ROOT}/critical.after" +diff -u "${RUN_ROOT}/tree.before" "${RUN_ROOT}/tree.native.after" diff -u "${RUN_ROOT}/critical.before" "${RUN_ROOT}/critical.after" service_status="$(${BIN} fs service status --json)" diff --git a/scripts/tests/test-activate-canonical-symlink-snapshot.sh b/scripts/tests/test-activate-canonical-symlink-snapshot.sh index 2a22efb..92fee63 100755 --- a/scripts/tests/test-activate-canonical-symlink-snapshot.sh +++ b/scripts/tests/test-activate-canonical-symlink-snapshot.sh @@ -4,11 +4,16 @@ set -euo pipefail repo_root=$(cd "$(dirname "$0")/../.." && pwd) script="$repo_root/scripts/activate-canonical-after-codex-exit.sh" -grep -Fq 'find -H sessions archived_sessions' "$script" -grep -Fq 'find -H "${CODEX_HOME}/sessions" "${CODEX_HOME}/archived_sessions"' "$script" +grep -Fq 'snapshot_native_tree()' "$script" +grep -Fq "-exec stat -f '%N|%z|%m|%i|%p|%u|%g'" "$script" +! grep -Fq 'snapshot_visible_tree()' "$script" +grep -Fq 'find -H "${root}/sessions" "${root}/archived_sessions"' "$script" grep -Fq 'shasum -a 256 "${rollout}"' "$script" +[[ "$(grep -Fc 'shasum -a 256' "$script")" == "1" ]] +grep -Fq 'snapshot_critical "${NATIVE_ROOT}" "${RUN_ROOT}/critical.after"' "$script" grep -Fq 'app_servers_running()' "$script" grep -Fq 'real-home Codex app servers did not drain' "$script" +grep -Fq 'Codex restarted during activation preflight; waiting again' "$script" stop_line=$(grep -n 'fs service stop --apply' "$script" | head -n 1 | cut -d: -f1) deactivate_line=$(grep -n 'fs namespace deactivate --apply' "$script" | head -n 1 | cut -d: -f1) @@ -35,12 +40,33 @@ sqlite3 "$runtime_root/home/state_5.sqlite" 'create table threads (rollout_path cat >"$runtime_root/bin/pgrep" <<'EOF' #!/bin/sh +if [ -e "$CODEXFOLD_FAKE_REOPEN_MARKER" ]; then + rm -f "$CODEXFOLD_FAKE_REOPEN_MARKER" + exit 0 +fi exit 1 EOF cat >"$runtime_root/bin/sleep" <<'EOF' #!/bin/sh exit 0 EOF +cat >"$runtime_root/bin/find" <<'EOF' +#!/bin/sh +case "$*" in + '-H sessions archived_sessions'*) + if [ ! -e "$CODEXFOLD_FAKE_FIND_TRIGGERED" ]; then + : >"$CODEXFOLD_FAKE_FIND_TRIGGERED" + : >"$CODEXFOLD_FAKE_REOPEN_MARKER" + fi + ;; +esac +exec /usr/bin/find "$@" +EOF +cat >"$runtime_root/bin/shasum" <<'EOF' +#!/bin/sh +echo "full-tree SHA must not run during activation" >&2 +exit 99 +EOF cat >"$runtime_root/bin/codexfold" <<'EOF' #!/bin/sh printf '%s\n' "$*" >>"$CODEXFOLD_FAKE_LOG" @@ -56,12 +82,15 @@ case "$*" in ;; esac EOF -chmod +x "$runtime_root/bin/pgrep" "$runtime_root/bin/sleep" "$runtime_root/bin/codexfold" +chmod +x "$runtime_root/bin/pgrep" "$runtime_root/bin/sleep" "$runtime_root/bin/find" "$runtime_root/bin/shasum" "$runtime_root/bin/codexfold" runtime_script="$runtime_root/activate.zsh" sed "s|^export PATH=.*|export PATH=\"$runtime_root/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin\"|" "$script" >"$runtime_script" runtime_log="$runtime_root/commands.log" -if CODEXFOLD_FAKE_LOG="$runtime_log" /bin/zsh "$runtime_script" \ +if CODEXFOLD_FAKE_LOG="$runtime_log" \ + CODEXFOLD_FAKE_REOPEN_MARKER="$runtime_root/reopened" \ + CODEXFOLD_FAKE_FIND_TRIGGERED="$runtime_root/find-triggered" \ + /bin/zsh "$runtime_script" \ "$runtime_root/home" "$runtime_root/store" "$runtime_root/mount" "$runtime_root/native" \ "$runtime_root/bin/codexfold" 0; then echo "activation unexpectedly succeeded" >&2 @@ -71,6 +100,12 @@ fi grep -Fqx 'fs service stop --apply' "$runtime_log" grep -Fq 'fs namespace deactivate --apply' "$runtime_log" grep -Fq 'fs service start --apply' "$runtime_log" +[[ "$(grep -Fc 'fs namespace activate' "$runtime_log")" == "1" ]] +run_log=$(find "$runtime_root/store/activation" -type f -name run.log -print -quit) +if ! grep -Fq 'Codex restarted during activation preflight; waiting again' "$run_log"; then + sed -n '1,120p' "$run_log" >&2 + exit 1 +fi failed_marker=$(find "$runtime_root/store/activation" -type f -name FAILED -print -quit) [[ -n "$failed_marker" ]] From 238930fc46f20ca0e123a454192267802d451cc4 Mon Sep 17 00:00:00 2001 From: King Star Date: Thu, 16 Jul 2026 03:03:24 +0800 Subject: [PATCH 30/33] docs: record synchronous macOS canary evidence --- ...ransparent-session-filesystem-implementation-alignment.md | 4 ++-- docs/validation-macos-canary.md | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md b/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md index 2d72f49..c6d7ec1 100644 --- a/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md +++ b/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md @@ -44,7 +44,7 @@ Baseline reviewed: commit `045eea1` on 2026-07-14. | `TF-007` | Immutable packs, in-memory index, bounded cache, random-read resolver | Pack round-trip/corruption tests and 758 MiB packed-read benchmark | Implemented | | `TF-008` | `internal/fsctl` benchmark and `internal/testfs` stress harness | `docs/validation-fs-preview.md` and synchronous FUSE-T read/write measurements | Partial: shared-core and measured warm macOS gates pass; cold/full-distribution metrics and other platforms remain open | | `TF-009` | Journal recovery, generation recovery, service keep-alive, restart-safe retirement | Recovery tests, daemon restart canaries, managed Deep Idle sleep/wake, and actual retained-source host reboot | Partial: no actual power loss during an in-flight transaction | -| `TF-010` | Shadow compare, optimistic routes, retained snapshots, current-byte fallback | 90,000 real random-range comparisons, rollback and failure-containment canaries | Implemented for isolated canaries | +| `TF-010` | Shadow compare, optimistic routes, retained snapshots, current-byte fallback | 90,000 real random-range comparisons, rollback and failure-containment canaries, and one bounded retained-source user-home canary | Implemented for macOS canaries; retention remains open | | `TF-011` | Codex state discovery primitives exist in `internal/codex` | Discovery unit tests | Missing: no stability policy, batch planner, or automatic enrollment loop | | `TF-012` | Shared Go core and macOS FUSE-T adapter | macOS real adapter tests; Linux and Windows non-CGO compile checks | Partial: Linux and Windows real adapters are missing | | `TF-013` | Canonical capability type in `internal/fsctl/status.go` | Status rejection tests and CLI status tests | Implemented; current status is `fs-engine-preview` | @@ -72,7 +72,7 @@ Baseline reviewed: commit `045eea1` on 2026-07-14. | Task 8: standalone CLI and automatic enrollment | Partial | Commit `3352b87`; command surface and guarded lifecycle exist | Task 8 Step 5, bounded automatic enrollment, is missing | | Task 9: service lifecycle and update guard | Complete | Commit `4589ffa`; launchd and preflight tests pass | Stronger automatic update claims remain release-gated | | Task 10: synthetic, crash, performance, and compile gates | Complete for the shared engine | Commit `a1ac76e`; preview validation report | It cannot satisfy real-adapter or retention gates | -| Task 11: real macOS trace, adapter, shadow, and canary | Partial | Sanitized real CLI/Desktop/FUSE-T evidence, managed sleep/wake, retained-source host reboot, current-client contracts, canonical user-home activation, and a synchronous isolated real CLI canary are public | Dedicated user-home canary retention, actual in-flight power loss, and seven-day retention remain | +| Task 11: real macOS trace, adapter, shadow, and canary | Partial | Sanitized real CLI/Desktop/FUSE-T evidence, managed sleep/wake, retained-source host reboot, current-client contracts, canonical user-home activation, and synchronous isolated plus bounded user-home real CLI canaries are public | Dedicated user-home canary retention, actual in-flight power loss, and seven-day retention remain | ## Missing Product Behavior And Exact Next Work diff --git a/docs/validation-macos-canary.md b/docs/validation-macos-canary.md index d7e2ee2..c0510a7 100644 --- a/docs/validation-macos-canary.md +++ b/docs/validation-macos-canary.md @@ -2,7 +2,7 @@ ## Current Status -The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have passed for read, append, resume, fork, child-session enrollment, canonical archive/unarchive moves, launchd restart, rollback, namespace deactivation, and unknown-version quarantine. A retained-source CLI canary survived an actual host reboot while managed, then resumed through the recovered mount and rolled back to an exact native JSONL. The currently installed CLI and Desktop versions also passed exact compatibility and isolated retained-source canaries. Process-level interruption recovery now covers append, compaction, migration, and rollback, and a managed session passed an actual Deep Idle sleep/wake cycle followed by a real model turn. The user Codex home now uses the canonical namespace with ordinary sessions remaining native passthrough and zero managed sessions at activation. The project remains at `fs-engine-preview` because its dedicated retained-source user-home canary has not completed retention, in-flight transaction evidence does not claim an actual power-loss test, and the seven-day incident-free gate has not started. +The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have passed for read, append, resume, fork, child-session enrollment, canonical archive/unarchive moves, launchd restart, rollback, namespace deactivation, and unknown-version quarantine. A retained-source CLI canary survived an actual host reboot while managed, then resumed through the recovered mount and rolled back to an exact native JSONL. The currently installed CLI and Desktop versions also passed exact compatibility and isolated retained-source canaries. Process-level interruption recovery now covers append, compaction, migration, and rollback, and a managed session passed an actual Deep Idle sleep/wake cycle followed by a real model turn. The user Codex home now uses the canonical namespace with ordinary sessions remaining native passthrough and one explicitly selected retained-source canary managed for observation. The project remains at `fs-engine-preview` because that canary has not completed retention, in-flight transaction evidence does not claim an actual power-loss test, and the seven-day incident-free gate has not completed. Additional synchronous-write and canonical-activation evidence on 2026-07-16: @@ -13,6 +13,7 @@ Additional synchronous-write and canonical-activation evidence on 2026-07-16: - Updating only the mounted localhost NFS volume with `mount -u -o sync` made the previously deterministic stale-offset regression pass. The Darwin adapter now withholds its health identity until that update succeeds and `MNT_SYNCHRONOUS` is visible through `statfs`; a failure unmounts the host instead of advertising readiness. No global NFS configuration, patched FUSE-T binary, privileged helper, or system-wide mount change is used. - A 64 MiB mounted read measured 7,045 MiB/s versus 7,435 MiB/s from the native APFS file, or 95% of native throughput. Across 200 JSONL append-plus-`fsync` operations, the synchronous mount averaged 3.99 ms with a 5.03 ms p95, versus 3.84 ms and 5.26 ms natively. - A fresh isolated real CLI canary used the official unarchive flow and then resumed through the synchronous canonical mount. The complete view grew from 97,388 to 120,859 bytes and 33 valid JSONL records. The complete original prefix retained SHA-256 `4cd4bcc1807d875b70e04b3028441f330f9c7ee0cd41cbcff08c18c9ec44d416`, the 23,471-byte delta parsed independently, the expected historical and new markers were recalled, generation remained 1, and no writable backing appeared. +- The same dedicated canary was then enrolled in the canonical user home while every ordinary rollout remained native passthrough. A real current CLI unarchive and resume produced a 120,864-byte, 33-record valid JSONL with the same exact 97,388-byte prefix SHA-256, a separately valid 23,476-byte delta, generation 1, and no writable backing. The model recalled the historical marker and emitted the new acceptance marker. Official archive moved the managed route back to `archived_sessions`; exactly one managed session remains, the mounted volume reports synchronous I/O, and the complete filesystem doctor is healthy. - Default, FUSE-tagged, race, vet, shell, cross-platform compile, and complete real FUSE-T suites passed after the fix. The real FUSE-T suite explicitly requires synchronous mount readiness before exercising the stale-offset regression. Additional current-client, interruption, and sleep/wake evidence on 2026-07-15: @@ -164,7 +165,7 @@ A direct `SIGTERM` stopped the foreground service and removed the mount cleanly. The following gates are still open: -- Retained-source canary routes in the real Codex home. +- Completion of the dedicated retained-source user-home canary retention window. - An actual power-loss or host-restart interruption while a transaction is in flight; simultaneous process termination and a separate idle managed-session host reboot have passed, but they are recorded as distinct evidence. - Seven incident-free days after reaching `platform-canary`. From 055d2db4f7bb183b8e717bb52021698c0996d349 Mon Sep 17 00:00:00 2001 From: King Star Date: Sat, 18 Jul 2026 19:16:31 +0800 Subject: [PATCH 31/33] feat: add native transparent session filesystem --- .gitignore | 2 + README.md | 31 + cmd/codexfold/main.go | 10 +- ...arent-session-filesystem-implementation.md | 75 +- ...1-transparent-session-filesystem-design.md | 4 +- ...ion-filesystem-implementation-alignment.md | 61 +- docs/validation-linux-fuse3.md | 51 + docs/validation-macos-canary.md | 34 +- internal/archive/archive.go | 638 ++++++++++++ internal/archive/archive_test.go | 338 +++++++ internal/archive/sync_unix.go | 14 + internal/archive/sync_windows.go | 5 + internal/buildid/buildid.go | 40 + internal/cli/archive.go | 121 +++ internal/cli/archive_test.go | 213 ++++ internal/cli/content_boundary_test.go | 45 + internal/cli/fold.go | 9 +- internal/cli/fork_family.go | 106 ++ internal/cli/fs.go | 394 +++++++- internal/cli/fs_activation.go | 43 + internal/cli/fs_activation_test.go | 73 ++ internal/cli/fs_enroll.go | 309 ++++++ internal/cli/fs_enroll_writer.go | 62 ++ internal/cli/fs_enroll_writer_other.go | 14 + internal/cli/fs_enroll_writer_test.go | 92 ++ internal/cli/fs_enroll_writer_unix.go | 31 + internal/cli/fs_enroll_writer_windows.go | 46 + internal/cli/fs_namespace.go | 5 + internal/cli/fs_reconcile.go | 10 +- internal/cli/fs_service.go | 905 +++++++++++++++-- internal/cli/fs_service_fskit.go | 10 + internal/cli/fs_service_fskit_darwin.go | 388 +++++++ internal/cli/fs_service_fskit_other.go | 12 + .../cli/fs_service_linux_integration_test.go | 170 ++++ internal/cli/fs_service_runtime_other.go | 7 + internal/cli/fs_service_runtime_windows.go | 147 +++ internal/cli/fs_service_transaction_test.go | 174 ++++ internal/cli/fs_supervisor.go | 74 ++ internal/cli/fs_supervisor_test.go | 32 + internal/cli/fs_test.go | 407 +++++++- internal/cli/root.go | 2 + internal/cli/root_test.go | 71 +- internal/codex/edges.go | 55 + internal/codex/edges_test.go | 54 + internal/enroll/apply.go | 55 + internal/enroll/observations.go | 88 ++ internal/enroll/observations_sync_unix.go | 14 + internal/enroll/observations_sync_windows.go | 5 + internal/enroll/observations_test.go | 35 + internal/enroll/planner.go | 243 +++++ internal/enroll/planner_test.go | 243 +++++ internal/family/family.go | 507 ++++++++++ internal/family/family_test.go | 175 ++++ internal/fold/doctor.go | 35 +- internal/fold/doctor_gc_test.go | 71 +- internal/fold/fold.go | 92 +- internal/fold/fold_test.go | 112 ++- internal/fold/gc.go | 46 +- internal/fold/unfold.go | 49 +- internal/fsctl/doctor.go | 16 +- internal/fsctl/status.go | 9 +- internal/fskitproto/client.go | 149 +++ internal/fskitproto/codec.go | 268 +++++ internal/fskitproto/protocol.go | 250 +++++ internal/fskitproto/protocol_test.go | 93 ++ internal/launcher/parent.go | 50 + internal/launcher/parent_test.go | 46 + internal/mountfs/dependency_boundary_test.go | 54 + internal/mountfs/file_metadata_darwin.go | 19 + internal/mountfs/file_metadata_linux.go | 19 + internal/mountfs/file_metadata_other.go | 12 + internal/mountfs/filesystem.go | 608 ++++++++++- internal/mountfs/filesystem_test.go | 264 ++++- .../mountfs/fuse_integration_linux_test.go | 405 ++++++++ internal/mountfs/fuse_integration_test.go | 256 ++++- internal/mountfs/fuse_provider_darwin.go | 45 + internal/mountfs/fuse_provider_darwin_test.go | 41 + internal/mountfs/fuse_provider_other.go | 5 + internal/mountfs/host.go | 10 + internal/mountfs/host_cgofuse.go | 155 +-- internal/mountfs/host_platform_posix.go | 56 ++ internal/mountfs/host_platform_windows.go | 53 + internal/mountfs/host_safety_test.go | 14 + internal/mountfs/host_statfs_darwin.go | 30 + internal/mountfs/host_statfs_linux.go | 26 + internal/mountfs/host_stub.go | 2 +- internal/mountfs/mount_backing_linux.go | 60 ++ internal/mountfs/mount_backing_other.go | 12 + internal/mountfs/mount_linux.go | 89 ++ internal/mountfs/mount_policy_linux.go | 24 + ...olicy_other.go => mount_policy_windows.go} | 2 +- internal/mountfs/mount_stale_other.go | 5 + internal/mountfs/native_append.go | 598 +++++++++++ .../native_append_real_integration_test.go | 368 +++++++ internal/mountfs/native_append_test.go | 409 ++++++++ .../native_fskit_metadata_darwin_test.go | 139 +++ .../mountfs/native_fskit_mount_darwin_test.go | 796 +++++++++++++++ internal/mountfs/native_fskit_server.go | 948 ++++++++++++++++++ internal/mountfs/native_fskit_server_test.go | 419 ++++++++ internal/mountfs/native_fskit_stat_darwin.go | 34 + internal/mountfs/native_fskit_stat_other.go | 19 + .../mountfs/native_namespace_watch_darwin.go | 102 ++ .../native_namespace_watch_darwin_test.go | 55 + .../mountfs/native_namespace_watch_other.go | 10 + internal/mountfs/native_preflight.go | 81 ++ internal/mountfs/native_preflight_audit.go | 72 ++ internal/mountfs/native_preflight_cache.go | 276 +++++ internal/mountfs/native_preflight_test.go | 195 ++++ .../testdata/codex-real-resume-write.trace | 27 + internal/mountfs/xattr_darwin.go | 67 ++ internal/mountfs/xattr_linux.go | 67 ++ internal/mountfs/xattr_other.go | 15 + internal/mountid/identity.go | 50 +- internal/mountid/identity_test.go | 40 + internal/pack/build.go | 54 +- internal/pack/pack_test.go | 81 ++ internal/pack/resolver.go | 18 +- internal/prune/remove_contained_test.go | 5 +- internal/reconcile/budget.go | 18 + internal/reconcile/reconcile.go | 59 +- internal/reconcile/reconcile_test.go | 35 + internal/reconcile/repair.go | 221 +++- internal/reconcile/repair_test.go | 99 ++ internal/reconcile/semantic.go | 63 ++ internal/reconcile/semantic_test.go | 71 ++ internal/service/binary_update.go | 168 ++++ internal/service/binary_update_test.go | 77 ++ internal/service/binary_update_unix.go | 22 + internal/service/binary_update_windows.go | 21 + internal/service/build_status.go | 351 +++++++ internal/service/build_status_test.go | 154 +++ internal/service/definition_update.go | 150 +++ internal/service/definition_update_test.go | 88 ++ internal/service/fskit_app.go | 44 + internal/service/fskit_app_test.go | 33 + internal/service/mount_probe_darwin.go | 12 +- internal/service/mount_probe_darwin_test.go | 24 + internal/service/mount_probe_linux.go | 54 + internal/service/mount_probe_other.go | 2 +- internal/service/mount_probe_windows.go | 18 + .../service/native_fskit_operations_darwin.go | 100 ++ .../service/native_fskit_operations_other.go | 11 + internal/service/native_fskit_supervisor.go | 160 +++ .../service/native_fskit_supervisor_test.go | 165 +++ internal/service/platform.go | 40 + internal/service/process_lock.go | 41 + internal/service/process_parent_darwin.go | 24 + .../service/process_parent_darwin_test.go | 18 + internal/service/process_parent_other.go | 9 + internal/service/service.go | 207 +++- internal/service/service_test.go | 209 +++- internal/service/systemd.go | 183 ++++ internal/service/systemd_linux_test.go | 102 ++ internal/service/wait.go | 31 + internal/service/windows.go | 197 ++++ internal/storage/accounting.go | 32 + internal/storage/budget.go | 103 ++ internal/storage/budget_test.go | 105 ++ internal/storage/gc.go | 699 +++++++++++++ internal/storage/gc_test.go | 211 ++++ internal/storage/guard.go | 99 ++ internal/storage/guard_test.go | 45 + internal/storage/inventory.go | 599 +++++++++++ internal/storage/inventory_test.go | 222 ++++ internal/storage/lease.go | 157 +++ internal/storage/lease_other.go | 16 + internal/storage/lease_test.go | 65 ++ internal/storage/lease_unix.go | 21 + internal/storage/lease_windows.go | 23 + internal/storage/mountpoints_darwin.go | 29 + internal/storage/mountpoints_linux.go | 36 + internal/storage/mountpoints_other.go | 7 + internal/storage/physical_other.go | 16 + internal/storage/physical_unix.go | 24 + internal/storage/physical_windows.go | 45 + internal/storage/policy.go | 70 ++ internal/storage/policy_test.go | 52 + internal/storage/space.go | 23 + internal/storage/space_other.go | 9 + internal/storage/space_unix.go | 29 + internal/storage/space_windows.go | 24 + internal/testfs/corpus_test.go | 3 +- internal/testfs/large_test.go | 3 +- internal/vfs/handles.go | 3 +- internal/vfs/recovery_test.go | 99 ++ internal/vfs/session.go | 89 +- internal/vfs/session_test.go | 113 +++ .../darwin/fskit/CodexFoldFSKit.entitlements | 10 + .../CodexFoldFSKit.xcodeproj/project.pbxproj | 419 ++++++++ .../contents.xcworkspacedata | 7 + .../CodexFoldFSKitModule.entitlements | 16 + platform/darwin/fskit/Extension/Info.plist | 54 + .../fskit/Extension/ProfileModule.swift | 940 +++++++++++++++++ platform/darwin/fskit/Extension/Wire.swift | 622 ++++++++++++ platform/darwin/fskit/Host/Host.swift | 111 ++ platform/darwin/fskit/Host/Info.plist | 26 + platform/darwin/fskit/project.yml | 39 + 197 files changed, 23370 insertions(+), 472 deletions(-) create mode 100644 docs/validation-linux-fuse3.md create mode 100644 internal/archive/archive.go create mode 100644 internal/archive/archive_test.go create mode 100644 internal/archive/sync_unix.go create mode 100644 internal/archive/sync_windows.go create mode 100644 internal/buildid/buildid.go create mode 100644 internal/cli/archive.go create mode 100644 internal/cli/archive_test.go create mode 100644 internal/cli/content_boundary_test.go create mode 100644 internal/cli/fork_family.go create mode 100644 internal/cli/fs_activation.go create mode 100644 internal/cli/fs_activation_test.go create mode 100644 internal/cli/fs_enroll.go create mode 100644 internal/cli/fs_enroll_writer.go create mode 100644 internal/cli/fs_enroll_writer_other.go create mode 100644 internal/cli/fs_enroll_writer_test.go create mode 100644 internal/cli/fs_enroll_writer_unix.go create mode 100644 internal/cli/fs_enroll_writer_windows.go create mode 100644 internal/cli/fs_service_fskit.go create mode 100644 internal/cli/fs_service_fskit_darwin.go create mode 100644 internal/cli/fs_service_fskit_other.go create mode 100644 internal/cli/fs_service_linux_integration_test.go create mode 100644 internal/cli/fs_service_runtime_other.go create mode 100644 internal/cli/fs_service_runtime_windows.go create mode 100644 internal/cli/fs_service_transaction_test.go create mode 100644 internal/cli/fs_supervisor.go create mode 100644 internal/cli/fs_supervisor_test.go create mode 100644 internal/codex/edges.go create mode 100644 internal/codex/edges_test.go create mode 100644 internal/enroll/apply.go create mode 100644 internal/enroll/observations.go create mode 100644 internal/enroll/observations_sync_unix.go create mode 100644 internal/enroll/observations_sync_windows.go create mode 100644 internal/enroll/observations_test.go create mode 100644 internal/enroll/planner.go create mode 100644 internal/enroll/planner_test.go create mode 100644 internal/family/family.go create mode 100644 internal/family/family_test.go create mode 100644 internal/fskitproto/client.go create mode 100644 internal/fskitproto/codec.go create mode 100644 internal/fskitproto/protocol.go create mode 100644 internal/fskitproto/protocol_test.go create mode 100644 internal/launcher/parent.go create mode 100644 internal/launcher/parent_test.go create mode 100644 internal/mountfs/dependency_boundary_test.go create mode 100644 internal/mountfs/file_metadata_darwin.go create mode 100644 internal/mountfs/file_metadata_linux.go create mode 100644 internal/mountfs/file_metadata_other.go create mode 100644 internal/mountfs/fuse_integration_linux_test.go create mode 100644 internal/mountfs/fuse_provider_darwin.go create mode 100644 internal/mountfs/fuse_provider_darwin_test.go create mode 100644 internal/mountfs/fuse_provider_other.go create mode 100644 internal/mountfs/host_platform_posix.go create mode 100644 internal/mountfs/host_platform_windows.go create mode 100644 internal/mountfs/host_statfs_darwin.go create mode 100644 internal/mountfs/host_statfs_linux.go create mode 100644 internal/mountfs/mount_backing_linux.go create mode 100644 internal/mountfs/mount_backing_other.go create mode 100644 internal/mountfs/mount_linux.go create mode 100644 internal/mountfs/mount_policy_linux.go rename internal/mountfs/{mount_policy_other.go => mount_policy_windows.go} (77%) create mode 100644 internal/mountfs/mount_stale_other.go create mode 100644 internal/mountfs/native_append.go create mode 100644 internal/mountfs/native_append_real_integration_test.go create mode 100644 internal/mountfs/native_append_test.go create mode 100644 internal/mountfs/native_fskit_metadata_darwin_test.go create mode 100644 internal/mountfs/native_fskit_mount_darwin_test.go create mode 100644 internal/mountfs/native_fskit_server.go create mode 100644 internal/mountfs/native_fskit_server_test.go create mode 100644 internal/mountfs/native_fskit_stat_darwin.go create mode 100644 internal/mountfs/native_fskit_stat_other.go create mode 100644 internal/mountfs/native_namespace_watch_darwin.go create mode 100644 internal/mountfs/native_namespace_watch_darwin_test.go create mode 100644 internal/mountfs/native_namespace_watch_other.go create mode 100644 internal/mountfs/native_preflight.go create mode 100644 internal/mountfs/native_preflight_audit.go create mode 100644 internal/mountfs/native_preflight_cache.go create mode 100644 internal/mountfs/native_preflight_test.go create mode 100644 internal/mountfs/testdata/codex-real-resume-write.trace create mode 100644 internal/mountfs/xattr_darwin.go create mode 100644 internal/mountfs/xattr_linux.go create mode 100644 internal/mountfs/xattr_other.go create mode 100644 internal/mountid/identity_test.go create mode 100644 internal/reconcile/budget.go create mode 100644 internal/reconcile/semantic.go create mode 100644 internal/reconcile/semantic_test.go create mode 100644 internal/service/binary_update.go create mode 100644 internal/service/binary_update_test.go create mode 100644 internal/service/binary_update_unix.go create mode 100644 internal/service/binary_update_windows.go create mode 100644 internal/service/build_status.go create mode 100644 internal/service/build_status_test.go create mode 100644 internal/service/definition_update.go create mode 100644 internal/service/definition_update_test.go create mode 100644 internal/service/fskit_app.go create mode 100644 internal/service/fskit_app_test.go create mode 100644 internal/service/mount_probe_darwin_test.go create mode 100644 internal/service/mount_probe_linux.go create mode 100644 internal/service/mount_probe_windows.go create mode 100644 internal/service/native_fskit_operations_darwin.go create mode 100644 internal/service/native_fskit_operations_other.go create mode 100644 internal/service/native_fskit_supervisor.go create mode 100644 internal/service/native_fskit_supervisor_test.go create mode 100644 internal/service/platform.go create mode 100644 internal/service/process_parent_darwin.go create mode 100644 internal/service/process_parent_darwin_test.go create mode 100644 internal/service/process_parent_other.go create mode 100644 internal/service/systemd.go create mode 100644 internal/service/systemd_linux_test.go create mode 100644 internal/service/wait.go create mode 100644 internal/service/windows.go create mode 100644 internal/storage/accounting.go create mode 100644 internal/storage/budget.go create mode 100644 internal/storage/budget_test.go create mode 100644 internal/storage/gc.go create mode 100644 internal/storage/gc_test.go create mode 100644 internal/storage/guard.go create mode 100644 internal/storage/guard_test.go create mode 100644 internal/storage/inventory.go create mode 100644 internal/storage/inventory_test.go create mode 100644 internal/storage/lease.go create mode 100644 internal/storage/lease_other.go create mode 100644 internal/storage/lease_test.go create mode 100644 internal/storage/lease_unix.go create mode 100644 internal/storage/lease_windows.go create mode 100644 internal/storage/mountpoints_darwin.go create mode 100644 internal/storage/mountpoints_linux.go create mode 100644 internal/storage/mountpoints_other.go create mode 100644 internal/storage/physical_other.go create mode 100644 internal/storage/physical_unix.go create mode 100644 internal/storage/physical_windows.go create mode 100644 internal/storage/policy.go create mode 100644 internal/storage/policy_test.go create mode 100644 internal/storage/space.go create mode 100644 internal/storage/space_other.go create mode 100644 internal/storage/space_unix.go create mode 100644 internal/storage/space_windows.go create mode 100644 platform/darwin/fskit/CodexFoldFSKit.entitlements create mode 100644 platform/darwin/fskit/CodexFoldFSKit.xcodeproj/project.pbxproj create mode 100644 platform/darwin/fskit/CodexFoldFSKit.xcodeproj/project.xcworkspace/contents.xcworkspacedata create mode 100644 platform/darwin/fskit/Extension/CodexFoldFSKitModule.entitlements create mode 100644 platform/darwin/fskit/Extension/Info.plist create mode 100644 platform/darwin/fskit/Extension/ProfileModule.swift create mode 100644 platform/darwin/fskit/Extension/Wire.swift create mode 100644 platform/darwin/fskit/Host/Host.swift create mode 100644 platform/darwin/fskit/Host/Info.plist create mode 100644 platform/darwin/fskit/project.yml diff --git a/.gitignore b/.gitignore index 9235d5a..6059122 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ *.test .DS_Store .worktrees/ +xcuserdata/ +*.xcuserstate diff --git a/README.md b/README.md index 7fa8ba4..d33f8c4 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,15 @@ It finds exact duplicate raw JSON string tokens, complete JSONL records, and con The requirements and release gates for normal JSONL paths backed transparently by shared storage are defined in [the transparent filesystem product contract](docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md). No release may claim `随点随开`, transparent session access, or production-ready virtual sessions before the platform-specific gates in that contract pass. +The unreleased transparent-filesystem branch remains `fs-engine-preview`: + +- macOS uses the explicitly selected synchronous FUSE-T NFS backend and has real Codex CLI/Desktop canary evidence. FUSE-T's FSKit backend was tested and rejected after a deterministic same-offset JSONL byte-loss failure. +- Linux FUSE3 has real unprivileged read, append, copy-on-write, truncate, archive rename, crash recovery, remount, performance, and `systemd --user` lifecycle evidence. +- Windows has a WinFsp adapter and native Windows Service host that cross-compile, but no real Windows/WinFsp host has validated them yet. +- Retention, actual in-flight power loss, and the remaining platform-specific client and upgrade gates still block promotion. + +See [the Linux FUSE3 validation](docs/validation-linux-fuse3.md) and [the macOS canary validation](docs/validation-macos-canary.md) for the evidence boundary. The default build remains storage-only; platform mounts require explicit build tags and installed host prerequisites. + ## Install ```bash @@ -81,6 +90,27 @@ codexfold remove-contained --apply The first command is proof-only. `--apply` additionally requires an existing verified fold, a current source SHA-256 match, and a successful temporary unfold. It then isolates the source file, removes the archived thread and associated local state in one SQLite transaction, cleans exact thread-ID references from Codex global state, and finally deletes the isolated source. A tombstone and fold manifest remain for byte-level recovery. Concurrent global-state changes abort the operation instead of being overwritten. +## Fork Families And Archival + +Inspect the explicit Codex spawn graph and compare two selected rollouts without mutation: + +```bash +codexfold fork-family show +codexfold fork-family compare +``` + +The report keeps graph ancestry separate from exact content evidence. It can identify identical applicable records, complete containment, shared prefixes with independent tails, other exact shared records, or an unknown relationship. It never labels a branch useless from ancestry, age, title, or size. + +Preview and explicitly archive one active session: + +```bash +codexfold archive +codexfold archive --apply +codexfold archive recover --apply +``` + +Archive is dry-run-first and preserves the rollout bytes. Apply requires the native writer probe, revalidates the selected SQLite route and complete source SHA-256, moves the rollout to Codex's flat `archived_sessions` path, and updates the official archive fields in one guarded transaction. A durable journal supports deterministic recovery if file and database commit acknowledgement are interrupted. Archive never deletes a session; exact-contained deletion remains the separate archived-only `remove-contained` operation. + ## Maintenance Verify every manifest and referenced object: @@ -107,6 +137,7 @@ codexfold gc --apply - Restore writes to a temporary file, verifies the complete SHA-256, then atomically replaces the target. - Existing indexes, manifests, and restore targets are never replaced without an explicit overwrite flag. - Contained-session removal is archived-only, proof-first, transaction-guarded, and retains recovery evidence. +- Fork-family reporting is evidence-only, archive is explicit and recoverable, and neither operation triggers deletion. ## Development diff --git a/cmd/codexfold/main.go b/cmd/codexfold/main.go index 9115fb3..9845a7a 100644 --- a/cmd/codexfold/main.go +++ b/cmd/codexfold/main.go @@ -5,13 +5,21 @@ import ( "fmt" "os" "os/signal" + "syscall" "github.com/jstar0/codexfold/internal/cli" + "github.com/jstar0/codexfold/internal/launcher" ) func main() { - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() + ctx, stopLauncher, err := launcher.MonitorContext(ctx) + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } + defer stopLauncher() command := cli.NewRootCommand() command.SetContext(ctx) if err := command.Execute(); err != nil { diff --git a/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md b/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md index 28d761f..c3c4f05 100644 --- a/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md +++ b/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md @@ -6,11 +6,11 @@ **Architecture:** Extend Fold V1 with a block-addressable packed resolver, then place a platform-neutral exact-byte session engine above it. The engine composes an immutable manifest base with an append delta or verified writable backing; platform adapters only translate native file operations. Migration, compatibility quarantine, fallback, and promotion remain explicit journaled transactions, with real Codex routing disabled until shadow and platform gates pass. -**Tech Stack:** Go 1.26, zstd, Cobra, modernc SQLite, `cgofuse` v1.6.0 behind platform/build tags, and FUSE-T 1.2.7 as the validated macOS host. +**Tech Stack:** Go 1.26, zstd, Cobra, modernc SQLite, `cgofuse` v1.6.0 behind platform/build tags, FUSE-T 1.2.7 on macOS, FUSE3 on Linux, and WinFsp plus Windows SCM on Windows. ## Alignment Snapshot -Current public status remains `fs-engine-preview`. Tasks 1 through 7 and 9 through 10 are implemented. Task 8 lacks bounded automatic enrollment. Task 11 has substantial isolated macOS evidence, including one idle retained-source managed CLI session surviving an actual host reboot, but has not passed managed-session sleep/wake, transaction-interruption restart cases, current Desktop compatibility, real-home retained-source canaries, or seven-day retention. +Current public status remains `fs-engine-preview`. Tasks 1 through 10 and 12 through 14 are implemented. Task 11 has substantial isolated and bounded real-home macOS evidence, including current-client compatibility, sleep/wake, process-interruption recovery, and one idle retained-source managed CLI session surviving an actual host reboot. Linux FUSE3 now has real unprivileged operation, crash/restart, performance, mount-policy, and `systemd --user` lifecycle evidence. Windows WinFsp and SCM support are implemented and cross-compile, but lack a real Windows host. Actual in-flight power loss, the dedicated retention window, seven incident-free days, Linux client/upgrade/rollback/retention gates, and all real Windows gates remain open. | Task | Status | Current evidence | Remaining work | | --- | --- | --- | --- | @@ -20,11 +20,15 @@ Current public status remains `fs-engine-preview`. Tasks 1 through 7 and 9 throu | 4 | Complete | Commit `076d772`; journal, compaction, and fallback tests | None in this task | | 5 | Complete | Commit `35a53fc`; status, shadow, doctor, and benchmark tests | Platform evidence remains outside this task | | 6 | Complete | Commit `5be10d2`; exact-version compatibility and optimistic route tests | New installed client versions still require fresh contracts | -| 7 | Complete | Commit `3f51aa5`; neutral operations and real FUSE-T adapter tests | Linux and Windows real adapters remain separate product gates | -| 8 | Partial | Commit `3352b87`; standalone CLI and guarded lifecycle commands | Implement bounded automatic discovery and enrollment | -| 9 | Complete | Commit `4589ffa`; launchd lifecycle and update preflight tests | Production update promotion remains gated by platform readiness | +| 7 | Complete | Commit `3f51aa5`; neutral operations, real macOS FUSE-T, and real Linux FUSE3 adapter tests | Windows real-adapter evidence remains a separate product gate | +| 8 | Complete | Standalone CLI, guarded lifecycle, bounded planner/apply loop, and isolated automatic-enrollment evidence | Production enablement remains gated by platform readiness | +| 9 | Complete | Commit `4589ffa`; launchd, real `systemd --user`, Windows SCM compile, and update preflight tests | Windows service runtime and production update promotion remain platform-gated | | 10 | Complete | Commit `a1ac76e`; synthetic, crash, race, cross-compile, and 758 MiB evidence | This task proves only the shared engine preview | | 11 | Partial | Real macOS CLI/Desktop, FUSE-T, rollback, daemon restart, idle managed-session host reboot, and quarantine evidence | Complete the remaining disruptive and retention gates | +| 12 | Complete | Bounded planner/apply/service tests plus isolated canonical automatic enrollment, native-writer probing, restart, append, quarantine, and failed-cutover evidence | Real-home automatic apply remains promotion-gated | +| 13 | Complete | Fork graph reports, exact content comparison, guarded official-compatible archive transactions, recovery, static content-change boundaries, and isolated native plus managed FUSE-T round trips | None in this task | +| 14 | Complete | Physical inventory, hard mutation budgets, lease-aware bounded GC, and truthful projected/actual accounting | Destructive retention remains promotion-gated | +| 15 | Partial | Current macOS client contracts plus restart gates; real Linux FUSE3 operation, crash, performance, and systemd lifecycle; Windows WinFsp/SCM cross-compile | Actual in-flight power loss, retention windows, Linux client/upgrade/rollback gates, and all real Windows gates remain | ## Global Constraints @@ -584,10 +588,12 @@ git commit -m "feat: expose transparent filesystem command surface" **Files:** - Create: `internal/service/service.go` -- Create: `internal/service/launchd_darwin.go` -- Create: `internal/service/service_other.go` +- Create: `internal/service/systemd.go` +- Create: `internal/service/windows.go` +- Create: `internal/service/mount_probe_*.go` - Create: `internal/service/service_test.go` -- Modify: `internal/cli/fs.go` +- Modify: `internal/cli/fs_service.go` +- Create: `internal/cli/fs_service_runtime_windows.go` **Interfaces:** - Consumes: built tagged binary, mount path, installed-client compatibility result, and doctor status. @@ -595,7 +601,7 @@ git commit -m "feat: expose transparent filesystem command surface" - [x] **Step 1: Write failing service-render and update-guard tests** -Assert launchd arguments are absolute, logs contain no session content, daemon and mount health are separate, preview auto-update is rejected, and a client version change enters quarantine before restart. +Assert launchd, systemd, and Windows SCM definitions use the same absolute `fs serve` arguments, logs contain no session content, daemon and mount health are separate, preview auto-update is rejected, and a client version change enters quarantine before restart. - [x] **Step 2: Run focused tests and verify missing service API** @@ -605,7 +611,7 @@ Expected: FAIL because service APIs are absent. - [x] **Step 3: Implement service lifecycle without self-elevation** -Render a per-user launchd plist and use `launchctl bootstrap/bootout/kickstart` only after an explicit apply command. Detect prerequisites but never install FUSE-T or request elevation from library code. +Render and manage launchd on macOS, `systemd --user` on Linux, and the native SCM host on Windows only after an explicit apply command. Detect FUSE-T, FUSE3, or WinFsp prerequisites but never install them, enable Linux linger, or request elevation from library code. - [x] **Step 4: Implement update compatibility guard** @@ -730,63 +736,66 @@ No private path, session ID, trace content, credential, or control-plane name ma ### Task 12: Bounded Automatic Discovery And Enrollment -**Status:** Missing. This completes Task 8 Step 5 and is required before production operation can be called automatic. +**Status:** Complete in the current implementation. Production enablement remains gated by platform readiness and retention. **Requirements:** `TF-001`, `TF-010`, `TF-011`, `TF-014`, `TF-015`, `TF-021`. **Exact next work:** -- [ ] Add policy tests for existing sessions, newly created sessions, forks, active writers, changing files, archived eligibility, unknown client versions, failed doctor state, insufficient disk budget, bounded batches, restart idempotency, and failed cutover. -- [ ] Implement a read-only enrollment planner that consumes Codex state, rollout stability evidence, compatibility, doctor, writer state, promotion stage, and storage-budget preflight, and emits explicit eligible/ineligible reasons without changing routes. -- [ ] Implement bounded apply transactions that fold, pack, shadow, stage at most one retained native snapshot, wait for exact mount acknowledgement, and only then update routing. A failure leaves the original native route and source unchanged. -- [ ] Integrate the planner into the standalone service with a bounded interval and batch size. Newly created sessions and forks remain native while active and need no per-session command when they later become eligible. -- [ ] Validate in an isolated Codex home across daemon restart and client-version quarantine before any real-home enrollment is allowed. +- [x] Add policy tests for existing sessions, newly created sessions, forks, active writers, changing files, archived eligibility, unknown client versions, failed doctor state, insufficient disk budget, bounded batches, restart idempotency, and failed cutover. +- [x] Implement a read-only enrollment planner that consumes Codex state, rollout stability evidence, compatibility, doctor, writer state, promotion stage, and storage-budget preflight, and emits explicit eligible/ineligible reasons without changing routes. +- [x] Implement bounded apply transactions that fold, pack, shadow, stage at most one retained native snapshot, wait for exact mount acknowledgement, and only then update routing. A failure leaves the original native route and source unchanged. +- [x] Integrate the planner into the standalone service with a bounded interval and batch size. Newly created sessions and forks remain native while active and need no per-session command when they later become eligible. +- [x] Validate in an isolated Codex home across daemon restart, real CLI append, client-version quarantine, and failed canonical cutover before any real-home automatic enrollment is allowed. ### Task 13: Conservative Branch Lifecycle And Content-Change Boundary -**Status:** Partial. Exact containment deletion and separate repair/reconciliation outputs exist; conservative family classification and guarded archive execution are missing. +**Status:** Complete. Conservative family classification, guarded archive execution, exact-contained deletion, and explicit content-changing repair/reconciliation boundaries are implemented and verified. **Requirements:** `TF-018`, `TF-019`, `TF-020`. **Exact next work:** -- [ ] Add read-only fork-family reports that distinguish shared exact content, independent tails, complete containment, active/archived state, and unknown relationships. Never label a branch useless from ancestry, age, title, or size alone. -- [ ] Trace and test the current official Codex archive operation, then add a dry-run-first archive mutation that requires an explicit session selection, revalidates database route and source digest, preserves the rollout, and updates file and state atomically. -- [ ] Keep `remove-contained` as a separate archived-only operation and add integration coverage proving that family classification or archive never triggers deletion automatically. -- [ ] Add CLI regression tests proving `repair-rollout` and `reconcile-rollout` require explicit separate outputs and cannot be called by fold, migrate, compact, enrollment, rollback, or GC paths. +- [x] Add read-only fork-family reports that distinguish shared exact content, independent tails, complete containment, active/archived state, and unknown relationships. Never label a branch useless from ancestry, age, title, or size alone. +- [x] Trace and test the current official Codex archive operation, then add a dry-run-first archive mutation that requires an explicit session selection, revalidates database route and source digest, preserves the rollout, and updates file and state atomically. +- [x] Keep `remove-contained` as a separate archived-only operation and add integration coverage proving that family classification or archive never triggers deletion automatically. +- [x] Add CLI and static boundary regression tests proving `repair-rollout` and `reconcile-rollout` remain the only content-changing reconciliation entrypoints and cannot be called by fold, migrate, compact, enrollment, rollback, GC, archive, or family paths. ### Task 14: Hard Storage Budgets, Retention, Cleanup, And Reclamation Accounting -**Status:** Missing. Current correctness paths create verified snapshots and retirement state, but no product-wide hard budget or bounded retirement cleanup policy exists. +**Status:** Complete in the current implementation. Destructive retention promotion remains disabled before platform readiness. **Requirements:** `TF-009`, `TF-014`, `TF-021`. **Exact next work:** -- [ ] Add a platform-neutral storage inventory that accounts separately for logical session bytes, unique loose objects, packs, native sources, retained snapshots, current fallbacks, writable backings, old generations, retirement state, journal-owned recovery files, and unowned temporary files. -- [ ] Add preflight APIs that calculate projected peak bytes and reject fold, pack, migrate, rollback, compact, enrollment, and content-changing output before writing when the hard temporary budget or free-space reserve would be exceeded. -- [ ] Enforce one immutable migration snapshot and one current writable fallback per managed session, one full-session scratch file per transaction, and current-plus-previous pack-generation retention until leases close. -- [ ] Add startup and explicit GC for abandoned temporary files, expired unleased generations, and retired state whose journal and retention proofs allow removal. Never remove the sole recoverable generation. -- [ ] Extend status, doctor, and mutation results with projected versus actual physical reclamation. Add low-space, interrupted-cleanup, retained-fallback, and repeated-enrollment tests that prove disk use remains bounded. +- [x] Add a platform-neutral storage inventory that accounts separately for logical session bytes, unique loose objects, packs, native sources, retained snapshots, current fallbacks, active deltas, writable backings, old generations, retirement state, journal-owned recovery files, unowned temporary files, and metadata. +- [x] Add preflight APIs that calculate projected peak bytes and reject fold, pack, migrate, rollback, compact, enrollment, copy-on-write, materialization, repair, and reconciliation before writing when the hard temporary budget or free-space reserve would be exceeded. +- [x] Enforce one immutable migration snapshot and one current writable fallback per managed session, one full-session scratch file per transaction, and current-plus-previous pack-generation retention until leases close. +- [x] Add startup and explicit GC for abandoned temporary files, expired unleased generations, and bounded retired state. Journal-owned recovery files, active leases, and the sole recoverable generation are retained. +- [x] Extend status, doctor, mutation, and GC results with physical inventory, projected peak, projected final, projected reclaimable, and actual reclaimed bytes. Low-space, interrupted-cleanup, retained-fallback, hard-link, lease, and repeated-GC tests pass. ### Task 15: Remaining Platform And Retention Gates -**Status:** Missing as release evidence; it does not block continued engine development but blocks stronger capability claims. +**Status:** Partial as release evidence; Linux adapter and service execution are now real, but the remaining platform and retention gates still block stronger capability claims. **Requirements:** `TF-003`, `TF-008`, `TF-009`, `TF-011`, `TF-012`, `TF-014`, `TF-015`, `TF-017`, `TF-021`. **Exact next work:** -- [ ] Import exact compatibility contracts for the currently installed Codex Desktop and CLI versions and return `fs doctor` to a clean client state. -- [ ] Run retained-source managed macOS canaries through sleep/wake and real host restart, including interrupted append, compaction, migration, and rollback cases required by the contract. -- [ ] Activate only bounded real-home canaries after Tasks 12 and 14 pass, then complete seven incident-free days before any `platform-canary` promotion decision. -- [ ] Implement and execute real Linux FUSE3 and Windows WinFsp adapters and their independent operation, crash, performance, upgrade, and rollback gates. +- [x] Import exact compatibility contracts for the currently installed Codex Desktop and CLI versions and return the isolated canary doctor to a clean client state. +- [x] Run retained-source managed macOS canaries through sleep/wake and real host restart, including process-interrupted append, compaction, migration, and rollback cases required by the contract. +- [x] Implement the Linux FUSE3 adapter with explicit `fuse fuse3` build tags and execute real unprivileged read, append, copy-on-write, truncate, canonical rename, native fallback, `SIGKILL` stale-mount recovery, remount, performance, backing-seal, and `systemd --user` install/start/status/stop gates. +- [x] Implement the Windows WinFsp adapter and native SCM service host, mount probe, configuration, start/stop/status, and restart policy; default and WinFsp binaries and tests cross-compile. +- [ ] Continue only bounded real-home canary observation and complete seven incident-free days before any `platform-canary` promotion decision; general automatic apply remains disabled. +- [ ] Execute Linux real-client compatibility, upgrade quarantine, rollback, and retention gates. +- [ ] Execute Windows WinFsp operation, crash/restart, performance, real-client compatibility, upgrade quarantine, rollback, and retention gates on a real Windows host. ## Plan Self-Review - `TF-001` through `TF-022` each map to implementation and verification tasks or an explicitly identified storage-engine baseline. - Real-client, real-adapter, restart, upgrade, and retention gates remain in Tasks 11 and 15 and cannot be satisfied by Task 10 fixtures. -- FUSE-T is the validated macOS host and remains an explicit authorization boundary; Linux and Windows adapters are certified independently. +- FUSE-T is the validated macOS host and FUSE3 is the native-gated Linux host; Windows remains implementation and cross-compile only until independent WinFsp execution passes. - The stale migration snapshot is never used as current fallback after virtual writes diverge. - The default build remains portable and does not require installed FUSE headers. - No task changes a real Codex route before shadow, compatibility, doctor, and explicit apply gates pass. diff --git a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md index 39239f5..e25a8da 100644 --- a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md +++ b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md @@ -270,11 +270,13 @@ The trace suite covers listing, opening, scrolling old history, resume, sending ### macOS -- Current reference adapter: FUSE-T `1.2.7`. +- Selected production adapter: FUSE-T `1.2.7` with the NFS backend explicitly requested as `backend=nfs`; relying on FUSE-T's default backend is not allowed. - Service: user launch service with keep-alive and mount health monitoring. - Required tests: APFS native baseline, Apple Silicon, Codex Desktop, Codex CLI, canonical `sessions` and `archived_sessions` namespace moves, sleep/wake, network changes, user logout/login, daemon kill, mount restart, and Codex upgrade. - FUSE-T is the validated userspace host for this project; macFUSE is not a prerequisite for the current macOS route. - The FUSE-T NFS mount must use synchronous write requests before its health identity becomes readable. This is verified through the live `MNT_SYNCHRONOUS` mount flag and a real same-offset JSONL write regression; libfuse `direct_io` or disabled attribute caching alone is not accepted as evidence. +- FUSE-T `1.2.7`'s FSKit backend is rejected for production. An isolated real mount lost the first of two complete JSONL records written at the same stale EOF, and managed-to-native route changes remained cached past the five-second correctness gate. Basic read/write, `F_FULLFSYNC`, truncate, remount, and throughput results do not override a byte-loss failure. FUSE-T also documents that notifications are unavailable for its FSKit backend. +- Native FSKit remains a research adapter rather than a fallback selected at runtime. The earlier probes did not provide a complete canonical namespace and did not recover automatically from every extension-process failure; no native FSKit route may replace the selected NFS backend without passing the complete platform contract independently. - Platform readiness requires a directory-level canonical namespace or an equivalent mechanism that keeps Codex archive and unarchive moves native-compatible. ### Linux diff --git a/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md b/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md index c6d7ec1..be96ee1 100644 --- a/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md +++ b/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md @@ -4,9 +4,9 @@ This document aligns the original product commitments, the canonical transparent-filesystem contract, the implementation plan, the current repository, and the available validation evidence. -It does not redesign CodexFold, authorize real-session enrollment, promote the capability above `fs-engine-preview`, or treat fixtures as production evidence. CodexFold remains a standalone public product. External installation or supervision is outside its runtime architecture. +It does not redesign CodexFold, authorize real-session enrollment, promote the capability above `fs-engine-preview`, or treat fixtures as production evidence. CodexFold remains a standalone public product. Native launchd, `systemd --user`, and Windows SCM supervision are part of the standalone runtime; private or external control-plane coupling remains outside it. -Baseline reviewed: commit `045eea1` on 2026-07-14. +Baseline refreshed against the current Task 12 through Task 15 implementation and validation evidence on 2026-07-16. ## Original Commitment To Requirement Mapping @@ -20,43 +20,43 @@ Baseline reviewed: commit `045eea1` on 2026-07-14. | Normal writes append to a durable delta; non-append mutations use safe copy-on-write | `TF-005`, `TF-006` | Preserved | | Runtime reads use packed storage rather than tens of thousands of loose-object opens | `TF-007`, `TF-008` | Preserved | | Correctness includes performance, bounded memory, crash recovery, restart recovery, and exact rollback | `TF-008`, `TF-009`, `TF-010` | Preserved | -| Stable production operation discovers existing sessions, new sessions, and forks automatically | `TF-011` | Preserved; implementation missing | +| Stable production operation discovers existing sessions, new sessions, and forks automatically | `TF-011` | Preserved and implemented behind preview, compatibility, health, stability, and storage gates | | macOS, Linux, and Windows share one storage engine but have independent adapters and readiness gates | `TF-012`, `TF-016`, `TF-017` | Preserved | | Capability language cannot overstate a storage engine, preview, or one successful canary | `TF-013` | Preserved | | Native sources and current recoverable bytes remain available until the relevant gates pass | `TF-010`, `TF-014`, `TF-015` | Preserved | -| Useless or closed branches can be identified and archived, but the tool must not guess destructively | `TF-018` | Added to the contract; implementation missing | +| Useless or closed branches can be identified and archived, but the tool must not guess destructively | `TF-018` | Implemented with evidence-only family reports and explicit recoverable archive transactions | | A branch that is exactly 100% contained in another retained session can be deleted only after exact recovery proof | `TF-019` | Added to the contract; implementation exists | -| Prompt cleanup, repair, and reconciliation are separate content-changing workflows, not storage folding | `TF-020` | Added to the contract; implementation boundary exists, regression coverage is incomplete | -| Temporary files, recovery generations, retained snapshots, and repeated operations must not consume unbounded disk | `TF-021` | Added to the contract; implementation missing | -| Reported savings distinguish logical reuse from actual physical bytes reclaimed | `TF-021` | Added to the contract; implementation missing | +| Prompt cleanup, repair, and reconciliation are separate content-changing workflows, not storage folding | `TF-020` | Added to the contract; implementation and static regression boundaries exist | +| Temporary files, recovery generations, retained snapshots, and repeated operations must not consume unbounded disk | `TF-021` | Added to the contract; hard budgets, bounded retention, leases, and GC are implemented | +| Reported savings distinguish logical reuse from actual physical bytes reclaimed | `TF-021` | Added to the contract; projected and actual physical accounting is implemented | | CodexFold is an independent open-source product with no private control-plane dependency | `TF-022` | Added to the contract; current repository is aligned | ## Requirement To Implementation And Evidence | Requirement | Current implementation | Tests or evidence | Status | | --- | --- | --- | --- | -| `TF-001` | `internal/cli/fs.go`, `internal/mountfs`, canonical migration and routing | Isolated CLI/Desktop direct-open and resume canaries | Partial: verified for isolated macOS canaries; automatic general enrollment is missing | +| `TF-001` | `internal/cli/fs.go`, `internal/mountfs`, canonical migration, automatic enrollment, and routing | Isolated CLI/Desktop direct-open, resume, and automatic-enrollment canaries | Partial only at release level: implemented and verified on macOS canaries; real-home automatic apply remains preview-gated | | `TF-002` | `internal/mountfs`, `internal/sessionns`, `internal/mountid` | Real FUSE-T operation tests and isolated unmodified clients | Implemented for the validated macOS client versions | -| `TF-003` | Neutral operation layer plus exact compatibility contracts in `internal/compat` | Real macOS traces and adapter canaries | Partial: current installed macOS clients are covered; Linux and Windows are not validated | +| `TF-003` | Neutral operation layer plus exact compatibility contracts in `internal/compat` | Real macOS traces and adapter canaries plus real Linux FUSE3 operations | Partial: current installed macOS clients and Linux adapter operations are covered; real Linux Codex clients and Windows are not validated | | `TF-004` | `internal/scan`, `internal/cdc`, `internal/fold`, `internal/pack` | Repeated field, record, CDC, fork, and non-prefix corpus tests | Implemented | | `TF-005` | `internal/vfs` append delta and writer leases | Append-without-hydration tests and real CLI/Desktop append evidence | Implemented | | `TF-006` | `internal/vfs` copy-on-write backing and neutral write operations | Random-write, truncate, interruption, and real FUSE-T mutation tests | Implemented | | `TF-007` | Immutable packs, in-memory index, bounded cache, random-read resolver | Pack round-trip/corruption tests and 758 MiB packed-read benchmark | Implemented | -| `TF-008` | `internal/fsctl` benchmark and `internal/testfs` stress harness | `docs/validation-fs-preview.md` and synchronous FUSE-T read/write measurements | Partial: shared-core and measured warm macOS gates pass; cold/full-distribution metrics and other platforms remain open | +| `TF-008` | `internal/fsctl` benchmark and `internal/testfs` stress harness | `docs/validation-fs-preview.md`, synchronous FUSE-T measurements, and Linux FUSE3 race performance | Partial: shared-core, measured macOS, and Linux adapter safety floors pass; cold/full-distribution and Windows metrics remain open | | `TF-009` | Journal recovery, generation recovery, service keep-alive, restart-safe retirement | Recovery tests, daemon restart canaries, managed Deep Idle sleep/wake, and actual retained-source host reboot | Partial: no actual power loss during an in-flight transaction | | `TF-010` | Shadow compare, optimistic routes, retained snapshots, current-byte fallback | 90,000 real random-range comparisons, rollback and failure-containment canaries, and one bounded retained-source user-home canary | Implemented for macOS canaries; retention remains open | -| `TF-011` | Codex state discovery primitives exist in `internal/codex` | Discovery unit tests | Missing: no stability policy, batch planner, or automatic enrollment loop | -| `TF-012` | Shared Go core and macOS FUSE-T adapter | macOS real adapter tests; Linux and Windows non-CGO compile checks | Partial: Linux and Windows real adapters are missing | +| `TF-011` | `internal/enroll`, `fs enroll`, and the bounded standalone-service loop discover existing, new, and forked sessions, persist stability observations, take a fail-closed native-writer snapshot, and reuse fold/pack/migrate transactions | Policy tests plus a real writable-descriptor probe and isolated canonical FUSE enrollment, daemon restart, real CLI append, quarantine, and failed-cutover evidence | Implemented; real-home automatic apply remains disabled until platform promotion | +| `TF-012` | Shared Go core, explicitly selected macOS FUSE-T NFS, Linux FUSE3, and Windows WinFsp adapters | Real macOS and Linux adapter tests; rejected FUSE-T FSKit correctness canary; default and WinFsp Windows cross-compiles | Partial: Windows has implementation and compile evidence only | | `TF-013` | Canonical capability type in `internal/fsctl/status.go` | Status rejection tests and CLI status tests | Implemented; current status is `fs-engine-preview` | | `TF-014` | Snapshot retention and destructive-action guards | Migration, rollback, and quarantine tests | Implemented as a safety rule; retention promotion gates remain open | | `TF-015` | Exact-version compatibility and update preflight quarantine | Unknown-version fallback and isolated canary tests | Implemented; the currently installed macOS CLI and Desktop are covered | -| `TF-016` | Tagged adapter prerequisite errors and non-elevating service lifecycle | Stub, service, and authorization-gated FUSE-T evidence | Implemented | -| `TF-017` | Canonical namespace, write-sealed backing, mount identity, synchronous Darwin mount policy, route normalization, process lock | Neutral, real FUSE-T, launchd, Desktop restart, stale-offset write, and rollback tests | Implemented for macOS canaries | -| `TF-018` | Canonical archive/unarchive file operations exist, but no conservative family classifier or guarded archive product workflow exists | No qualifying end-to-end tests | Missing | +| `TF-016` | Tagged adapter prerequisite errors and non-elevating native launchd, `systemd --user`, and Windows SCM lifecycle | Stub, authorization-gated FUSE-T, real Linux service, and Windows cross-compile evidence | Implemented; Windows runtime execution remains unverified | +| `TF-017` | Canonical namespace, write-sealed backing, mount identity, platform mount policy, route normalization, process lock | Neutral, real FUSE-T, real Linux FUSE3, launchd/systemd restart, stale-offset write, crash recovery, and rollback tests | Implemented for macOS canaries and Linux adapter gates; Windows remains unverified | +| `TF-018` | `internal/codex` spawn edges, `internal/family` graph/content evidence, `internal/archive` guarded transactions, and public `fork-family` plus `archive` commands | Diverse relationship fixtures, repeated-record performance regression, source-change rejection, official archive trace, native apply/recovery, and isolated managed FUSE-T archive/unarchive plus daemon restart | Implemented | | `TF-019` | `internal/contain` and `internal/prune`; public `contains` and `remove-contained` commands | Exact containment, archived-only apply, transaction rollback, and recovery-manifest tests | Implemented | -| `TF-020` | Exact fold/migrate paths are byte-preserving; `repair-rollout` and `reconcile-rollout` write separate explicit outputs | `internal/reconcile` and repair tests | Partial: add direct CLI boundary and non-invocation regression tests | -| `TF-021` | Individual temporary files are transactional, but no global inventory, hard preflight budget, bounded retired-state cleanup, or truthful reclamation report exists | No qualifying product-wide tests | Missing | -| `TF-022` | Standalone CLI, daemon, launchd renderer, configuration, storage, doctor, GC, rollback, and enrollment code | Public coupling scan and sanitization test | Implemented | +| `TF-020` | Exact fold/migrate paths are byte-preserving; `repair-rollout` and `reconcile-rollout` write separate explicit outputs; a static production-import boundary prevents other workflows from invoking reconciliation | `internal/reconcile`, CLI behavior, and AST boundary tests | Implemented | +| `TF-021` | `internal/storage` provides physical inventory, configurable hard budgets, mutation preflight, generation and retired-state retention, lease-aware startup/explicit GC, and projected versus actual reclamation | Hard-link accounting, low-space refusal, lease retention, interrupted cleanup, repeated GC, cross-platform compile, and live read-only inventory evidence | Implemented; destructive retention remains promotion-gated | +| `TF-022` | Standalone CLI, daemon, launchd/systemd/SCM service management, configuration, storage, doctor, GC, rollback, and enrollment code | Public coupling scan and sanitization test | Implemented | ## Implementation Plan Task Status @@ -68,31 +68,30 @@ Baseline reviewed: commit `045eea1` on 2026-07-14. | Task 4: journal, compaction, and fallback | Complete | Commit `076d772`; recovery, compaction, and latest-byte fallback tests pass | None in Task 4 | | Task 5: shadow, doctor, benchmark, and status | Complete | Commit `35a53fc`; focused and shared-core evidence exists | Real-platform promotion remains outside Task 5 | | Task 6: compatibility and route transactions | Complete | Commit `5be10d2`; route race and exact-version tests pass | New client versions require new contracts, not a redesign | -| Task 7: neutral filesystem and tagged FUSE host | Complete | Commit `3f51aa5`; neutral and real macOS FUSE-T tests pass | Linux and Windows real adapters remain platform work | -| Task 8: standalone CLI and automatic enrollment | Partial | Commit `3352b87`; command surface and guarded lifecycle exist | Task 8 Step 5, bounded automatic enrollment, is missing | -| Task 9: service lifecycle and update guard | Complete | Commit `4589ffa`; launchd and preflight tests pass | Stronger automatic update claims remain release-gated | +| Task 7: neutral filesystem and tagged FUSE host | Complete | Commit `3f51aa5`; neutral, real macOS FUSE-T, and real Linux FUSE3 tests pass; Windows WinFsp cross-compiles | Windows real-adapter execution remains platform work | +| Task 8: standalone CLI and automatic enrollment | Complete | Command surface, guarded lifecycle, bounded planner/apply loop, native service arguments, and isolated real FUSE enrollment evidence | Production enablement remains outside Task 8 | +| Task 9: service lifecycle and update guard | Complete | Commit `4589ffa`; launchd, real `systemd --user`, Windows SCM compile, and preflight tests pass | Windows runtime and stronger automatic update claims remain release-gated | | Task 10: synthetic, crash, performance, and compile gates | Complete for the shared engine | Commit `a1ac76e`; preview validation report | It cannot satisfy real-adapter or retention gates | | Task 11: real macOS trace, adapter, shadow, and canary | Partial | Sanitized real CLI/Desktop/FUSE-T evidence, managed sleep/wake, retained-source host reboot, current-client contracts, canonical user-home activation, and synchronous isolated plus bounded user-home real CLI canaries are public | Dedicated user-home canary retention, actual in-flight power loss, and seven-day retention remain | +| Task 12: bounded automatic discovery and enrollment | Complete | Planner/apply/service tests plus isolated canonical FUSE enrollment, daemon restart, real CLI append, quarantine, and failed-cutover evidence | Real-home automatic apply remains platform-gated | +| Task 13: conservative branch lifecycle and content-change boundary | Complete | Spawn-edge family reports, exact relationship comparison, official-compatible guarded archive and recovery, separate exact-contained deletion, and static content-change boundaries pass unit, race, native, and managed FUSE-T validation | None in this task | +| Task 14: hard storage budgets, retention, cleanup, and accounting | Complete | Platform-neutral inventory, hard preflight, lease-aware bounded GC, truthful accounting, low-space and repeated-GC tests | Destructive retention remains platform-gated | +| Task 15: remaining platform and retention gates | Partial | Current macOS client contracts; real Linux FUSE3 operation, crash, performance, backing policy, and systemd lifecycle; Windows WinFsp/SCM cross-compile | Actual in-flight power loss, retention period, Linux client/upgrade/rollback gates, and all real Windows gates remain | -## Missing Product Behavior And Exact Next Work +## Remaining Product Behavior And Exact Next Work | Missing behavior | Exact implementation work | Required verification | | --- | --- | --- | -| Bounded automatic discovery and enrollment | Add a read-only policy planner over Codex state, stability evidence, writer state, doctor, compatibility, promotion stage, and storage budget; then add an idempotent bounded apply loop that does not change a route before fold, pack, shadow, snapshot, and mount acknowledgement succeed | Existing/new/forked sessions, active and changing files, unknown clients, failed doctor, low disk, batch limits, restart idempotency, and failed cutover in an isolated home | -| Conservative fork-family classification | Add evidence-only reports for exact shared content, independent tails, complete containment, active/archive state, and unknown relationships; never infer uselessness from ancestry, age, title, or size | Diverse fork and non-fork fixtures plus real sanitized families; zero automatic mutation | -| Guarded branch archival | Trace the current official Codex archive behavior, then implement a dry-run-first explicit archive transaction that revalidates the selected route and digest and preserves the rollout | Concurrent route change, active writer, source mutation, archive/unarchive round trip, daemon restart, and recovery | -| Content-changing boundary regression | Add CLI tests proving repair and reconciliation require a separate output and cannot be invoked by fold, migration, compaction, enrollment, rollback, or GC | Command-tree tests, call-boundary tests, unchanged source hashes, and verified output hashes | -| Hard disk budgets and bounded retention | Add a storage inventory and preflight budget used by every operation that can create a full copy or generation; enforce one migration snapshot, one current fallback, one transaction scratch file, and bounded old generations | Low-space refusal before write, repeated migration/rollback/enrollment, interrupted cleanup, live lease retention, and no unbounded retired-state growth | -| Actual physical reclamation reporting | Extend status, doctor, and mutating results with logical, unique, pack, source, snapshot, fallback, temporary/recovery, projected peak, projected reclaimable, and actual reclaimed bytes | Fixture accounting checked against filesystem allocation before and after GC/removal; zero reclaimed bytes while full copies remain | | Remaining macOS disruptive and retention gates | Keep the dedicated retained-source user-home canary bounded to one explicitly selected session; perform an actual in-flight power-loss test only in a disposable host or VM; then complete the incident-free retention window | Clean doctor, exact SHA after restart and recovery cases, rollback, no route loss, and seven incident-free days | -| Linux and Windows readiness | Implement FUSE3 and WinFsp adapters without moving shared behavior out of the core | Native operation traces, crash/restart, performance, upgrade quarantine, rollback, and retention on each platform | +| Linux remaining readiness | Keep the implemented FUSE3 adapter and systemd lifecycle behind platform gates | Real Linux Codex traces, client upgrade quarantine, rollback, and retention | +| Windows readiness | Execute the implemented WinFsp adapter and SCM host without moving shared behavior out of the core | Native operations, crash/restart, performance, real Codex traces, upgrade quarantine, rollback, and retention on a real Windows host | ## Current Capability Decision -The shared storage and virtual-file engines are implemented and validated strongly enough for `fs-engine-preview`. The repository does not yet satisfy automatic stable enrollment, bounded physical-space governance, complete macOS disruptive gates and retention, or real Linux and Windows adapter gates. Therefore: +The shared storage, virtual-file, bounded automatic-enrollment, and physical-space governance engines are implemented and validated strongly enough for `fs-engine-preview`. Linux now has real adapter and native service evidence, while Windows remains implementation and cross-compile only. The repository still lacks the remaining macOS retention and disruptive gates, Linux real-client and lifecycle promotion gates, and all real Windows gates. Therefore: - Keep the capability at `fs-engine-preview`. - Keep real user sessions native unless they are explicitly selected for a retained-source canary. - Do not claim physical disk reclamation from logical deduplication alone. -- Do not enable automatic enrollment until Tasks 12 and 14 pass. +- Keep real-home automatic enrollment disabled until the macOS promotion and retention gates pass, even though Tasks 12 and 14 are implemented. - Do not introduce a private control-plane dependency into any public surface. diff --git a/docs/validation-linux-fuse3.md b/docs/validation-linux-fuse3.md new file mode 100644 index 0000000..2eeaf2f --- /dev/null +++ b/docs/validation-linux-fuse3.md @@ -0,0 +1,51 @@ +# Linux FUSE3 Validation + +## Status + +The Linux adapter has passed a real FUSE3 gate on Debian 12 as an unprivileged user. The same validation run used a race-enabled binary built with `CGO_ENABLED=1` and `-tags "fuse fuse3"`. The default non-CGo Linux build still compiles to the explicit prerequisite stub and does not silently select FUSE2. + +This evidence validates the Linux adapter and the `systemd --user` service lifecycle. It does not validate a real Linux Codex client, a client upgrade, the retention window, or a real Windows host. The project therefore remains `fs-engine-preview`. + +## Real Adapter Gate + +The race-enabled FUSE3 suite passed all of the following against disposable roots: + +- Exact managed reads, append plus `fsync`, random writes through complete copy-on-write, truncate, clean unmount, and remount. +- Canonical archive and unarchive renames, managed-over-native preference, native fallback, and managed-state removal. +- A separately hosted filesystem process killed with `SIGKILL`, strict detection of the resulting disconnected mount, `fusermount3 -uz` recovery, backing-directory resealing, and a successful replacement mount. +- A `0500` unmounted backing directory before activation and after every normal or crash-recovery shutdown. +- A dependency boundary that keeps `internal/fold` independent of Codex SQLite discovery and keeps `internal/mountfs` independent of `internal/codex`, `internal/service`, and all `modernc.org` packages. + +The latest race run measured a 16 MiB sequential managed read at `65.65 MiB/s` and 50 append-plus-`fsync` operations at `1.131011 ms` p95. These exceed the current Linux safety floors of `25 MiB/s` sequential read and `250 ms` append-plus-`fsync` p95. They are adapter gates, not universal hardware claims. + +## Real Systemd User Gate + +The generated unit passed `systemd-analyze --user verify`. A FUSE3-enabled CodexFold binary then completed this isolated lifecycle through the public CLI: + +1. `fs service install --apply` wrote the user unit, enabled it, started it, and returned only after both the process and CodexFold mount identity were healthy. +2. `fs service status` reported `daemon_running=true` and `mount_healthy=true`; the kernel exposed a `fuse` mount with source `codexfold`. +3. A direct `SIGKILL` of the running service process triggered the unit's restart policy. The replacement used a new main PID, recovered the disconnected FUSE mount, reset `ExecMainStatus` to zero, and returned `daemon_running=true` plus `mount_healthy=true` with exactly one restart. +4. `fs service start --apply` also stopped and restarted the unit explicitly, produced a new main PID, and restored a healthy mount. +5. `fs service stop --apply` removed the mount and left its ordinary backing directory at mode `0500`. +6. A CLI-level real FUSE regression now starts `fs serve`, kills it with `SIGKILL`, starts the same command against the stale mount, verifies recovery, then uses `SIGTERM` for a clean `0500` shutdown. +7. The validation unit, enable symlink, process, mount, build toolchain, and temporary data were removed after the gate. + +`systemd --user` must already be available to the invoking user. A headless machine that must start the user service before login may require an administrator to enable user lingering. CodexFold does not self-elevate or change linger policy. + +## Windows Boundary + +The Windows path currently includes a WinFsp-tagged host, Windows mount identity probe, SCM configuration renderer, native service installation and status commands, restart policy, and an SCM handler that runs the same in-process `fs serve` implementation and cancels it on stop or shutdown. Default and `winfsp` CLI, adapter, and test binaries cross-compile as PE32+ x86-64 executables. + +No real Windows plus WinFsp machine has executed mount, append, copy-on-write, rename, crash, service restart, performance, upgrade quarantine, or rollback tests. Windows remains implementation and compile evidence only. + +## Reproduction Gates + +```bash +go test ./... -count=1 +go test -race ./... -count=1 +go vet ./... +CGO_ENABLED=1 go test -tags fuse ./... -count=1 -timeout 5m +CODEXFOLD_RUN_FUSE3_TEST=1 CGO_ENABLED=1 go test -race -tags "fuse fuse3" ./internal/mountfs -count=1 -timeout 5m +CODEXFOLD_RUN_SYSTEMD_USER_TEST=1 go test ./internal/service -count=1 -timeout 2m +GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -tags winfsp ./cmd/codexfold +``` diff --git a/docs/validation-macos-canary.md b/docs/validation-macos-canary.md index c0510a7..11e51e9 100644 --- a/docs/validation-macos-canary.md +++ b/docs/validation-macos-canary.md @@ -4,6 +4,38 @@ The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have passed for read, append, resume, fork, child-session enrollment, canonical archive/unarchive moves, launchd restart, rollback, namespace deactivation, and unknown-version quarantine. A retained-source CLI canary survived an actual host reboot while managed, then resumed through the recovered mount and rolled back to an exact native JSONL. The currently installed CLI and Desktop versions also passed exact compatibility and isolated retained-source canaries. Process-level interruption recovery now covers append, compaction, migration, and rollback, and a managed session passed an actual Deep Idle sleep/wake cycle followed by a real model turn. The user Codex home now uses the canonical namespace with ordinary sessions remaining native passthrough and one explicitly selected retained-source canary managed for observation. The project remains at `fs-engine-preview` because that canary has not completed retention, in-flight transaction evidence does not claim an actual power-loss test, and the seven-day incident-free gate has not completed. +Backend selection evidence on 2026-07-17: + +- The installed FUSE-T `1.2.7` FSKit extension was enabled through the official macOS settings UI and mounted only disposable paths under `/private/tmp` and Go test temporary directories. `statfs` reported filesystem `fuse-t`, source `file:///private/tmp/fuset-session-*/session.json`, and the `fskit` mount flag, proving that the test did not fall back to NFS. +- The FSKit backend passed exact reads, ordinary append, `fsync`, `F_FULLFSYNC`, random write, truncate, copy-on-write, unmount, remount, and ten consecutive performance mounts. Those ten runs read about 2.0-2.7 GiB/s, or 25-38% of their same-run APFS baselines, with append-plus-`fsync` p95 between about 5.7 and 12.0 ms. +- The FSKit backend failed the deterministic stale-offset correctness gate. Two complete records written through one descriptor at the same previous EOF produced only the second record; the visible result was `record 0, record 2` instead of `record 0, record 1, record 2`. The transport delivered a coalesced write and offers no equivalent of the NFS synchronous-mount correction. +- Three managed-to-native route transition tests also failed to expose the new bytes within five seconds. This matches FUSE-T's published limitation that its FSKit backend does not support FUSE notifications. Requiring an unmount/remount for rollback or route invalidation is incompatible with transparent active-session behavior. +- FUSE-T FSKit is therefore rejected, despite passing basic operations and acceptable throughput. The shipped macOS path remains explicitly selected FUSE-T NFS with `MNT_SYNCHRONOUS`; FSKit is not an automatic fallback. +- After removing the rejected backend and retaining `F_FULLFSYNC`, pure-managed `statfs`, and nested-mount scan protections, the final complete real NFS adapter suite passed in 9.168 seconds. In that run, mounted read throughput was 3.61 GiB/s versus 10.31 GiB/s on the same-run APFS baseline, and append-plus-`fsync` p95 was 5.006 ms versus 3.990 ms. These cache-sensitive figures establish ample headroom, not a fixed APFS percentage guarantee. + +Additional conservative branch-lifecycle evidence on 2026-07-16: + +- The current official CLI archive transaction was traced in a disposable Codex home. It moves the rollout byte-for-byte into the flat `archived_sessions` directory, sets `archived` and `archived_at`, advances only `updated_at_ms` using the database maximum plus one, preserves `updated_at`, updates `rollout_path`, and leaves global UI state unchanged. +- `fork-family show` reports the explicit spawn-edge component and active/archive state. `fork-family compare` keeps graph ancestry separate from exact content evidence and distinguishes identical applicable records, complete left/right containment, shared-prefix independent tails, other exact shared records, and unknown relationships. It never infers usefulness from age, title, size, or ancestry. +- Family comparison opens each rollout once, uses cursor-based duplicate matching, and rejects size, modification-time, or file-identity changes before returning evidence. A 1,000-identical-record regression completes within its bounded test context rather than performing quadratic file reopens. +- `archive` is dry-run-first. Apply requires the native writer probe, revalidates SQLite route and complete source SHA-256 inside an immediate transaction, writes a durable prepared/renamed journal, preserves official archive field behavior, and rolls back file and database state on failure. If commit acknowledgement is ambiguous, it leaves the exact target and journal intact instead of guessing; explicit recovery then either rolls back or finalizes from observed state. +- A disposable native rollout archived with an unchanged complete SHA-256 and no residual journal. A second disposable valid Codex session was folded, packed, migrated through the synchronous canonical FUSE-T namespace, officially unarchived, archived with CodexFold, and read back with the same complete SHA-256. After a complete filesystem-daemon stop and restart, the official CLI unarchived the same managed bytes again. Final rollback, service stop, and namespace deactivation left no validation mount or process behind. +- A static production-import regression allows the content-changing reconciliation package only in the explicit `fs_reconcile.go` CLI boundary. Fold, migration, compaction, enrollment, rollback, GC, archive, family reporting, and exact-contained deletion cannot invoke repair or reconciliation implicitly. +- Default, race, vet, FUSE-tagged, complete real FUSE-T, real Linux FUSE3 race, real Linux `systemd --user`, Linux/Windows default build, Windows WinFsp/SCM cross-compile, public-sanitization, and diff checks pass with Task 13 complete. The validation used only disposable homes and did not update the installed binary, the production daemon, or any real user session. Linux evidence and its remaining boundary are recorded separately in [the Linux FUSE3 validation](validation-linux-fuse3.md). + +Additional bounded automatic-enrollment evidence on 2026-07-16: + +- The standalone FUSE service ran a bounded periodic enrollment loop against a fresh isolated canonical Codex home. The first cycle persisted a stability observation; a later unchanged cycle selected one archived session and reused the public `fold`, `pack build`, and canonical `fs migrate` transactions rather than a separate migration implementation. +- The first empty-store run exposed and fixed a bootstrap gate: the absence of a committed pack generation is valid before the first enrollment only while no managed session state exists. A missing or broken committed generation still blocks enrollment. +- The service now takes one batched native-writer snapshot per planning cycle. A real isolated rollout held open through a writable descriptor was reported as `writer-active`; probe failure blocks planning instead of assuming that no writer exists. +- Automatic enrollment completed exact shadow verification, 10,000 random-range comparisons, retained one immutable native snapshot, received the exact mount acknowledgement, and preserved the canonical SQLite route. The visible file and retained snapshot matched the original SHA-256; generation remained 1 with an empty delta and no writable backing. +- After a complete daemon stop and restart, the synchronous FUSE-T mount restored the same visible bytes. Repeated policy cycles reported the session as already managed and created neither a second snapshot nor another managed state. +- The current unmodified Codex CLI unarchived and resumed the automatically enrolled session. The original 68,707-byte base prefix remained exact, the new 5,819 bytes were durable only in `append.delta`, the complete 74,526-byte JSONL parsed, and no copy-on-write backing appeared. +- An unknown-client compatibility canary materialized the latest 74,526 visible bytes, verified their SHA-256, routed SQLite to a normal native quarantine fallback, and retired managed state. It did not fall back to the stale migration snapshot. +- A separate isolated failure canary used a healthy noncanonical mount so canonical acknowledgement could never arrive. Migration timed out, left the SQLite route unchanged, restored the exact native source, removed the candidate snapshot, and left no active managed state. +- The lease regression discovered by the full real FUSE-T suite was fixed at the source: reader and resolver leases can no longer recursively recreate a retired session or missing pack generation. The failing same-path republish case then passed three consecutive real FUSE-T runs, followed by the complete real adapter suite. +- Default tests, FUSE-tagged tests, full race tests, `go vet`, real FUSE-T tests, real Linux FUSE3 and `systemd --user` gates, and Linux/Windows cross-platform build gates passed after these changes. Windows remains compile-only. Automatic apply remains disabled for the real Codex home while capability is `fs-engine-preview`. + Additional synchronous-write and canonical-activation evidence on 2026-07-16: - Canonical activation preserved all 2,313 native rollouts. The mounted and native path/size/mtime inventories matched exactly, and the pre/post underlying native inventory also matched path, size, mtime, inode, mode, owner, and group. Explicit critical canaries retained full SHA-256 checks. Ordinary sessions remained native passthrough and the managed-session count stayed zero. @@ -11,7 +43,7 @@ Additional synchronous-write and canonical-activation evidence on 2026-07-16: - The first dedicated real-home migration passed exact shadow verification and 10,000 random reads, but a real CLI turn exposed a corruption bug: the original 97,388-byte prefix remained exact while the final JSONL contained a partially overwritten record. The canary was rolled back and restored byte-for-byte before further work. - The real write trace showed that Codex opens rollout JSONL with `O_RDWR` and explicit offsets. A same-handle JSONL append guard was added, but the failing FUSE-T regression proved that the macOS NFS client could merge two same-offset `pwrite` calls before either reached CodexFold. Per-open and global libfuse `direct_io`, plus disabled NFS attribute caching, did not change that behavior. - Updating only the mounted localhost NFS volume with `mount -u -o sync` made the previously deterministic stale-offset regression pass. The Darwin adapter now withholds its health identity until that update succeeds and `MNT_SYNCHRONOUS` is visible through `statfs`; a failure unmounts the host instead of advertising readiness. No global NFS configuration, patched FUSE-T binary, privileged helper, or system-wide mount change is used. -- A 64 MiB mounted read measured 7,045 MiB/s versus 7,435 MiB/s from the native APFS file, or 95% of native throughput. Across 200 JSONL append-plus-`fsync` operations, the synchronous mount averaged 3.99 ms with a 5.03 ms p95, versus 3.84 ms and 5.26 ms natively. +- One historical warm-cache 64 MiB run measured 7,045 MiB/s versus 7,435 MiB/s from APFS, or 95%. That ratio is not a general performance guarantee: later same-run comparisons varied materially with APFS cache speed. The enforced gate is mounted throughput of at least 1 GiB/s and at least 25% of the same-run APFS baseline, plus bounded append-and-sync latency. - A fresh isolated real CLI canary used the official unarchive flow and then resumed through the synchronous canonical mount. The complete view grew from 97,388 to 120,859 bytes and 33 valid JSONL records. The complete original prefix retained SHA-256 `4cd4bcc1807d875b70e04b3028441f330f9c7ee0cd41cbcff08c18c9ec44d416`, the 23,471-byte delta parsed independently, the expected historical and new markers were recalled, generation remained 1, and no writable backing appeared. - The same dedicated canary was then enrolled in the canonical user home while every ordinary rollout remained native passthrough. A real current CLI unarchive and resume produced a 120,864-byte, 33-record valid JSONL with the same exact 97,388-byte prefix SHA-256, a separately valid 23,476-byte delta, generation 1, and no writable backing. The model recalled the historical marker and emitted the new acceptance marker. Official archive moved the managed route back to `archived_sessions`; exactly one managed session remains, the mounted volume reports synchronous I/O, and the complete filesystem doctor is healthy. - Default, FUSE-tagged, race, vet, shell, cross-platform compile, and complete real FUSE-T suites passed after the fix. The real FUSE-T suite explicitly requires synchronous mount readiness before exercising the stale-offset regression. diff --git a/internal/archive/archive.go b/internal/archive/archive.go new file mode 100644 index 0000000..0bb6995 --- /dev/null +++ b/internal/archive/archive.go @@ -0,0 +1,638 @@ +package archive + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/url" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/jstar0/codexfold/internal/codex" + _ "modernc.org/sqlite" +) + +type Options struct { + Apply bool + Now time.Time + WriterActive func(context.Context, codex.Session) (bool, error) + BeforeRename func() error + AfterRename func() error +} + +type Result struct { + SessionID string `json:"session_id"` + SourcePath string `json:"source_path"` + TargetPath string `json:"target_path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` + DryRun bool `json:"dry_run"` + Archived bool `json:"archived"` +} + +type RecoveryResult struct { + SessionID string `json:"session_id"` + SourcePath string `json:"source_path"` + TargetPath string `json:"target_path"` + RolledBack bool `json:"rolled_back"` + Finalized bool `json:"finalized"` +} + +type phase string + +const ( + phasePrepared phase = "prepared" + phaseRenamed phase = "renamed" +) + +type journal struct { + Version int `json:"version"` + SessionID string `json:"session_id"` + SourcePath string `json:"source_path"` + TargetPath string `json:"target_path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` + Phase phase `json:"phase"` +} + +type snapshot struct { + Bytes int64 + SHA256 string +} + +type threadRow struct { + RolloutPath string + Archived bool +} + +var commitArchiveTransaction = func(ctx context.Context, conn *sql.Conn) error { + _, err := conn.ExecContext(ctx, `commit`) + return err +} + +func JournalPath(store string, sessionID string) string { + return filepath.Join(filepath.Clean(store), "archive", "journals", sessionID+".json") +} + +func Archive(ctx context.Context, home string, store string, session codex.Session, options Options) (Result, error) { + if !validSessionID(session.ID) || !filepath.IsAbs(home) || !filepath.IsAbs(store) || !filepath.IsAbs(session.RolloutPath) { + return Result{}, errors.New("absolute home, store, rollout path, and safe session ID are required") + } + home = filepath.Clean(home) + store = filepath.Clean(store) + sourcePath := filepath.Clean(session.RolloutPath) + if session.Archived { + return Result{}, errors.New("session is already archived") + } + if _, err := relativeWithin(filepath.Join(home, "sessions"), sourcePath); err != nil { + return Result{}, errors.New("active rollout is outside the Codex sessions directory") + } + targetPath := filepath.Join(home, "archived_sessions", filepath.Base(sourcePath)) + if sourcePath == targetPath { + return Result{}, errors.New("archive source and target must differ") + } + current, err := hashPath(sourcePath) + if err != nil { + return Result{}, fmt.Errorf("hash active rollout: %w", err) + } + result := Result{ + SessionID: session.ID, SourcePath: sourcePath, TargetPath: targetPath, + Bytes: current.Bytes, SHA256: current.SHA256, DryRun: !options.Apply, + } + if _, err := os.Lstat(targetPath); err == nil { + return Result{}, errors.New("archive target already exists") + } else if !errors.Is(err, os.ErrNotExist) { + return Result{}, err + } + db, err := openStateDB(home) + if err != nil { + return Result{}, err + } + defer func() { _ = db.Close() }() + row, err := readThread(ctx, db, session.ID) + if err != nil { + return Result{}, err + } + if row.Archived || filepath.Clean(row.RolloutPath) != sourcePath { + return Result{}, errors.New("selected Codex thread is no longer active at the expected rollout") + } + if options.Apply && options.WriterActive == nil { + return Result{}, errors.New("archive apply requires a native writer probe") + } + if active, err := writerActive(ctx, session, options.WriterActive); err != nil { + return Result{}, err + } else if active { + return Result{}, errors.New("cannot archive a session with an active writer") + } + if !options.Apply { + return result, nil + } + journalPath := JournalPath(store, session.ID) + if _, err := os.Lstat(journalPath); err == nil { + return Result{}, errors.New("pending archive journal already exists") + } else if !errors.Is(err, os.ErrNotExist) { + return Result{}, err + } + if err := os.MkdirAll(filepath.Dir(targetPath), 0o700); err != nil { + return Result{}, err + } + pending := journal{ + Version: 1, SessionID: session.ID, SourcePath: sourcePath, TargetPath: targetPath, + Bytes: current.Bytes, SHA256: current.SHA256, Phase: phasePrepared, + } + if err := writeJournal(journalPath, pending); err != nil { + return Result{}, err + } + journalOwned := true + removeJournal := func() error { + if !journalOwned { + return nil + } + if err := os.Remove(journalPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + journalOwned = false + return syncArchiveDirectory(filepath.Dir(journalPath)) + } + if options.BeforeRename != nil { + if err := options.BeforeRename(); err != nil { + return Result{}, errors.Join(err, removeJournal()) + } + } + conn, err := db.Conn(ctx) + if err != nil { + return Result{}, errors.Join(err, removeJournal()) + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, `begin immediate`); err != nil { + return Result{}, errors.Join(fmt.Errorf("begin Codex archive transaction: %w", err), removeJournal()) + } + transactionClosed := false + defer func() { + if !transactionClosed { + _, _ = conn.ExecContext(context.Background(), `rollback`) + } + }() + row, err = readThreadConn(ctx, conn, session.ID) + if err != nil { + return Result{}, errors.Join(err, removeJournal()) + } + if row.Archived || filepath.Clean(row.RolloutPath) != sourcePath { + return Result{}, errors.Join(errors.New("Codex route or archive state changed before rename"), removeJournal()) + } + verified, err := hashPath(sourcePath) + if err != nil || verified != current { + if err == nil { + err = errors.New("active rollout changed before archive rename") + } + return Result{}, errors.Join(err, removeJournal()) + } + if active, err := writerActive(ctx, session, options.WriterActive); err != nil { + return Result{}, errors.Join(err, removeJournal()) + } else if active { + return Result{}, errors.Join(errors.New("cannot archive a session with an active writer"), removeJournal()) + } + if _, err := os.Lstat(targetPath); err == nil { + return Result{}, errors.Join(errors.New("archive target appeared before rename"), removeJournal()) + } else if !errors.Is(err, os.ErrNotExist) { + return Result{}, errors.Join(err, removeJournal()) + } + if err := os.Rename(sourcePath, targetPath); err != nil { + return Result{}, errors.Join(fmt.Errorf("rename rollout into archive: %w", err), removeJournal()) + } + renamed := true + rollbackFile := func() error { + if !renamed { + return nil + } + if _, err := os.Lstat(sourcePath); err == nil { + return errors.New("cannot roll back archive while source path exists") + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + matches, err := pathMatches(targetPath, current) + if err != nil { + return err + } + if !matches { + return errors.New("cannot roll back archive because target bytes changed") + } + if err := os.MkdirAll(filepath.Dir(sourcePath), 0o700); err != nil { + return err + } + if err := os.Rename(targetPath, sourcePath); err != nil { + return err + } + renamed = false + return syncArchiveRename(targetPath, sourcePath) + } + if err := syncArchiveRename(sourcePath, targetPath); err != nil { + return Result{}, errors.Join(err, rollbackFile(), removeJournal()) + } + pending.Phase = phaseRenamed + if err := writeJournal(journalPath, pending); err != nil { + return Result{}, errors.Join(err, rollbackFile(), removeJournal()) + } + if options.AfterRename != nil { + if err := options.AfterRename(); err != nil { + return Result{}, errors.Join(err, rollbackFile(), removeJournal()) + } + } + columns, err := threadColumns(ctx, conn) + if err != nil { + return Result{}, errors.Join(err, rollbackFile(), removeJournal()) + } + now := options.Now + if now.IsZero() { + now = time.Now() + } + query := `update threads set rollout_path = ?, archived = 1` + args := []any{targetPath} + if columns["archived_at"] { + query += `, archived_at = ?` + args = append(args, now.Unix()) + } + if columns["updated_at_ms"] { + var maximum sql.NullInt64 + if err := conn.QueryRowContext(ctx, `select max(updated_at_ms) from threads`).Scan(&maximum); err != nil { + return Result{}, errors.Join(err, rollbackFile(), removeJournal()) + } + if maximum.Valid && maximum.Int64 == math.MaxInt64 { + return Result{}, errors.Join(errors.New("thread update clock overflow"), rollbackFile(), removeJournal()) + } + next := int64(1) + if maximum.Valid { + next = maximum.Int64 + 1 + } + query += `, updated_at_ms = ?` + args = append(args, next) + } + query += ` where id = ? and rollout_path = ? and archived = 0` + args = append(args, session.ID, sourcePath) + update, err := conn.ExecContext(ctx, query, args...) + if err != nil { + return Result{}, errors.Join(err, rollbackFile(), removeJournal()) + } + rows, err := update.RowsAffected() + if err != nil || rows != 1 { + if err == nil { + err = fmt.Errorf("archive update affected %d rows", rows) + } + return Result{}, errors.Join(err, rollbackFile(), removeJournal()) + } + if err := commitArchiveTransaction(ctx, conn); err != nil { + _, _ = conn.ExecContext(context.Background(), `rollback`) + transactionClosed = true + verifyCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + after, stateErr := readThread(verifyCtx, db, session.ID) + if stateErr != nil { + return result, errors.Join(fmt.Errorf("archive commit outcome is unknown: %w", err), stateErr) + } + switch { + case after.Archived && filepath.Clean(after.RolloutPath) == targetPath: + renamed = false + result.DryRun = false + result.Archived = true + return result, fmt.Errorf("archive committed but commit acknowledgement failed; run archive recover --apply: %w", err) + case !after.Archived && filepath.Clean(after.RolloutPath) == sourcePath: + fileErr := rollbackFile() + var journalErr error + if fileErr == nil { + journalErr = removeJournal() + } + return result, errors.Join(fmt.Errorf("commit Codex archive transaction: %w", err), fileErr, journalErr) + default: + return result, errors.Join(fmt.Errorf("archive commit outcome is ambiguous: %w", err), errors.New("Codex thread route no longer matches either archive state")) + } + } + transactionClosed = true + renamed = false + result.DryRun = false + result.Archived = true + if err := removeJournal(); err != nil { + return result, err + } + return result, nil +} + +func Recover(ctx context.Context, home string, store string, sessionID string) (RecoveryResult, error) { + if !validSessionID(sessionID) || !filepath.IsAbs(home) || !filepath.IsAbs(store) { + return RecoveryResult{}, errors.New("absolute home and store paths and a safe session ID are required") + } + home = filepath.Clean(home) + store = filepath.Clean(store) + path := JournalPath(store, sessionID) + pending, err := readJournal(path) + if err != nil { + return RecoveryResult{}, err + } + if pending.SessionID != sessionID || !validSessionID(sessionID) { + return RecoveryResult{}, errors.New("archive journal session does not match recovery request") + } + if _, err := relativeWithin(filepath.Join(home, "sessions"), pending.SourcePath); err != nil { + return RecoveryResult{}, errors.New("archive journal source is outside the Codex sessions directory") + } + expectedTarget := filepath.Join(home, "archived_sessions", filepath.Base(pending.SourcePath)) + if filepath.Clean(pending.TargetPath) != expectedTarget { + return RecoveryResult{}, errors.New("archive journal target does not match the official flat archive path") + } + result := RecoveryResult{SessionID: sessionID, SourcePath: pending.SourcePath, TargetPath: pending.TargetPath} + db, err := openStateDB(home) + if err != nil { + return RecoveryResult{}, err + } + defer func() { _ = db.Close() }() + conn, err := db.Conn(ctx) + if err != nil { + return RecoveryResult{}, err + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, `begin immediate`); err != nil { + return RecoveryResult{}, fmt.Errorf("begin Codex archive recovery transaction: %w", err) + } + transactionClosed := false + defer func() { + if !transactionClosed { + _, _ = conn.ExecContext(context.Background(), `rollback`) + } + }() + row, err := readThreadConn(ctx, conn, sessionID) + if err != nil { + return RecoveryResult{}, err + } + want := snapshot{Bytes: pending.Bytes, SHA256: pending.SHA256} + sourceExists, sourceMatches, err := inspectPath(pending.SourcePath, want) + if err != nil { + return RecoveryResult{}, err + } + targetExists, targetMatches, err := inspectPath(pending.TargetPath, want) + if err != nil { + return RecoveryResult{}, err + } + switch { + case !row.Archived && filepath.Clean(row.RolloutPath) == filepath.Clean(pending.SourcePath): + switch { + case sourceExists && sourceMatches && !targetExists: + case !sourceExists && targetExists && targetMatches: + if err := os.MkdirAll(filepath.Dir(pending.SourcePath), 0o700); err != nil { + return RecoveryResult{}, err + } + if err := os.Rename(pending.TargetPath, pending.SourcePath); err != nil { + return RecoveryResult{}, err + } + if err := syncArchiveRename(pending.TargetPath, pending.SourcePath); err != nil { + return RecoveryResult{}, err + } + default: + return RecoveryResult{}, errors.New("archive rollback state is ambiguous or changed") + } + result.RolledBack = true + case row.Archived && filepath.Clean(row.RolloutPath) == filepath.Clean(pending.TargetPath): + if sourceExists || !targetExists || !targetMatches { + return RecoveryResult{}, errors.New("committed archive files are ambiguous or changed") + } + result.Finalized = true + default: + return RecoveryResult{}, errors.New("Codex thread state no longer matches the archive journal") + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return RecoveryResult{}, err + } + if err := syncArchiveDirectory(filepath.Dir(path)); err != nil { + return RecoveryResult{}, err + } + if _, err := conn.ExecContext(ctx, `commit`); err != nil { + return RecoveryResult{}, fmt.Errorf("commit Codex archive recovery transaction: %w", err) + } + transactionClosed = true + return result, nil +} + +func openStateDB(home string) (*sql.DB, error) { + dbPath := filepath.Join(filepath.Clean(home), "state_5.sqlite") + info, err := os.Stat(dbPath) + if err != nil { + return nil, fmt.Errorf("locate Codex archive database: %w", err) + } + if !info.Mode().IsRegular() { + return nil, errors.New("Codex archive database is not a regular file") + } + db, err := sql.Open("sqlite", sqliteReadWriteDSN(dbPath)) + if err != nil { + return nil, fmt.Errorf("open Codex archive database: %w", err) + } + if _, err := db.Exec(`pragma busy_timeout = 10000`); err != nil { + _ = db.Close() + return nil, err + } + return db, nil +} + +func sqliteReadWriteDSN(path string) string { + slashPath := filepath.ToSlash(path) + if runtime.GOOS == "windows" { + slashPath = strings.ReplaceAll(path, "\\", "/") + if !strings.HasPrefix(slashPath, "/") { + slashPath = "/" + slashPath + } + } + uri := &url.URL{Scheme: "file", Path: slashPath} + query := uri.Query() + query.Set("mode", "rw") + uri.RawQuery = query.Encode() + return uri.String() +} + +func readThread(ctx context.Context, db *sql.DB, sessionID string) (threadRow, error) { + var row threadRow + var archived int + if err := db.QueryRowContext(ctx, `select rollout_path, archived from threads where id = ?`, sessionID).Scan(&row.RolloutPath, &archived); err != nil { + return threadRow{}, fmt.Errorf("read Codex archive thread: %w", err) + } + row.Archived = archived != 0 + return row, nil +} + +func readThreadConn(ctx context.Context, conn *sql.Conn, sessionID string) (threadRow, error) { + var row threadRow + var archived int + if err := conn.QueryRowContext(ctx, `select rollout_path, archived from threads where id = ?`, sessionID).Scan(&row.RolloutPath, &archived); err != nil { + return threadRow{}, fmt.Errorf("revalidate Codex archive thread: %w", err) + } + row.Archived = archived != 0 + return row, nil +} + +func threadColumns(ctx context.Context, conn *sql.Conn) (map[string]bool, error) { + rows, err := conn.QueryContext(ctx, `pragma table_info(threads)`) + if err != nil { + return nil, err + } + defer rows.Close() + columns := make(map[string]bool) + for rows.Next() { + var cid int + var name, dataType string + var notNull, primaryKey int + var defaultValue any + if err := rows.Scan(&cid, &name, &dataType, ¬Null, &defaultValue, &primaryKey); err != nil { + return nil, err + } + columns[name] = true + } + return columns, rows.Err() +} + +func writerActive(ctx context.Context, session codex.Session, probe func(context.Context, codex.Session) (bool, error)) (bool, error) { + if probe == nil { + return false, nil + } + active, err := probe(ctx, session) + if err != nil { + return false, fmt.Errorf("probe archive writer: %w", err) + } + return active, nil +} + +func hashPath(path string) (snapshot, error) { + info, err := os.Lstat(path) + if err != nil { + return snapshot{}, err + } + if !info.Mode().IsRegular() { + return snapshot{}, errors.New("rollout path is not a regular file") + } + file, err := os.Open(path) + if err != nil { + return snapshot{}, err + } + hasher := sha256.New() + written, copyErr := io.Copy(hasher, file) + closeErr := file.Close() + if copyErr != nil || closeErr != nil { + return snapshot{}, errors.Join(copyErr, closeErr) + } + return snapshot{Bytes: written, SHA256: hex.EncodeToString(hasher.Sum(nil))}, nil +} + +func hashBytes(data []byte) string { + digest := sha256.Sum256(data) + return hex.EncodeToString(digest[:]) +} + +func pathMatches(path string, want snapshot) (bool, error) { + got, err := hashPath(path) + if err != nil { + return false, err + } + return got == want, nil +} + +func inspectPath(path string, want snapshot) (bool, bool, error) { + got, err := hashPath(path) + if errors.Is(err, os.ErrNotExist) { + return false, false, nil + } + if err != nil { + return false, false, err + } + return true, got == want, nil +} + +func validSessionID(sessionID string) bool { + return sessionID != "" && sessionID != "." && sessionID != ".." && filepath.Base(sessionID) == sessionID && !strings.ContainsAny(sessionID, "/\\\x00") +} + +func relativeWithin(root string, target string) (string, error) { + relative, err := filepath.Rel(filepath.Clean(root), filepath.Clean(target)) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) || filepath.IsAbs(relative) { + return "", errors.New("path is outside root") + } + return relative, nil +} + +func syncArchiveRename(sourcePath string, targetPath string) error { + sourceDirectory := filepath.Dir(sourcePath) + targetDirectory := filepath.Dir(targetPath) + if err := syncArchiveDirectory(sourceDirectory); err != nil { + return err + } + if targetDirectory == sourceDirectory { + return nil + } + return syncArchiveDirectory(targetDirectory) +} + +func writeJournal(path string, value journal) error { + if value.Version != 1 || !validSessionID(value.SessionID) || !filepath.IsAbs(value.SourcePath) || !filepath.IsAbs(value.TargetPath) || value.Bytes < 0 || len(value.SHA256) != 64 || value.Phase != phasePrepared && value.Phase != phaseRenamed { + return errors.New("complete archive journal metadata is required") + } + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + directory := filepath.Dir(path) + if err := os.MkdirAll(directory, 0o700); err != nil { + return err + } + temporary, err := os.CreateTemp(directory, ".archive-*.tmp") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return err + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + if err := os.Rename(temporaryPath, path); err != nil { + return err + } + return syncArchiveDirectory(directory) +} + +func readJournal(path string) (journal, error) { + data, err := os.ReadFile(path) + if err != nil { + return journal{}, err + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + var value journal + if err := decoder.Decode(&value); err != nil { + return journal{}, err + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + if err == nil { + err = errors.New("multiple JSON values") + } + return journal{}, err + } + if value.Version != 1 || !validSessionID(value.SessionID) || !filepath.IsAbs(value.SourcePath) || !filepath.IsAbs(value.TargetPath) || value.Bytes < 0 || len(value.SHA256) != 64 || value.Phase != phasePrepared && value.Phase != phaseRenamed { + return journal{}, errors.New("invalid archive journal") + } + return value, nil +} diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go new file mode 100644 index 0000000..3f0bb43 --- /dev/null +++ b/internal/archive/archive_test.go @@ -0,0 +1,338 @@ +package archive + +import ( + "context" + "database/sql" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/jstar0/codexfold/internal/codex" + _ "modernc.org/sqlite" +) + +func TestArchiveDryRunAndApplyMatchOfficialFileAndStateBehavior(t *testing.T) { + fixture := archiveFixture(t) + originalGlobal, err := os.ReadFile(fixture.globalPath) + if err != nil { + t.Fatal(err) + } + dry, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{Now: fixture.now}) + if err != nil { + t.Fatal(err) + } + if !dry.DryRun || dry.Archived || dry.TargetPath != filepath.Join(fixture.home, "archived_sessions", filepath.Base(fixture.session.RolloutPath)) { + t.Fatalf("dry-run result = %#v", dry) + } + assertActiveSource(t, fixture) + + result, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{ + Apply: true, Now: fixture.now, WriterActive: idleWriter, + }) + if err != nil { + t.Fatal(err) + } + if result.DryRun || !result.Archived || result.Bytes != int64(len(fixture.source)) || result.SHA256 == "" { + t.Fatalf("archive result = %#v", result) + } + if _, err := os.Lstat(fixture.session.RolloutPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("active source remains after archive: %v", err) + } + archived, err := os.ReadFile(result.TargetPath) + if err != nil || string(archived) != string(fixture.source) { + t.Fatalf("archived bytes changed: %q err=%v", archived, err) + } + var path string + var archivedFlag int + var archivedAt sql.NullInt64 + var updatedAt int64 + var updatedAtMillis int64 + if err := fixture.db.QueryRow(`select rollout_path, archived, archived_at, updated_at, updated_at_ms from threads where id = ?`, fixture.session.ID).Scan( + &path, &archivedFlag, &archivedAt, &updatedAt, &updatedAtMillis, + ); err != nil { + t.Fatal(err) + } + if path != result.TargetPath || archivedFlag != 1 || !archivedAt.Valid || archivedAt.Int64 != fixture.now.Unix() || updatedAt != 100 || updatedAtMillis != 301 { + t.Fatalf("archived database row = path=%s archived=%d archived_at=%#v updated=%d/%d", path, archivedFlag, archivedAt, updatedAt, updatedAtMillis) + } + afterGlobal, err := os.ReadFile(fixture.globalPath) + if err != nil || string(afterGlobal) != string(originalGlobal) { + t.Fatalf("archive changed global state: %q err=%v", afterGlobal, err) + } + if _, err := os.Lstat(JournalPath(fixture.store, fixture.session.ID)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("completed archive left a journal: %v", err) + } +} + +func TestArchiveApplyRequiresWriterProbe(t *testing.T) { + fixture := archiveFixture(t) + _, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{Apply: true}) + if err == nil { + t.Fatal("archive apply accepted a missing writer probe") + } + assertActiveSource(t, fixture) +} + +func TestArchiveRejectsWriterRouteChangeAndSourceMutationBeforeRename(t *testing.T) { + t.Run("writer", func(t *testing.T) { + fixture := archiveFixture(t) + _, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{ + Apply: true, + WriterActive: func(context.Context, codex.Session) (bool, error) { + return true, nil + }, + }) + if err == nil { + t.Fatal("active writer was not rejected") + } + assertActiveSource(t, fixture) + }) + + t.Run("route change", func(t *testing.T) { + fixture := archiveFixture(t) + _, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{ + Apply: true, WriterActive: idleWriter, + BeforeRename: func() error { + _, err := fixture.db.Exec(`update threads set rollout_path = ? where id = ?`, filepath.Join(fixture.home, "other.jsonl"), fixture.session.ID) + return err + }, + }) + if err == nil { + t.Fatal("concurrent route change was not rejected") + } + if data, readErr := os.ReadFile(fixture.session.RolloutPath); readErr != nil || string(data) != string(fixture.source) { + t.Fatalf("route-race source changed: %q err=%v", data, readErr) + } + }) + + t.Run("source mutation", func(t *testing.T) { + fixture := archiveFixture(t) + _, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{ + Apply: true, WriterActive: idleWriter, + BeforeRename: func() error { + return os.WriteFile(fixture.session.RolloutPath, append(append([]byte(nil), fixture.source...), []byte("{\"changed\":true}\n")...), 0o600) + }, + }) + if err == nil { + t.Fatal("concurrent source mutation was not rejected") + } + var archived int + if dbErr := fixture.db.QueryRow(`select archived from threads where id = ?`, fixture.session.ID).Scan(&archived); dbErr != nil || archived != 0 { + t.Fatalf("source-race database changed: archived=%d err=%v", archived, dbErr) + } + }) +} + +func TestArchiveFailureAfterRenameRollsBackFileDatabaseAndJournal(t *testing.T) { + fixture := archiveFixture(t) + _, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{ + Apply: true, WriterActive: idleWriter, + AfterRename: func() error { + return errors.New("injected failure") + }, + }) + if err == nil { + t.Fatal("injected archive failure returned nil") + } + assertActiveSource(t, fixture) + if _, err := os.Lstat(filepath.Join(fixture.home, "archived_sessions", filepath.Base(fixture.session.RolloutPath))); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("failed archive left target: %v", err) + } + if _, err := os.Lstat(JournalPath(fixture.store, fixture.session.ID)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("rolled-back archive left journal: %v", err) + } +} + +func TestArchiveCommitFailureDoesNotGuessTransactionOutcome(t *testing.T) { + originalCommit := commitArchiveTransaction + t.Cleanup(func() { commitArchiveTransaction = originalCommit }) + + t.Run("not committed rolls back", func(t *testing.T) { + fixture := archiveFixture(t) + commitArchiveTransaction = func(context.Context, *sql.Conn) error { + return errors.New("injected commit failure") + } + _, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{ + Apply: true, WriterActive: idleWriter, + }) + if err == nil { + t.Fatal("commit failure returned nil") + } + assertActiveSource(t, fixture) + if _, statErr := os.Lstat(JournalPath(fixture.store, fixture.session.ID)); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("rolled-back commit failure left journal: %v", statErr) + } + }) + + t.Run("committed but acknowledgement failed leaves recovery journal", func(t *testing.T) { + fixture := archiveFixture(t) + commitArchiveTransaction = func(ctx context.Context, conn *sql.Conn) error { + if _, err := conn.ExecContext(ctx, `commit`); err != nil { + return err + } + return errors.New("lost commit acknowledgement") + } + result, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{ + Apply: true, WriterActive: idleWriter, + }) + if err == nil { + t.Fatal("ambiguous commit acknowledgement returned nil") + } + if _, statErr := os.Lstat(result.TargetPath); statErr != nil { + t.Fatalf("committed archive target missing: %v", statErr) + } + if _, statErr := os.Lstat(JournalPath(fixture.store, fixture.session.ID)); statErr != nil { + t.Fatalf("ambiguous commit did not retain recovery journal: %v", statErr) + } + recovered, recoverErr := Recover(context.Background(), fixture.home, fixture.store, fixture.session.ID) + if recoverErr != nil || !recovered.Finalized || recovered.RolledBack { + t.Fatalf("recover committed archive = %#v err=%v", recovered, recoverErr) + } + }) +} + +func TestRecoverRollsBackRenamedFileOrFinalizesCommittedState(t *testing.T) { + t.Run("rollback", func(t *testing.T) { + fixture := archiveFixture(t) + target := filepath.Join(fixture.home, "archived_sessions", filepath.Base(fixture.session.RolloutPath)) + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(fixture.session.RolloutPath, target); err != nil { + t.Fatal(err) + } + if err := writeJournal(JournalPath(fixture.store, fixture.session.ID), journal{ + Version: 1, SessionID: fixture.session.ID, SourcePath: fixture.session.RolloutPath, TargetPath: target, + Bytes: int64(len(fixture.source)), SHA256: hashBytes(fixture.source), Phase: phaseRenamed, + }); err != nil { + t.Fatal(err) + } + result, err := Recover(context.Background(), fixture.home, fixture.store, fixture.session.ID) + if err != nil || !result.RolledBack || result.Finalized { + t.Fatalf("rollback recovery = %#v err=%v", result, err) + } + assertActiveSource(t, fixture) + }) + + t.Run("finalize", func(t *testing.T) { + fixture := archiveFixture(t) + target := filepath.Join(fixture.home, "archived_sessions", filepath.Base(fixture.session.RolloutPath)) + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(fixture.session.RolloutPath, target); err != nil { + t.Fatal(err) + } + if _, err := fixture.db.Exec(`update threads set rollout_path = ?, archived = 1, archived_at = ? where id = ?`, target, fixture.now.Unix(), fixture.session.ID); err != nil { + t.Fatal(err) + } + if err := writeJournal(JournalPath(fixture.store, fixture.session.ID), journal{ + Version: 1, SessionID: fixture.session.ID, SourcePath: fixture.session.RolloutPath, TargetPath: target, + Bytes: int64(len(fixture.source)), SHA256: hashBytes(fixture.source), Phase: phaseRenamed, + }); err != nil { + t.Fatal(err) + } + result, err := Recover(context.Background(), fixture.home, fixture.store, fixture.session.ID) + if err != nil || result.RolledBack || !result.Finalized { + t.Fatalf("finalize recovery = %#v err=%v", result, err) + } + if data, readErr := os.ReadFile(target); readErr != nil || string(data) != string(fixture.source) { + t.Fatalf("finalized target changed: %q err=%v", data, readErr) + } + }) +} + +func TestRecoverRejectsUnsafeIdentityAndJournalPaths(t *testing.T) { + fixture := archiveFixture(t) + if _, err := Recover(context.Background(), fixture.home, fixture.store, "../session"); err == nil { + t.Fatal("recovery accepted an unsafe session ID") + } + outside := filepath.Join(t.TempDir(), "outside.jsonl") + if err := os.WriteFile(outside, fixture.source, 0o600); err != nil { + t.Fatal(err) + } + if err := writeJournal(JournalPath(fixture.store, fixture.session.ID), journal{ + Version: 1, SessionID: fixture.session.ID, SourcePath: outside, + TargetPath: filepath.Join(fixture.home, "archived_sessions", filepath.Base(outside)), + Bytes: int64(len(fixture.source)), SHA256: hashBytes(fixture.source), Phase: phasePrepared, + }); err != nil { + t.Fatal(err) + } + if _, err := Recover(context.Background(), fixture.home, fixture.store, fixture.session.ID); err == nil { + t.Fatal("recovery accepted a journal outside the active sessions tree") + } +} + +type fixture struct { + home string + store string + db *sql.DB + session codex.Session + source []byte + globalPath string + now time.Time +} + +func archiveFixture(t *testing.T) fixture { + t.Helper() + home := t.TempDir() + store := filepath.Join(home, "fold-store") + rollout := filepath.Join(home, "sessions", "2026", "07", "16", "rollout-session.jsonl") + if err := os.MkdirAll(filepath.Dir(rollout), 0o700); err != nil { + t.Fatal(err) + } + source := []byte("{\"type\":\"session_meta\"}\n{\"value\":1}\n") + if err := os.WriteFile(rollout, source, 0o600); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(` + create table threads ( + id text primary key, + rollout_path text not null, + archived integer not null, + archived_at integer, + updated_at integer not null, + updated_at_ms integer not null + ); + insert into threads values ('session', ?, 0, null, 100, 200); + insert into threads values ('newer', '/tmp/newer.jsonl', 0, null, 300, 300); + `, rollout); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + globalPath := filepath.Join(home, ".codex-global-state.json") + if err := os.WriteFile(globalPath, []byte("{\"selectedThreadId\":\"session\"}\n"), 0o600); err != nil { + t.Fatal(err) + } + return fixture{ + home: home, store: store, db: db, + session: codex.Session{ID: "session", RolloutPath: rollout}, + source: source, globalPath: globalPath, now: time.Unix(1_800_000_000, 0), + } +} + +func assertActiveSource(t *testing.T, fixture fixture) { + t.Helper() + data, err := os.ReadFile(fixture.session.RolloutPath) + if err != nil || string(data) != string(fixture.source) { + t.Fatalf("active source = %q err=%v", data, err) + } + var path string + var archived int + if err := fixture.db.QueryRow(`select rollout_path, archived from threads where id = ?`, fixture.session.ID).Scan(&path, &archived); err != nil { + t.Fatal(err) + } + if path != fixture.session.RolloutPath || archived != 0 { + t.Fatalf("active row = path=%s archived=%d", path, archived) + } +} + +func idleWriter(context.Context, codex.Session) (bool, error) { + return false, nil +} diff --git a/internal/archive/sync_unix.go b/internal/archive/sync_unix.go new file mode 100644 index 0000000..8fb845c --- /dev/null +++ b/internal/archive/sync_unix.go @@ -0,0 +1,14 @@ +//go:build !windows + +package archive + +import "os" + +func syncArchiveDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} diff --git a/internal/archive/sync_windows.go b/internal/archive/sync_windows.go new file mode 100644 index 0000000..8409326 --- /dev/null +++ b/internal/archive/sync_windows.go @@ -0,0 +1,5 @@ +//go:build windows + +package archive + +func syncArchiveDirectory(string) error { return nil } diff --git a/internal/buildid/buildid.go b/internal/buildid/buildid.go new file mode 100644 index 0000000..30532dd --- /dev/null +++ b/internal/buildid/buildid.go @@ -0,0 +1,40 @@ +package buildid + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" +) + +func CurrentSHA256() (string, error) { + executable, err := os.Executable() + if err != nil { + return "", err + } + return FileSHA256(executable) +} + +func FileSHA256(path string) (string, error) { + if !filepath.IsAbs(path) { + return "", errors.New("absolute executable path is required") + } + file, err := os.Open(filepath.Clean(path)) + if err != nil { + return "", err + } + hasher := sha256.New() + _, copyErr := io.Copy(hasher, file) + closeErr := file.Close() + if copyErr != nil || closeErr != nil { + return "", errors.Join(copyErr, closeErr) + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func ValidSHA256(value string) bool { + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == sha256.Size +} diff --git a/internal/cli/archive.go b/internal/cli/archive.go new file mode 100644 index 0000000..3f7bb66 --- /dev/null +++ b/internal/cli/archive.go @@ -0,0 +1,121 @@ +package cli + +import ( + "context" + "errors" + "fmt" + + archivepkg "github.com/jstar0/codexfold/internal/archive" + "github.com/jstar0/codexfold/internal/codex" + "github.com/spf13/cobra" +) + +type archiveFlags struct { + codexHome string + storeDir string + apply bool + json bool +} + +func newArchiveCommand() *cobra.Command { + var flags archiveFlags + command := &cobra.Command{ + Use: "archive ", + Short: "Preview or explicitly archive one selected Codex session", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + home, err := codex.ResolveHome(flags.codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, flags.storeDir) + sessions, err := codex.LoadSessions(home) + if err != nil { + return err + } + session, err := findSession(sessions, args[0]) + if err != nil { + return err + } + writers, err := probeArchiveWriters(command.Context(), sessions) + if err != nil { + return err + } + result, err := archivepkg.Archive(command.Context(), home, store, session, archivepkg.Options{ + Apply: flags.apply, + WriterActive: func(_ context.Context, selected codex.Session) (bool, error) { + return writers[selected.ID], nil + }, + }) + if err != nil { + return err + } + if flags.json { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "dry_run=%t archived=%t session=%s bytes=%s sha256=%s target=%s\n", + result.DryRun, result.Archived, result.SessionID, formatBytes(result.Bytes), result.SHA256, result.TargetPath) + return err + }, + } + addArchiveFlags(command, &flags) + command.AddCommand(newArchiveRecoverCommand()) + return command +} + +func newArchiveRecoverCommand() *cobra.Command { + var flags archiveFlags + command := &cobra.Command{ + Use: "recover ", + Short: "Recover one interrupted archive transaction", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + if !flags.apply { + return errors.New("archive recovery requires --apply") + } + home, err := codex.ResolveHome(flags.codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, flags.storeDir) + sessions, err := codex.LoadSessions(home) + if err != nil { + return err + } + writers, err := probeArchiveWriters(command.Context(), sessions) + if err != nil { + return err + } + if writers[args[0]] { + return errors.New("cannot recover an archive transaction while the selected session has an active writer") + } + result, err := archivepkg.Recover(command.Context(), home, store, args[0]) + if err != nil { + return err + } + if flags.json { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "rolled_back=%t finalized=%t session=%s source=%s target=%s\n", + result.RolledBack, result.Finalized, result.SessionID, result.SourcePath, result.TargetPath) + return err + }, + } + addArchiveFlags(command, &flags) + return command +} + +func addArchiveFlags(command *cobra.Command, flags *archiveFlags) { + command.Flags().StringVar(&flags.codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&flags.storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().BoolVar(&flags.apply, "apply", false, "Apply the explicit archive or recovery mutation") + command.Flags().BoolVar(&flags.json, "json", false, "Emit JSON output") +} + +func probeArchiveWriters(ctx context.Context, sessions []codex.Session) (map[string]bool, error) { + writers, err := enrollmentWriterProbe(ctx, sessions) + if err != nil { + return nil, fmt.Errorf("probe native session writers: %w", err) + } + return writers, nil +} diff --git a/internal/cli/archive_test.go b/internal/cli/archive_test.go new file mode 100644 index 0000000..1dfb7df --- /dev/null +++ b/internal/cli/archive_test.go @@ -0,0 +1,213 @@ +package cli + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + archivepkg "github.com/jstar0/codexfold/internal/archive" + "github.com/jstar0/codexfold/internal/codex" + _ "modernc.org/sqlite" +) + +func TestArchiveCommandIsDryRunFirstAndMatchesOfficialApply(t *testing.T) { + fixture := archiveCLIFixture(t) + allowArchiveWriterProbe(t, nil) + + var output bytes.Buffer + root := NewRootCommand() + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"archive", fixture.sessionID, "--codex-home", fixture.home, "--store", fixture.store, "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("archive dry-run: %v", err) + } + var dry archivepkg.Result + if err := json.Unmarshal(output.Bytes(), &dry); err != nil || !dry.DryRun || dry.Archived { + t.Fatalf("archive dry-run = %#v err=%v output=%s", dry, err, output.String()) + } + if data, err := os.ReadFile(fixture.sourcePath); err != nil || string(data) != string(fixture.source) { + t.Fatalf("dry-run changed source: %q err=%v", data, err) + } + + output.Reset() + root = NewRootCommand() + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"archive", fixture.sessionID, "--codex-home", fixture.home, "--store", fixture.store, "--apply", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("archive apply: %v", err) + } + var applied archivepkg.Result + if err := json.Unmarshal(output.Bytes(), &applied); err != nil || applied.DryRun || !applied.Archived { + t.Fatalf("archive apply = %#v err=%v output=%s", applied, err, output.String()) + } + if data, err := os.ReadFile(applied.TargetPath); err != nil || string(data) != string(fixture.source) { + t.Fatalf("archived source = %q err=%v", data, err) + } + var rolloutPath string + var archived int + if err := fixture.db.QueryRow(`select rollout_path, archived from threads where id = ?`, fixture.sessionID).Scan(&rolloutPath, &archived); err != nil { + t.Fatal(err) + } + if rolloutPath != applied.TargetPath || archived != 1 { + t.Fatalf("archive route = %s archived=%d", rolloutPath, archived) + } +} + +func TestArchiveCommandFailsClosedWhenWriterProbeFailsOrReportsWriter(t *testing.T) { + t.Run("probe failure", func(t *testing.T) { + fixture := archiveCLIFixture(t) + allowArchiveWriterProbe(t, errors.New("lsof unavailable")) + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"archive", fixture.sessionID, "--codex-home", fixture.home, "--store", fixture.store, "--apply"}) + if err := root.Execute(); err == nil { + t.Fatal("archive accepted a failed native writer probe") + } + assertArchiveCLIActive(t, fixture) + }) + + t.Run("active writer", func(t *testing.T) { + fixture := archiveCLIFixture(t) + previous := enrollmentWriterProbe + enrollmentWriterProbe = func(context.Context, []codex.Session) (map[string]bool, error) { + return map[string]bool{fixture.sessionID: true}, nil + } + t.Cleanup(func() { enrollmentWriterProbe = previous }) + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"archive", fixture.sessionID, "--codex-home", fixture.home, "--store", fixture.store, "--apply"}) + if err := root.Execute(); err == nil { + t.Fatal("archive accepted an active native writer") + } + assertArchiveCLIActive(t, fixture) + }) +} + +func TestArchiveRecoverCommandRequiresApplyAndRestoresInterruptedRename(t *testing.T) { + fixture := archiveCLIFixture(t) + allowArchiveWriterProbe(t, nil) + target := filepath.Join(fixture.home, "archived_sessions", filepath.Base(fixture.sourcePath)) + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(fixture.sourcePath, target); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(fixture.source) + journal := map[string]any{ + "version": 1, "session_id": fixture.sessionID, + "source_path": fixture.sourcePath, "target_path": target, + "bytes": len(fixture.source), "sha256": hex.EncodeToString(digest[:]), "phase": "renamed", + } + journalData, err := json.MarshalIndent(journal, "", " ") + if err != nil { + t.Fatal(err) + } + journalPath := archivepkg.JournalPath(fixture.store, fixture.sessionID) + if err := os.MkdirAll(filepath.Dir(journalPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(journalPath, append(journalData, '\n'), 0o600); err != nil { + t.Fatal(err) + } + + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"archive", "recover", fixture.sessionID, "--codex-home", fixture.home, "--store", fixture.store}) + if err := root.Execute(); err == nil { + t.Fatal("archive recovery ran without --apply") + } + if _, err := os.Stat(target); err != nil { + t.Fatalf("recovery preview changed target: %v", err) + } + + var output bytes.Buffer + root = NewRootCommand() + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"archive", "recover", fixture.sessionID, "--codex-home", fixture.home, "--store", fixture.store, "--apply", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("archive recover apply: %v", err) + } + var recovered archivepkg.RecoveryResult + if err := json.Unmarshal(output.Bytes(), &recovered); err != nil || !recovered.RolledBack || recovered.Finalized { + t.Fatalf("archive recover = %#v err=%v output=%s", recovered, err, output.String()) + } + assertArchiveCLIActive(t, fixture) +} + +type archiveCLIState struct { + home string + store string + db *sql.DB + sessionID string + sourcePath string + source []byte +} + +func archiveCLIFixture(t *testing.T) archiveCLIState { + t.Helper() + home := t.TempDir() + store := filepath.Join(home, "fold-store") + sourcePath := filepath.Join(home, "sessions", "2026", "07", "16", "rollout-session.jsonl") + if err := os.MkdirAll(filepath.Dir(sourcePath), 0o700); err != nil { + t.Fatal(err) + } + source := []byte("{\"type\":\"session_meta\"}\n{\"value\":1}\n") + if err := os.WriteFile(sourcePath, source, 0o600); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(` + create table threads ( + id text primary key, title text, cwd text, rollout_path text, + model_provider text, model text, updated_at integer, updated_at_ms integer, + archived integer, archived_at integer, git_branch text + ); + insert into threads values ('session', 'Session', '/workspace', ?, 'provider', 'model', 100, 200, 0, null, 'main'); + `, sourcePath); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + return archiveCLIState{home: home, store: store, db: db, sessionID: "session", sourcePath: sourcePath, source: source} +} + +func allowArchiveWriterProbe(t *testing.T, probeErr error) { + t.Helper() + previous := enrollmentWriterProbe + enrollmentWriterProbe = func(context.Context, []codex.Session) (map[string]bool, error) { + return map[string]bool{}, probeErr + } + t.Cleanup(func() { enrollmentWriterProbe = previous }) +} + +func assertArchiveCLIActive(t *testing.T, fixture archiveCLIState) { + t.Helper() + data, err := os.ReadFile(fixture.sourcePath) + if err != nil || string(data) != string(fixture.source) { + t.Fatalf("active source = %q err=%v", data, err) + } + var rolloutPath string + var archived int + if err := fixture.db.QueryRow(`select rollout_path, archived from threads where id = ?`, fixture.sessionID).Scan(&rolloutPath, &archived); err != nil { + t.Fatal(err) + } + if rolloutPath != fixture.sourcePath || archived != 0 { + t.Fatalf("active route = %s archived=%d", rolloutPath, archived) + } +} diff --git a/internal/cli/content_boundary_test.go b/internal/cli/content_boundary_test.go new file mode 100644 index 0000000..fa02ec1 --- /dev/null +++ b/internal/cli/content_boundary_test.go @@ -0,0 +1,45 @@ +package cli + +import ( + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestContentChangingReconcilePackageHasOneExplicitCLIBoundary(t *testing.T) { + _, currentFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("resolve test source path") + } + repoRoot := filepath.Clean(filepath.Join(filepath.Dir(currentFile), "../..")) + internalRoot := filepath.Join(repoRoot, "internal") + fset := token.NewFileSet() + err := filepath.Walk(internalRoot, func(path string, info fs.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } + if info.IsDir() || !strings.HasSuffix(info.Name(), ".go") || strings.HasSuffix(info.Name(), "_test.go") { + return nil + } + file, err := parser.ParseFile(fset, path, nil, 0) + if err != nil { + return err + } + for _, imported := range file.Imports { + if imported.Path.Value != `"github.com/jstar0/codexfold/internal/reconcile"` { + continue + } + if filepath.Clean(path) != filepath.Join(repoRoot, "internal", "cli", "fs_reconcile.go") { + t.Errorf("content-changing reconcile package imported by %s", path) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/cli/fold.go b/internal/cli/fold.go index 8206fee..bfa1891 100644 --- a/internal/cli/fold.go +++ b/internal/cli/fold.go @@ -32,7 +32,7 @@ func newFoldCommand() *cobra.Command { if err != nil { return err } - result, err := fold.Fold(command.Context(), session, options) + result, err := fold.Fold(command.Context(), toFoldSession(session), options) if err != nil { return err } @@ -176,6 +176,13 @@ func findSession(sessions []codex.Session, sessionID string) (codex.Session, err return codex.Session{}, fmt.Errorf("Codex session not found: %s", sessionID) } +func toFoldSession(session codex.Session) fold.Session { + return fold.Session{ + ID: session.ID, Title: session.Title, CWD: session.CWD, + RolloutPath: session.RolloutPath, Archived: session.Archived, + } +} + func writeJSON(command *cobra.Command, value any) error { encoder := json.NewEncoder(command.OutOrStdout()) encoder.SetIndent("", " ") diff --git a/internal/cli/fork_family.go b/internal/cli/fork_family.go new file mode 100644 index 0000000..d7ead15 --- /dev/null +++ b/internal/cli/fork_family.go @@ -0,0 +1,106 @@ +package cli + +import ( + "fmt" + + "github.com/jstar0/codexfold/internal/codex" + "github.com/jstar0/codexfold/internal/family" + "github.com/spf13/cobra" +) + +func newForkFamilyCommand() *cobra.Command { + command := &cobra.Command{Use: "fork-family", Short: "Report fork graph and exact content evidence without mutation"} + command.AddCommand(newForkFamilyShowCommand()) + command.AddCommand(newForkFamilyCompareCommand()) + return command +} + +func newForkFamilyShowCommand() *cobra.Command { + var codexHome string + var jsonOutput bool + command := &cobra.Command{ + Use: "show ", + Short: "List the spawn-edge family and active or archived state", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + sessions, err := codex.LoadSessions(home) + if err != nil { + return err + } + edges, err := codex.LoadSpawnEdges(home) + if err != nil { + return err + } + report, err := family.Build(args[0], sessions, edges) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, report) + } + if _, err := fmt.Fprintf(command.OutOrStdout(), "seed=%s members=%d edges=%d missing=%d\n", report.SeedID, len(report.Members), len(report.Edges), len(report.MissingSessionIDs)); err != nil { + return err + } + for _, member := range report.Members { + if _, err := fmt.Fprintf(command.OutOrStdout(), "session=%s relation=%s archived=%t path=%s\n", member.ID, member.RelationToSeed, member.Archived, member.RolloutPath); err != nil { + return err + } + } + return nil + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newForkFamilyCompareCommand() *cobra.Command { + var codexHome string + var jsonOutput bool + command := &cobra.Command{ + Use: "compare ", + Short: "Compare two explicitly selected rollouts using exact record evidence", + Args: cobra.ExactArgs(2), + RunE: func(command *cobra.Command, args []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + sessions, err := codex.LoadSessions(home) + if err != nil { + return err + } + left, err := findSession(sessions, args[0]) + if err != nil { + return err + } + right, err := findSession(sessions, args[1]) + if err != nil { + return err + } + edges, err := codex.LoadSpawnEdges(home) + if err != nil { + return err + } + comparison, err := family.Compare(command.Context(), left, right, edges) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, comparison) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "left=%s right=%s graph=%s relation=%s exact=%t shared_prefix=%d shared=%d left_archived=%t right_archived=%t\n", + comparison.LeftID, comparison.RightID, comparison.GraphRelation, comparison.Relation, + comparison.VerifiedExact, comparison.SharedPrefixRecords, comparison.SharedRecords, + comparison.LeftArchived, comparison.RightArchived) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} diff --git a/internal/cli/fs.go b/internal/cli/fs.go index d146172..7890ea0 100644 --- a/internal/cli/fs.go +++ b/internal/cli/fs.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "math" "os" "os/exec" "path/filepath" @@ -20,20 +21,23 @@ import ( "github.com/jstar0/codexfold/internal/compat" "github.com/jstar0/codexfold/internal/fold" "github.com/jstar0/codexfold/internal/fsctl" + "github.com/jstar0/codexfold/internal/fskitproto" "github.com/jstar0/codexfold/internal/mountfs" "github.com/jstar0/codexfold/internal/pack" "github.com/jstar0/codexfold/internal/service" + "github.com/jstar0/codexfold/internal/storage" "github.com/jstar0/codexfold/internal/vfs" "github.com/spf13/cobra" ) type FSMigrateResult struct { - SessionID string `json:"session_id"` - Native vfs.NativeFile `json:"native"` - Target string `json:"target"` - Shadow fsctl.ShadowResult `json:"shadow"` - DryRun bool `json:"dry_run"` - Routed bool `json:"routed"` + SessionID string `json:"session_id"` + Native vfs.NativeFile `json:"native"` + Target string `json:"target"` + Shadow fsctl.ShadowResult `json:"shadow"` + DryRun bool `json:"dry_run"` + Routed bool `json:"routed"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` } type FSCompatibilityResult struct { @@ -46,25 +50,29 @@ type FSCompatibilityResult struct { type FSServeResult struct { MountPoint string `json:"mount_point"` ManagedSessions int `json:"managed_sessions"` + Frontend string `json:"frontend"` + ResourcePath string `json:"resource_path,omitempty"` DryRun bool `json:"dry_run"` } type FSRollbackResult struct { - SessionID string `json:"session_id"` - From string `json:"from"` - Target vfs.NativeFile `json:"target"` - RetiredState string `json:"retired_state,omitempty"` - DryRun bool `json:"dry_run"` - Routed bool `json:"routed"` + SessionID string `json:"session_id"` + From string `json:"from"` + Target vfs.NativeFile `json:"target"` + RetiredState string `json:"retired_state,omitempty"` + DryRun bool `json:"dry_run"` + Routed bool `json:"routed"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` } type FSCompactResult struct { - SessionID string `json:"session_id"` - CurrentGeneration uint64 `json:"current_generation"` - NextGeneration uint64 `json:"next_generation"` - Bytes int64 `json:"bytes,omitempty"` - SHA256 string `json:"sha256,omitempty"` - DryRun bool `json:"dry_run"` + SessionID string `json:"session_id"` + CurrentGeneration uint64 `json:"current_generation"` + NextGeneration uint64 `json:"next_generation"` + Bytes int64 `json:"bytes,omitempty"` + SHA256 string `json:"sha256,omitempty"` + DryRun bool `json:"dry_run"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` } type FSRecoverResult struct { @@ -73,6 +81,12 @@ type FSRecoverResult struct { DryRun bool `json:"dry_run"` } +type FSNativeValidationResult struct { + Healthy bool `json:"healthy"` + Report mountfs.NativePreflightReport `json:"report"` + Issues []mountfs.NativePreflightIssue `json:"issues,omitempty"` +} + type compatibilityFlags struct { contractsPath string cliPath string @@ -85,14 +99,17 @@ func newFSCommand() *cobra.Command { command := &cobra.Command{Use: "fs", Short: "Operate the transparent session filesystem"} command.AddCommand(newFSStatusCommand()) command.AddCommand(newFSDoctorCommand()) + command.AddCommand(newFSValidateNativeCommand()) command.AddCommand(newFSCompatibilityCommand()) command.AddCommand(newFSCompatibilityImportCommand()) command.AddCommand(newFSBenchmarkCommand()) command.AddCommand(newFSServeCommand()) + command.AddCommand(newFSNativeSupervisorCommand()) command.AddCommand(newFSMigrateCommand()) command.AddCommand(newFSRollbackCommand()) command.AddCommand(newFSCompactCommand()) command.AddCommand(newFSRecoverCommand()) + command.AddCommand(newFSEnrollCommand()) command.AddCommand(newFSRepairRolloutCommand()) command.AddCommand(newFSReconcileRolloutCommand()) command.AddCommand(newFSNamespaceCommand()) @@ -100,24 +117,103 @@ func newFSCommand() *cobra.Command { return command } +func newFSValidateNativeCommand() *cobra.Command { + var codexHome string + var nativeRoot string + var auditAll bool + var jsonOutput bool + command := &cobra.Command{ + Use: "validate-native", + Short: "Validate active native rollout JSONL before writer routing", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + root := nativeRoot + if root == "" { + root = filepath.Join(home, "fold-native") + } + result := FSNativeValidationResult{Healthy: true} + if auditAll { + audit, err := mountfs.AuditNativeWriterRollouts(command.Context(), root) + if err != nil { + return err + } + result.Report = audit.NativePreflightReport + result.Issues = audit.Issues + result.Healthy = len(result.Issues) == 0 + } else { + filesystem := mountfs.NewCanonical() + filesystem.SetNativeRoot(root) + result.Report, err = filesystem.ValidateNativeWriterRollouts(command.Context()) + if err != nil { + result.Healthy = false + result.Issues = []mountfs.NativePreflightIssue{{Message: err.Error()}} + } + } + if jsonOutput { + if err := writeJSON(command, result); err != nil { + return err + } + } else { + if _, err := fmt.Fprintf(command.OutOrStdout(), "healthy=%t files=%d bytes=%d validated=%d incremental=%d cached=%d issues=%d\n", result.Healthy, result.Report.Files, result.Report.Bytes, result.Report.ValidatedFiles, result.Report.IncrementalFiles, result.Report.CachedFiles, len(result.Issues)); err != nil { + return err + } + } + if !result.Healthy { + return fmt.Errorf("native rollout validation failed with %d issue(s)", len(result.Issues)) + } + return nil + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&nativeRoot, "native-root", "", "Native rollout root; defaults to /fold-native") + command.Flags().BoolVar(&auditAll, "audit-all", false, "Bypass the cache and report every invalid active rollout") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + func newFSStatusCommand() *cobra.Command { + var codexHome string + var storeDir string var jsonOutput bool command := &cobra.Command{ Use: "status", Short: "Report the highest verified filesystem capability", Args: cobra.NoArgs, RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) status, err := fsctl.NewStatus(verifiedCapability(), runtime.GOOS) if err != nil { return err } + status.Storage, err = storage.Scan(command.Context(), storage.Options{StoreDir: store, AllowMetadataIssues: true}) + if err != nil { + return err + } + status.StorageLimits, err = storage.LoadLimits(store) + if err != nil { + return err + } + status.AvailableBytes, err = storage.AvailableBytes(store) + if err != nil { + return err + } if jsonOutput { return writeJSON(command, status) } - _, err = fmt.Fprintf(command.OutOrStdout(), "capability=%s platform=%s\n", status.Capability, status.Platform) + _, err = fmt.Fprintf(command.OutOrStdout(), "capability=%s platform=%s logical=%s physical=%s available=%s\n", status.Capability, status.Platform, formatBytes(status.Storage.LogicalSessionBytes), formatBytes(status.Storage.TotalPhysicalBytes), formatBytes(status.AvailableBytes)) return err }, } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") return command } @@ -237,7 +333,14 @@ func newFSServeCommand() *cobra.Command { var foreground bool var canonicalNamespace bool var nativeRoot string + var frontend string + var nativeFSKitSocket string + var nativeFSKitResource string var operationTracePath string + var enrollmentInterval time.Duration + var enrollmentStableFor time.Duration + var enrollmentBatchSize int + var enrollmentCanary bool var jsonOutput bool command := &cobra.Command{ Use: "serve", @@ -248,24 +351,63 @@ func newFSServeCommand() *cobra.Command { if err != nil { return err } + if apply { + if err := requireFilesystemActivationAllowed(home); err != nil { + return err + } + } if canonicalNamespace { if nativeRoot == "" || !filepath.IsAbs(nativeRoot) { return errors.New("canonical namespace requires an absolute native root") } nativeRoot = filepath.Clean(nativeRoot) } + if frontend != "fuse" && frontend != "native-fskit" { + return errors.New("filesystem frontend must be fuse or native-fskit") + } + if frontend == "native-fskit" { + if runtime.GOOS != "darwin" { + return errors.New("native-fskit frontend is available only on macOS") + } + if !canonicalNamespace { + return errors.New("native-fskit frontend requires the canonical namespace") + } + } + if enrollmentInterval < 0 || enrollmentStableFor < 0 || enrollmentBatchSize < 0 { + return errors.New("enrollment timing and batch values cannot be negative") + } + if enrollmentInterval > 0 { + if !canonicalNamespace { + return errors.New("periodic enrollment requires the canonical namespace") + } + if enrollmentStableFor <= 0 || enrollmentBatchSize <= 0 { + return errors.New("periodic enrollment requires a positive stable window and batch size") + } + } + if enrollmentCanary && enrollmentInterval <= 0 { + return errors.New("enrollment canary requires periodic enrollment") + } store := resolveFoldStore(home, storeDir) mount := defaultMountPoint(home, mountPoint) states, err := vfs.DiscoverSessionStates(store) if err != nil { return err } - result := FSServeResult{MountPoint: mount, ManagedSessions: len(states), DryRun: !apply} + if nativeFSKitResource == "" { + nativeFSKitResource = filepath.Join(store, "fs", "native-fskit") + } + if nativeFSKitSocket == "" { + nativeFSKitSocket = defaultNativeFSKitSocket(home, nativeFSKitResource) + } + result := FSServeResult{MountPoint: mount, ManagedSessions: len(states), Frontend: frontend, DryRun: !apply} + if frontend == "native-fskit" { + result.ResourcePath = nativeFSKitResource + } if !apply { if jsonOutput { return writeJSON(command, result) } - _, err = fmt.Fprintf(command.OutOrStdout(), "dry_run=true mount=%s sessions=%d\n", mount, len(states)) + _, err = fmt.Fprintf(command.OutOrStdout(), "dry_run=true frontend=%s mount=%s sessions=%d resource=%s\n", frontend, mount, len(states), result.ResourcePath) return err } processLock, err := service.AcquireProcessLock(filepath.Join(store, "fs", "service.lock")) @@ -285,9 +427,14 @@ func newFSServeCommand() *cobra.Command { } result.ManagedSessions = len(states) } - if err := os.MkdirAll(mount, 0o700); err != nil { + if _, _, err := startupStorageGC(command.Context(), store); err != nil { + return err + } + states, err = vfs.DiscoverSessionStates(store) + if err != nil { return err } + result.ManagedSessions = len(states) var operationRecorder func(string) if operationTracePath != "" { recorder, closer, err := newOperationRecorder(operationTracePath) @@ -308,9 +455,50 @@ func newFSServeCommand() *cobra.Command { if canonicalNamespace { filesystem = mountfs.NewCanonical() filesystem.SetNativeRoot(nativeRoot) + if err := filesystem.RecoverNativeAppendTransactions(); err != nil { + return fmt.Errorf("recover native append transactions: %w", err) + } + if _, err := filesystem.ValidateNativeWriterRollouts(command.Context()); err != nil { + return fmt.Errorf("validate native writer rollouts: %w", err) + } } ctx, cancel := context.WithCancel(command.Context()) defer cancel() + var nativeWatcherDone chan error + if frontend == "native-fskit" { + nativeWatcherDone = make(chan error, 1) + go func() { + err := filesystem.WatchNativeNamespace(ctx) + nativeWatcherDone <- err + if err != nil && !errors.Is(err, context.Canceled) { + cancel() + } + }() + } + var enrollmentDone chan struct{} + if enrollmentInterval > 0 { + enrollmentDone = make(chan struct{}) + flags := enrollmentFlags{ + codexHome: home, storeDir: store, mountPoint: mount, nativeRoot: nativeRoot, + stableFor: enrollmentStableFor, batchSize: enrollmentBatchSize, + canonicalNamespace: canonicalNamespace, canary: enrollmentCanary, + } + go func() { + defer close(enrollmentDone) + runPeriodicEnrollment(ctx, flags, enrollmentInterval, func(result FSEnrollmentApplyResult, cycleErr error) { + if cycleErr != nil { + if !errors.Is(cycleErr, context.Canceled) { + _, _ = fmt.Fprintf(command.ErrOrStderr(), "enrollment cycle failed: %v\n", cycleErr) + } + return + } + if len(result.Plan.Selected) == 0 && result.Apply.Applied == 0 { + return + } + _, _ = fmt.Fprintf(command.ErrOrStderr(), "enrollment cycle sessions=%d selected=%d applied=%d\n", len(result.Plan.Decisions), len(result.Plan.Selected), result.Apply.Applied) + }) + }() + } closers := make([]io.Closer, 0) known := make(map[string]uint64) knownRoutes := make(map[string]string) @@ -451,9 +639,26 @@ func newFSServeCommand() *cobra.Command { } } }() - mountErr := mountfs.Mount(ctx, mountfs.HostOptions{MountPoint: mount, Filesystem: filesystem, Foreground: foreground, OperationRecorder: operationRecorder}) + var mountErr error + if frontend == "native-fskit" { + mountErr = mountfs.ServeNativeFSKit(ctx, filesystem, mountfs.NativeFSKitServerOptions{ + SocketPath: nativeFSKitSocket, ResourcePath: nativeFSKitResource, Recorder: operationRecorder, + }) + } else { + mountErr = mountfs.Mount(ctx, mountfs.HostOptions{MountPoint: mount, Filesystem: filesystem, Foreground: foreground, OperationRecorder: operationRecorder}) + } cancel() <-watcherDone + if enrollmentDone != nil { + <-enrollmentDone + } + var nativeWatcherErr error + if nativeWatcherDone != nil { + nativeWatcherErr = <-nativeWatcherDone + if errors.Is(nativeWatcherErr, context.Canceled) { + nativeWatcherErr = nil + } + } for _, closer := range closers { _ = closer.Close() } @@ -461,7 +666,7 @@ func newFSServeCommand() *cobra.Command { case watcherErr := <-watcherErrors: return watcherErr default: - return mountErr + return errors.Join(mountErr, nativeWatcherErr) } }, } @@ -472,11 +677,30 @@ func newFSServeCommand() *cobra.Command { command.Flags().BoolVar(&foreground, "foreground", true, "Keep the FUSE host in the foreground") command.Flags().BoolVar(&canonicalNamespace, "canonical-namespace", false, "Expose sessions and archived_sessions as a shared virtual namespace") command.Flags().StringVar(&nativeRoot, "native-root", "", "Backing root for unmanaged canonical session files") + command.Flags().StringVar(&frontend, "frontend", "fuse", "Filesystem frontend: fuse or native-fskit") + command.Flags().StringVar(&nativeFSKitSocket, "fskit-socket", "", "Native FSKit daemon Unix socket; defaults to a short per-home path in /private/tmp") + command.Flags().StringVar(&nativeFSKitResource, "fskit-resource", "", "Native FSKit resource; defaults to the security-scoped /fs/native-fskit directory") command.Flags().StringVar(&operationTracePath, "operation-trace", "", "Absolute path for sanitized FUSE operation names") + command.Flags().DurationVar(&enrollmentInterval, "enrollment-interval", 0, "Periodic stable-session enrollment interval; zero disables the loop") + command.Flags().DurationVar(&enrollmentStableFor, "enrollment-stable-for", time.Hour, "Required unchanged interval before periodic enrollment") + command.Flags().IntVar(&enrollmentBatchSize, "enrollment-batch-size", 1, "Maximum sessions enrolled per periodic cycle") + command.Flags().BoolVar(&enrollmentCanary, "enrollment-canary", false, "Allow periodic enrollment only in an explicitly isolated Codex home while capability remains preview") command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output for dry-run") return command } +func defaultNativeFSKitSocket(home string, resourcePath string) string { + if fskitproto.UsesDirectoryResource(resourcePath) { + return filepath.Join(filepath.Clean(resourcePath), "daemon.sock") + } + digest := sha256.Sum256([]byte(filepath.Clean(home))) + userHome, err := os.UserHomeDir() + if err == nil && runtime.GOOS == "darwin" { + return filepath.Join(userHome, "Library", "Containers", "vip.jstar.codexfold.fskitprofileprobe.module", "Data", "tmp", fmt.Sprintf("cf-%s.sock", hex.EncodeToString(digest[:4]))) + } + return filepath.Join("/private/tmp", fmt.Sprintf("codexfold-fskit-%d-%s.sock", os.Getuid(), hex.EncodeToString(digest[:4]))) +} + func syncCanonicalRetirement( store string, home string, @@ -555,6 +779,11 @@ func newFSMigrateCommand() *cobra.Command { if err != nil { return err } + if apply { + if err := requireFilesystemActivationAllowed(home); err != nil { + return err + } + } store := resolveFoldStore(home, storeDir) session, manifest, resolver, view, err := openFoldView(home, store, args[0]) if err != nil { @@ -610,6 +839,14 @@ func newFSMigrateCommand() *cobra.Command { if err := mountHealthProbe(mount); err != nil { return fmt.Errorf("filesystem mount point is not healthy: %w", err) } + projectedPersistent := int64(1 << 20) + if canonicalNamespace { + projectedPersistent += native.Bytes + } + storageAssessment, err := assessStoreMutation(command.Context(), store, storage.Projection{Operation: "fs-migrate", AdditionalPersistentBytes: projectedPersistent}) + if err != nil { + return err + } canonicalSource := "" canonicalRoute := "" if canonicalNamespace { @@ -623,7 +860,7 @@ func newFSMigrateCommand() *cobra.Command { if err != nil { return err } - retained, err := retainCanonicalSnapshot(store, session.ID, native) + retained, err := retainCanonicalSnapshot(command.Context(), store, session.ID, native, nil) if err != nil { return err } @@ -699,6 +936,7 @@ func newFSMigrateCommand() *cobra.Command { } result.Routed = true result.DryRun = false + result.Storage = storage.CompleteAccounting(command.Context(), storageAssessment, store) } if jsonOutput { return writeJSON(command, result) @@ -771,6 +1009,10 @@ func newFSRollbackCommand() *cobra.Command { result := FSRollbackResult{SessionID: state.SessionID, From: current.RolloutPath, Target: vfs.NativeFile{Path: filepath.Clean(targetPath)}, DryRun: !apply} if apply { if currentNativeFallback { + storageAssessment, err := assessStoreMutation(command.Context(), store, storage.Projection{Operation: "fs-rollback"}) + if err != nil { + return err + } target, err := hashPath(current.RolloutPath) if err != nil { return err @@ -783,6 +1025,7 @@ func newFSRollbackCommand() *cobra.Command { result.RetiredState = retiredState result.Routed = true result.DryRun = false + result.Storage = storage.CompleteAccounting(command.Context(), storageAssessment, store) if jsonOutput { return writeJSON(command, result) } @@ -807,6 +1050,23 @@ func newFSRollbackCommand() *cobra.Command { return err } defer rollbackLease.Close() + visible, err := managed.VisibleInfo() + if err != nil { + return err + } + reclaimableBytes := int64(0) + if info, err := os.Stat(targetPath); err == nil && info.Mode().IsRegular() { + reclaimableBytes = info.Size() + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + storageAssessment, err := assessStoreMutation(command.Context(), store, storage.Projection{ + Operation: "fs-rollback", AdditionalPersistentBytes: visible.Size, TemporaryBytes: visible.Size, + TemporaryPersistentOverlapBytes: visible.Size, ReclaimableBytes: reclaimableBytes, + }) + if err != nil { + return err + } target, err := managed.MaterializeCurrent(command.Context(), filepath.Clean(targetPath), true) if err != nil { return err @@ -904,6 +1164,7 @@ func newFSRollbackCommand() *cobra.Command { result.Target = target result.Routed = true result.DryRun = false + result.Storage = storage.CompleteAccounting(command.Context(), storageAssessment, store) } if jsonOutput { return writeJSON(command, result) @@ -951,6 +1212,24 @@ func newFSCompactCommand() *cobra.Command { return err } defer resolver.Close() + visible, err := managed.VisibleInfo() + if err != nil { + return err + } + persistentBytes, err := conservativeStoredBytes(visible.Size) + if err != nil { + return err + } + if persistentBytes > math.MaxInt64-persistentBytes { + return errors.New("compact storage byte estimate overflow") + } + persistentBytes *= 2 + storageAssessment, err := assessStoreMutation(command.Context(), store, storage.Projection{ + Operation: "fs-compact", AdditionalPersistentBytes: persistentBytes, TemporaryBytes: visible.Size, + }) + if err != nil { + return err + } var preparedResolver *pack.Resolver defer func() { if preparedResolver != nil { @@ -968,7 +1247,7 @@ func newFSCompactCommand() *cobra.Command { FieldThreshold: currentManifest.Settings.FieldThreshold, MaxJSONLineBytes: currentManifest.Settings.MaxJSONLineBytes, CDC: cdc.Options{MinBytes: currentManifest.Settings.CDCMinBytes, AverageBytes: currentManifest.Settings.CDCAverageBytes, MaxBytes: currentManifest.Settings.CDCMaxBytes}, } - if _, err := fold.Fold(ctx, codex.Session{ID: state.SessionID, Title: currentManifest.Session.Title, CWD: currentManifest.Session.CWD, RolloutPath: current.Path, Archived: true}, options); err != nil { + if _, err := fold.Fold(ctx, fold.Session{ID: state.SessionID, Title: currentManifest.Session.Title, CWD: currentManifest.Session.CWD, RolloutPath: current.Path, Archived: true}, options); err != nil { return vfs.PreparedGeneration{}, err } if _, err := pack.Build(ctx, store, pack.BuildOptions{}); err != nil { @@ -995,6 +1274,7 @@ func newFSCompactCommand() *cobra.Command { result.Bytes = compact.Bytes result.SHA256 = compact.SHA256 result.DryRun = false + result.Storage = storage.CompleteAccounting(command.Context(), storageAssessment, store) } if jsonOutput { return writeJSON(command, result) @@ -1290,13 +1570,54 @@ func requireStorageHealth(ctx context.Context, store string) error { return nil } +func assessStoreMutation(ctx context.Context, store string, projection storage.Projection) (storage.Assessment, error) { + guard, err := storage.DefaultGuard(store) + if err != nil { + return storage.Assessment{}, err + } + return guard.Check(ctx, projection) +} + +func conservativeStoredBytes(rawBytes int64) (int64, error) { + if rawBytes < 0 { + return 0, errors.New("storage byte estimate cannot be negative") + } + overhead := rawBytes/16 + 1<<20 + if rawBytes > math.MaxInt64-overhead { + return 0, errors.New("storage byte estimate overflow") + } + return rawBytes + overhead, nil +} + +func startupStorageGC(ctx context.Context, store string) (storage.StorageGCResult, bool, error) { + if err := requireStorageHealth(ctx, store); err != nil { + return storage.StorageGCResult{}, false, nil + } + result, err := storage.Collect(ctx, storage.GCOptions{StoreDir: store, Apply: true}) + return result, true, err +} + func fsDoctor(ctx context.Context, home string, store string, mount string) fsctl.DoctorReport { - serviceStatus := service.Manager{}.Status(ctx, serviceLabel, mount) + var serviceStatus service.Status + platform, platformErr := service.CurrentPlatform() + definition, definitionErr := resolveServiceDefinitionPath("") + if platformErr == nil && definitionErr == nil { + serviceStatus, platformErr = platformServiceStatus(ctx, platform, mount, definition) + } + if platformErr != nil || definitionErr != nil { + serviceStatus.DaemonError = errors.Join(platformErr, definitionErr).Error() + } + var storageInventory storage.Inventory + var storageLimits storage.Limits + var availableBytes int64 checks := []fsctl.Check{ {Component: fsctl.ComponentDaemon, Run: func(context.Context) error { if !serviceStatus.DaemonRunning { return errors.New(serviceStatus.DaemonError) } + if !serviceStatus.Build.Healthy { + return errors.New(serviceStatus.Build.Error) + } return nil }}, {Component: fsctl.ComponentMount, Run: func(context.Context) error { @@ -1325,6 +1646,19 @@ func fsDoctor(ctx context.Context, home string, store string, mount string) fsct } return nil }}, + {Component: fsctl.ComponentStorage, Run: func(ctx context.Context) error { + var err error + storageInventory, err = storage.Scan(ctx, storage.Options{StoreDir: store, AllowMetadataIssues: true}) + if err != nil { + return err + } + storageLimits, err = storage.LoadLimits(store) + if err != nil { + return err + } + availableBytes, err = storage.AvailableBytes(store) + return err + }}, } states, stateErr := vfs.DiscoverSessionStates(store) stateCheck := func(kind string) fsctl.Check { @@ -1396,7 +1730,11 @@ func fsDoctor(ctx context.Context, home string, store string, mount string) fsct return nil }}, ) - return fsctl.Doctor(ctx, checks) + report := fsctl.Doctor(ctx, checks) + report.Storage = storageInventory + report.StorageLimits = storageLimits + report.AvailableBytes = availableBytes + return report } func defaultMountPoint(home string, explicit string) string { diff --git a/internal/cli/fs_activation.go b/internal/cli/fs_activation.go new file mode 100644 index 0000000..c58070a --- /dev/null +++ b/internal/cli/fs_activation.go @@ -0,0 +1,43 @@ +package cli + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/jstar0/codexfold/internal/fsctl" +) + +func requireFilesystemActivationAllowed(home string) error { + capability := verifiedCapability() + if capability != fsctl.FSEnginePreview && capability != fsctl.PlatformCanary { + return nil + } + + userHome, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("resolve real Codex home: %w", err) + } + realHomes := []string{filepath.Join(userHome, ".codex")} + if configured := strings.TrimSpace(os.Getenv("CODEX_HOME")); configured != "" { + realHomes = append(realHomes, configured) + } + for _, realHome := range realHomes { + if sameActivationPath(home, realHome) { + return fmt.Errorf("real Codex home activation is disabled while filesystem capability is %s; only an isolated compatibility canary is allowed", capability) + } + } + return nil +} + +func sameActivationPath(left string, right string) bool { + left = filepath.Clean(left) + right = filepath.Clean(right) + if left == right { + return true + } + resolvedLeft, leftErr := filepath.EvalSymlinks(left) + resolvedRight, rightErr := filepath.EvalSymlinks(right) + return leftErr == nil && rightErr == nil && filepath.Clean(resolvedLeft) == filepath.Clean(resolvedRight) +} diff --git a/internal/cli/fs_activation_test.go b/internal/cli/fs_activation_test.go new file mode 100644 index 0000000..83e6abd --- /dev/null +++ b/internal/cli/fs_activation_test.go @@ -0,0 +1,73 @@ +package cli + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestPreviewRejectsEveryRealCodexHomeActivationEntryPoint(t *testing.T) { + userHome := t.TempDir() + t.Setenv("HOME", userHome) + t.Setenv("CODEX_HOME", "") + + codexHome := filepath.Join(userHome, ".codex") + store := filepath.Join(codexHome, "fold-store") + mount := filepath.Join(codexHome, "fold-fs") + native := filepath.Join(codexHome, "fold-native") + if err := os.MkdirAll(codexHome, 0o700); err != nil { + t.Fatal(err) + } + definition := filepath.Join(userHome, "com.codexfold.fs.plist") + if err := os.WriteFile(definition, []byte("fixture"), 0o600); err != nil { + t.Fatal(err) + } + + previousProbe := mountHealthProbe + mountHealthProbe = func(string) error { return nil } + t.Cleanup(func() { mountHealthProbe = previousProbe }) + + want := "real Codex home activation is disabled while filesystem capability is fs-engine-preview" + tests := []struct { + name string + args []string + }{ + { + name: "serve", + args: []string{"fs", "serve", "--codex-home", codexHome, "--store", store, "--mount", mount, "--apply"}, + }, + { + name: "service install", + args: []string{ + "fs", "service", "install", "--codex-home", codexHome, "--store", store, "--mount", mount, + "--native-root", native, "--canonical-namespace", "--plist", definition, "--apply", + }, + }, + { + name: "service start", + args: []string{"fs", "service", "start", "--codex-home", codexHome, "--mount", mount, "--plist", definition, "--apply"}, + }, + { + name: "namespace activate", + args: []string{ + "fs", "namespace", "activate", "--codex-home", codexHome, "--mount", mount, + "--native-root", native, "--apply", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs(test.args) + err := root.Execute() + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("activation error = %v, want %q", err, want) + } + }) + } +} diff --git a/internal/cli/fs_enroll.go b/internal/cli/fs_enroll.go new file mode 100644 index 0000000..b74e2f1 --- /dev/null +++ b/internal/cli/fs_enroll.go @@ -0,0 +1,309 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/jstar0/codexfold/internal/codex" + "github.com/jstar0/codexfold/internal/enroll" + "github.com/jstar0/codexfold/internal/fold" + "github.com/jstar0/codexfold/internal/fsctl" + "github.com/jstar0/codexfold/internal/pack" + "github.com/jstar0/codexfold/internal/storage" + "github.com/jstar0/codexfold/internal/vfs" + "github.com/spf13/cobra" +) + +type enrollmentFlags struct { + codexHome string + storeDir string + mountPoint string + nativeRoot string + stableFor time.Duration + batchSize int + canonicalNamespace bool + canary bool + jsonOutput bool +} + +type FSEnrollmentApplyResult struct { + Plan enroll.Plan `json:"plan"` + Apply enroll.ApplyResult `json:"apply"` +} + +type enrollmentCycleReporter func(FSEnrollmentApplyResult, error) + +var runEnrollmentCommand = func(ctx context.Context, args []string) error { + binary, err := os.Executable() + if err != nil { + return err + } + output, err := exec.CommandContext(ctx, binary, args...).CombinedOutput() + if err != nil { + return fmt.Errorf("%s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(output))) + } + return nil +} + +var runServiceEnrollmentCycle = runEnrollmentCycle + +func newFSEnrollCommand() *cobra.Command { + command := &cobra.Command{Use: "enroll", Short: "Plan and apply bounded automatic session enrollment"} + command.AddCommand(newFSEnrollPlanCommand()) + command.AddCommand(newFSEnrollApplyCommand()) + return command +} + +func newFSEnrollPlanCommand() *cobra.Command { + var flags enrollmentFlags + var record bool + command := &cobra.Command{ + Use: "plan", + Short: "Report eligible and blocked sessions without changing routes", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + plan, store, err := buildEnrollmentPlan(command.Context(), flags) + if err != nil { + return err + } + if record { + if err := enroll.SaveObservations(enrollmentObservationPath(store), plan.Observations); err != nil { + return err + } + } + if flags.jsonOutput { + return writeJSON(command, plan) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "sessions=%d selected=%d observations=%d\n", len(plan.Decisions), len(plan.Selected), len(plan.Observations)) + return err + }, + } + addEnrollmentFlags(command, &flags) + command.Flags().BoolVar(&record, "record-observations", false, "Persist this read-only stability observation for the next planning cycle") + return command +} + +func newFSEnrollApplyCommand() *cobra.Command { + var flags enrollmentFlags + var apply bool + command := &cobra.Command{ + Use: "apply", + Short: "Apply the selected bounded enrollment batch", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + if !apply { + return errors.New("enrollment apply requires --apply") + } + result, err := runEnrollmentCycle(command.Context(), flags) + if err != nil { + return err + } + if flags.jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "selected=%d applied=%d changed=%d managed=%d\n", result.Apply.Selected, result.Apply.Applied, result.Apply.SkippedChanged, result.Apply.SkippedManaged) + return err + }, + } + addEnrollmentFlags(command, &flags) + command.Flags().BoolVar(&apply, "apply", false, "Run the bounded fold, pack, and canonical migration transactions") + return command +} + +func runEnrollmentCycle(ctx context.Context, flags enrollmentFlags) (FSEnrollmentApplyResult, error) { + home, err := codex.ResolveHome(flags.codexHome) + if err != nil { + return FSEnrollmentApplyResult{}, err + } + if err := requireFilesystemActivationAllowed(home); err != nil { + return FSEnrollmentApplyResult{}, err + } + plan, store, err := buildEnrollmentPlan(ctx, flags) + if err != nil { + return FSEnrollmentApplyResult{}, err + } + if err := enroll.SaveObservations(enrollmentObservationPath(store), plan.Observations); err != nil { + return FSEnrollmentApplyResult{}, err + } + mount := defaultMountPoint(home, flags.mountPoint) + nativeRoot := flags.nativeRoot + if nativeRoot == "" { + nativeRoot = filepath.Join(home, "fold-native") + } + applied, err := enroll.Apply(ctx, plan, enroll.ApplyOptions{ + Limit: flags.batchSize, + IsManaged: func(_ context.Context, sessionID string) (bool, error) { + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return false, err + } + for _, state := range states { + if state.SessionID == sessionID { + return true, nil + } + } + return false, nil + }, + Apply: func(ctx context.Context, decision enroll.Decision) error { + return applyEnrollmentCommands(ctx, home, store, mount, nativeRoot, decision.SessionID, flags.canary) + }, + }) + result := FSEnrollmentApplyResult{Plan: plan, Apply: applied} + return result, err +} + +func runPeriodicEnrollment(ctx context.Context, flags enrollmentFlags, interval time.Duration, report enrollmentCycleReporter) { + if interval <= 0 { + return + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + result, err := runServiceEnrollmentCycle(ctx, flags) + if report != nil { + report(result, err) + } + if ctx.Err() != nil { + return + } + } + } +} + +func addEnrollmentFlags(command *cobra.Command, flags *enrollmentFlags) { + command.Flags().StringVar(&flags.codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&flags.storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().StringVar(&flags.mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + command.Flags().BoolVar(&flags.canonicalNamespace, "canonical-namespace", false, "Plan canonical-path enrollment without changing SQLite routes") + command.Flags().StringVar(&flags.nativeRoot, "native-root", "", "Canonical native backing root; defaults to /fold-native") + command.Flags().DurationVar(&flags.stableFor, "stable-for", time.Hour, "Required unchanged observation window") + command.Flags().IntVar(&flags.batchSize, "batch-size", 1, "Maximum sessions selected per cycle") + command.Flags().BoolVar(&flags.canary, "enrollment-canary", false, "Allow explicit enrollment only in an isolated Codex home while capability remains preview") + command.Flags().BoolVar(&flags.jsonOutput, "json", false, "Emit JSON output") +} + +func buildEnrollmentPlan(ctx context.Context, flags enrollmentFlags) (enroll.Plan, string, error) { + home, err := codex.ResolveHome(flags.codexHome) + if err != nil { + return enroll.Plan{}, "", err + } + store := resolveFoldStore(home, flags.storeDir) + mount := defaultMountPoint(home, flags.mountPoint) + if flags.canary { + userHome, err := os.UserHomeDir() + if err != nil { + return enroll.Plan{}, "", err + } + if err := validateCompatibilityCanary(home, filepath.Join(userHome, ".codex"), store, flags.canonicalNamespace, compatibilityFlags{cliPath: "none", desktopPath: "none"}); err != nil { + return enroll.Plan{}, "", err + } + } + sessions, err := codex.LoadSessions(home) + if err != nil { + return enroll.Plan{}, "", err + } + writers, err := enrollmentWriterProbe(ctx, sessions) + if err != nil { + return enroll.Plan{}, "", fmt.Errorf("probe native session writers: %w", err) + } + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return enroll.Plan{}, "", err + } + managed := make(map[string]struct{}, len(states)) + for _, state := range states { + managed[state.SessionID] = struct{}{} + } + observations, err := enroll.LoadObservations(enrollmentObservationPath(store)) + if err != nil { + return enroll.Plan{}, "", err + } + doctorHealthy := requireEnrollmentStorageHealth(ctx, store) == nil + compatibilityApproved := flags.canary + if !flags.canary { + compatibility, err := evaluateCompatibility(ctx, store, defaultCompatibilityFlags()) + if err == nil { + compatibilityApproved = len(compatibility.DetectionErrors) == 0 && compatibility.Evaluation.Approved + } + } + mountHealthy := mountHealthProbe(mount) == nil + guard, err := storage.DefaultGuard(store) + if err != nil { + return enroll.Plan{}, "", err + } + plan, err := enroll.Build(ctx, enroll.Input{ + Sessions: sessions, Managed: managed, Previous: observations, Now: time.Now(), + Policy: enroll.Policy{StableFor: flags.stableFor, BatchSize: flags.batchSize, ArchivedOnly: true}, + Gates: enroll.Gates{ + DoctorHealthy: doctorHealthy, CompatibilityApproved: compatibilityApproved, MountHealthy: mountHealthy, + CanonicalNamespace: flags.canonicalNamespace, EnrollmentAllowed: flags.canary || automaticEnrollmentAllowed(verifiedCapability()), + }, + WriterActive: func(_ context.Context, session codex.Session) (bool, error) { + return writers[session.ID], nil + }, + Budget: guard, + }) + return plan, store, err +} + +func requireEnrollmentStorageHealth(ctx context.Context, store string) error { + foldReport, err := fold.Doctor(ctx, store) + if err != nil { + return err + } + if foldReport.IssueCount != 0 { + return fmt.Errorf("fold doctor reported %d issues", foldReport.IssueCount) + } + packReport, err := pack.Doctor(ctx, store) + if err != nil { + return err + } + if packReport.IssueCount == 0 { + return nil + } + if _, currentErr := os.Lstat(filepath.Join(filepath.Clean(store), "packs", "CURRENT")); errors.Is(currentErr, os.ErrNotExist) { + states, stateErr := vfs.DiscoverSessionStates(store) + if stateErr != nil { + return stateErr + } + if len(states) == 0 { + return nil + } + } + return fmt.Errorf("pack doctor reported %d issues", packReport.IssueCount) +} + +func automaticEnrollmentAllowed(capability fsctl.Capability) bool { + return capability == fsctl.CrossPlatformReady || strings.HasPrefix(string(capability), "production-ready:") +} + +func enrollmentObservationPath(store string) string { + return filepath.Join(filepath.Clean(store), "enrollment", "observations.json") +} + +func applyEnrollmentCommands(ctx context.Context, home string, store string, mount string, nativeRoot string, sessionID string, canary bool) error { + commands := [][]string{ + {"fold", sessionID, "--codex-home", home, "--store", store, "--apply", "--overwrite"}, + {"pack", "build", "--codex-home", home, "--store", store}, + {"fs", "migrate", sessionID, "--codex-home", home, "--store", store, "--mount", mount, "--canonical-namespace", "--native-root", nativeRoot, "--apply"}, + } + if canary { + commands[2] = append(commands[2], "--compatibility-canary", "--cli", "none", "--desktop-app", "none") + } + for _, command := range commands { + if err := runEnrollmentCommand(ctx, command); err != nil { + return err + } + } + return nil +} diff --git a/internal/cli/fs_enroll_writer.go b/internal/cli/fs_enroll_writer.go new file mode 100644 index 0000000..d3bfb59 --- /dev/null +++ b/internal/cli/fs_enroll_writer.go @@ -0,0 +1,62 @@ +package cli + +import ( + "path/filepath" + "strings" + + "github.com/jstar0/codexfold/internal/codex" +) + +var enrollmentWriterProbe = detectEnrollmentWriters + +func parseEnrollmentWriterSnapshot(output []byte, sessions []codex.Session) map[string]bool { + aliases := make(map[string][]string, len(sessions)*2) + for _, session := range sessions { + for _, path := range enrollmentPathAliases(session.RolloutPath) { + aliases[path] = append(aliases[path], session.ID) + } + } + writers := make(map[string]bool) + var access string + var name string + flush := func() { + if name == "" || !strings.ContainsAny(access, "wu") { + access = "" + name = "" + return + } + for _, sessionID := range aliases[canonicalEnrollmentPath(name)] { + writers[sessionID] = true + } + access = "" + name = "" + } + for _, line := range strings.Split(string(output), "\n") { + if line == "" { + continue + } + switch line[0] { + case 'p', 'f': + flush() + case 'a': + access = line[1:] + case 'n': + name = strings.TrimSuffix(line[1:], " (deleted)") + } + } + flush() + return writers +} + +func enrollmentPathAliases(path string) []string { + path = canonicalEnrollmentPath(path) + aliases := []string{path} + if resolved, err := filepath.EvalSymlinks(path); err == nil && resolved != path { + aliases = append(aliases, resolved) + } + return aliases +} + +func canonicalEnrollmentPath(path string) string { + return filepath.Clean(path) +} diff --git a/internal/cli/fs_enroll_writer_other.go b/internal/cli/fs_enroll_writer_other.go new file mode 100644 index 0000000..49b27fc --- /dev/null +++ b/internal/cli/fs_enroll_writer_other.go @@ -0,0 +1,14 @@ +//go:build !darwin && !linux && !windows + +package cli + +import ( + "context" + "errors" + + "github.com/jstar0/codexfold/internal/codex" +) + +func detectEnrollmentWriters(context.Context, []codex.Session) (map[string]bool, error) { + return nil, errors.New("native writer probe is unavailable on this platform") +} diff --git a/internal/cli/fs_enroll_writer_test.go b/internal/cli/fs_enroll_writer_test.go new file mode 100644 index 0000000..ac037f9 --- /dev/null +++ b/internal/cli/fs_enroll_writer_test.go @@ -0,0 +1,92 @@ +package cli + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/jstar0/codexfold/internal/codex" + "github.com/jstar0/codexfold/internal/enroll" +) + +func TestParseEnrollmentWriterSnapshotBlocksWriteAndUpdateDescriptorsOnly(t *testing.T) { + root := t.TempDir() + sessions := []codex.Session{ + {ID: "read", RolloutPath: filepath.Join(root, "read.jsonl")}, + {ID: "write", RolloutPath: filepath.Join(root, "write.jsonl")}, + {ID: "update", RolloutPath: filepath.Join(root, "update.jsonl")}, + } + output := []byte("p1\nf3\nar\nn" + sessions[0].RolloutPath + "\n" + + "f4\naw\nn" + sessions[1].RolloutPath + "\n" + + "f5\nau\nn" + sessions[2].RolloutPath + "\n") + writers := parseEnrollmentWriterSnapshot(output, sessions) + if writers["read"] || !writers["write"] || !writers["update"] { + t.Fatalf("writer snapshot = %#v", writers) + } +} + +func TestParseEnrollmentWriterSnapshotBlocksEverySessionSharingAPath(t *testing.T) { + path := filepath.Join(t.TempDir(), "shared.jsonl") + sessions := []codex.Session{{ID: "first", RolloutPath: path}, {ID: "second", RolloutPath: path}} + writers := parseEnrollmentWriterSnapshot([]byte("p1\nf3\naw\nn"+path+"\n"), sessions) + if !writers["first"] || !writers["second"] { + t.Fatalf("shared-path writer snapshot = %#v", writers) + } +} + +func TestEnrollmentWriterProbeFailureFailsClosed(t *testing.T) { + home, storeDir, _ := fsFixture(t, true) + oldProbe := enrollmentWriterProbe + defer func() { enrollmentWriterProbe = oldProbe }() + enrollmentWriterProbe = func(context.Context, []codex.Session) (map[string]bool, error) { + return nil, context.DeadlineExceeded + } + _, _, err := buildEnrollmentPlan(context.Background(), enrollmentFlags{ + codexHome: home, storeDir: storeDir, mountPoint: filepath.Join(home, "mount"), + nativeRoot: filepath.Join(home, "fold-native"), canonicalNamespace: true, canary: true, + }) + if err == nil { + t.Fatal("writer probe failure did not stop enrollment planning") + } +} + +func TestEnrollmentPlanBlocksSessionReportedByNativeWriterProbe(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + oldProbe := enrollmentWriterProbe + defer func() { enrollmentWriterProbe = oldProbe }() + enrollmentWriterProbe = func(context.Context, []codex.Session) (map[string]bool, error) { + return map[string]bool{"session": true}, nil + } + oldMountProbe := mountHealthProbe + mountHealthProbe = func(string) error { return nil } + defer func() { mountHealthProbe = oldMountProbe }() + info, err := os.Stat(nativePath) + if err != nil { + t.Fatal(err) + } + if err := enroll.SaveObservations(enrollmentObservationPath(storeDir), enroll.Observations{"session": { + Path: nativePath, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), StableSinceUnixNano: time.Now().Add(-time.Hour).UnixNano(), + }}); err != nil { + t.Fatal(err) + } + plan, _, err := buildEnrollmentPlan(context.Background(), enrollmentFlags{ + codexHome: home, storeDir: storeDir, mountPoint: filepath.Join(home, "mount"), + nativeRoot: filepath.Join(home, "fold-native"), canonicalNamespace: true, + canary: true, stableFor: time.Nanosecond, batchSize: 1, + }) + if err != nil { + t.Fatal(err) + } + if len(plan.Selected) != 0 || len(plan.Decisions) != 1 { + t.Fatalf("writer-active plan = %#v", plan) + } + found := false + for _, reason := range plan.Decisions[0].Reasons { + found = found || reason == enroll.ReasonWriterActive + } + if !found { + t.Fatalf("writer-active reason missing: %#v", plan.Decisions[0]) + } +} diff --git a/internal/cli/fs_enroll_writer_unix.go b/internal/cli/fs_enroll_writer_unix.go new file mode 100644 index 0000000..e4bcc1e --- /dev/null +++ b/internal/cli/fs_enroll_writer_unix.go @@ -0,0 +1,31 @@ +//go:build darwin || linux + +package cli + +import ( + "context" + "fmt" + "os" + "os/exec" + + "github.com/jstar0/codexfold/internal/codex" +) + +func detectEnrollmentWriters(ctx context.Context, sessions []codex.Session) (map[string]bool, error) { + if len(sessions) == 0 { + return map[string]bool{}, nil + } + lsof := "/usr/sbin/lsof" + if _, err := os.Stat(lsof); err != nil { + resolved, lookErr := exec.LookPath("lsof") + if lookErr != nil { + return nil, fmt.Errorf("native writer probe requires lsof: %w", lookErr) + } + lsof = resolved + } + output, err := exec.CommandContext(ctx, lsof, "-n", "-P", "-F", "pfan").Output() + if err != nil { + return nil, fmt.Errorf("run native writer probe: %w", err) + } + return parseEnrollmentWriterSnapshot(output, sessions), nil +} diff --git a/internal/cli/fs_enroll_writer_windows.go b/internal/cli/fs_enroll_writer_windows.go new file mode 100644 index 0000000..b4928ac --- /dev/null +++ b/internal/cli/fs_enroll_writer_windows.go @@ -0,0 +1,46 @@ +//go:build windows + +package cli + +import ( + "context" + "errors" + "fmt" + + "github.com/jstar0/codexfold/internal/codex" + "golang.org/x/sys/windows" +) + +func detectEnrollmentWriters(ctx context.Context, sessions []codex.Session) (map[string]bool, error) { + writers := make(map[string]bool) + for _, session := range sessions { + if err := ctx.Err(); err != nil { + return nil, err + } + path, err := windows.UTF16PtrFromString(session.RolloutPath) + if err != nil { + return nil, fmt.Errorf("encode rollout path for native handle probe: %w", err) + } + handle, err := windows.CreateFile( + path, + windows.GENERIC_READ|windows.GENERIC_WRITE, + 0, + nil, + windows.OPEN_EXISTING, + windows.FILE_ATTRIBUTE_NORMAL, + 0, + ) + if err == nil { + if closeErr := windows.CloseHandle(handle); closeErr != nil { + return nil, fmt.Errorf("close native rollout probe: %w", closeErr) + } + continue + } + if errors.Is(err, windows.ERROR_SHARING_VIOLATION) || errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + writers[session.ID] = true + continue + } + return nil, fmt.Errorf("probe native rollout handle %s: %w", session.ID, err) + } + return writers, nil +} diff --git a/internal/cli/fs_namespace.go b/internal/cli/fs_namespace.go index 6bf160b..ff42356 100644 --- a/internal/cli/fs_namespace.go +++ b/internal/cli/fs_namespace.go @@ -61,6 +61,11 @@ func newFSNamespaceActivateCommand() *cobra.Command { if err != nil { return err } + if apply { + if err := requireFilesystemActivationAllowed(options.Home); err != nil { + return err + } + } if !apply { result, err := sessionns.Inspect(options) if err != nil { diff --git a/internal/cli/fs_reconcile.go b/internal/cli/fs_reconcile.go index 4adda24..d56a3fb 100644 --- a/internal/cli/fs_reconcile.go +++ b/internal/cli/fs_reconcile.go @@ -28,7 +28,7 @@ func newFSRepairRolloutCommand() *cobra.Command { if !filepath.IsAbs(args[0]) || !filepath.IsAbs(outputPath) { return errors.New("source and --output paths must be absolute") } - result, err := reconcile.RepairWithOptions(args[0], outputPath, reconcile.RepairOptions{AllowOrphans: orphanPath != "", OrphanPath: orphanPath}) + result, err := reconcile.RepairWithOptions(args[0], outputPath, reconcile.RepairOptions{AllowOrphans: orphanPath != "", OrphanPath: orphanPath, Context: command.Context()}) if err != nil { return err } @@ -36,10 +36,14 @@ func newFSRepairRolloutCommand() *cobra.Command { return writeJSON(command, result) } _, err = fmt.Fprintf(command.OutOrStdout(), - "physical=%d invalid=%d reconstructed=%d orphans=%d output=%d regressions=%d max_buffer=%d path=%s sha256=%s\n", + "physical=%d invalid=%d reconstructed=%d conversations=%d preserved=%d reconstructed_conversations=%d conversation_verified=%t orphans=%d output=%d regressions=%d max_buffer=%d path=%s sha256=%s\n", result.PhysicalLines, result.InvalidPhysicalLines, result.ReconstructedRecords, + result.SourceConversationRecords, + result.PreservedConversationRecords, + result.ReconstructedConversationRecords, + result.ConversationIntegrityVerified, result.OrphanLines, result.OutputRecords, result.TimestampRegressions, @@ -74,7 +78,7 @@ func newFSReconcileRolloutCommand() *cobra.Command { if !filepath.IsAbs(outputPath) { return errors.New("--output must be absolute with --apply") } - result, err = reconcile.Merge(args[0], args[1], outputPath) + result, err = reconcile.MergeWithOptions(args[0], args[1], outputPath, reconcile.MergeOptions{Context: command.Context()}) } else { result, err = reconcile.Analyze(args[0], args[1]) } diff --git a/internal/cli/fs_service.go b/internal/cli/fs_service.go index 4e1c9f6..d1fdd8f 100644 --- a/internal/cli/fs_service.go +++ b/internal/cli/fs_service.go @@ -10,21 +10,24 @@ import ( "io" "os" "path/filepath" - "runtime" "strings" "sync" "syscall" "time" + "github.com/jstar0/codexfold/internal/buildid" "github.com/jstar0/codexfold/internal/codex" "github.com/jstar0/codexfold/internal/mountfs" "github.com/jstar0/codexfold/internal/service" + "github.com/jstar0/codexfold/internal/storage" "github.com/jstar0/codexfold/internal/vfs" "github.com/spf13/cobra" ) const serviceLabel = "com.codexfold.fs" +const nativeFSKitStartupTimeout = 45 * time.Second + type operationTrace struct { mu sync.Mutex file *os.File @@ -62,9 +65,31 @@ func (t *operationTrace) Close() error { } type FSServiceActionResult struct { - Action string `json:"action"` - Path string `json:"path,omitempty"` - DryRun bool `json:"dry_run"` + Action string `json:"action"` + Path string `json:"path,omitempty"` + SupervisorPath string `json:"supervisor_path,omitempty"` + DryRun bool `json:"dry_run"` +} + +type FSServiceInstallResult struct { + Path string `json:"path"` + DryRun bool `json:"dry_run"` + Bytes int `json:"bytes"` + SupervisorPath string `json:"supervisor_path,omitempty"` + SupervisorBytes int `json:"supervisor_bytes,omitempty"` + FSKitAppPath string `json:"fskit_app_path,omitempty"` + FSKitLauncherPath string `json:"fskit_launcher_path,omitempty"` + FSKitResourcePath string `json:"fskit_resource_path,omitempty"` + FSKitAppChanged bool `json:"fskit_app_changed,omitempty"` +} + +type FSServiceBinaryUpdateResult struct { + Candidate string `json:"candidate"` + Target string `json:"target"` + CurrentSHA256 string `json:"current_sha256"` + CandidateSHA256 string `json:"candidate_sha256"` + Changed bool `json:"changed"` + DryRun bool `json:"dry_run"` } type FSUpdatePreflightResult struct { @@ -75,27 +100,231 @@ type FSUpdatePreflightResult struct { } func newFSServiceCommand() *cobra.Command { - command := &cobra.Command{Use: "service", Short: "Manage the per-user transparent filesystem service"} + command := &cobra.Command{Use: "service", Short: "Manage the transparent filesystem service"} command.AddCommand(newFSServiceInstallCommand()) command.AddCommand(newFSServiceStartCommand()) command.AddCommand(newFSServiceStopCommand()) + command.AddCommand(newFSServiceRestartCommand()) command.AddCommand(newFSServiceStatusCommand()) + command.AddCommand(newFSServiceUpdateBinaryCommand()) command.AddCommand(newFSServiceUpdatePreflightCommand()) + addPlatformServiceCommands(command) + return command +} + +func newFSServiceUpdateBinaryCommand() *cobra.Command { + var codexHome, mountPoint, definitionPath string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "update-binary ", + Short: "Atomically replace, restart, verify, and roll back the service binary", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + candidate, err := filepath.Abs(args[0]) + if err != nil { + return err + } + definition, err := resolveServiceDefinitionPath(definitionPath) + if err != nil { + return err + } + platform, err := service.CurrentPlatform() + if err != nil { + return err + } + target, err := service.DefinitionBinary(platform, definition) + if err != nil { + return err + } + currentSHA256, err := buildid.FileSHA256(target) + if err != nil { + return err + } + candidateSHA256, err := buildid.FileSHA256(candidate) + if err != nil { + return err + } + result := FSServiceBinaryUpdateResult{ + Candidate: candidate, Target: target, CurrentSHA256: currentSHA256, + CandidateSHA256: candidateSHA256, Changed: currentSHA256 != candidateSHA256, DryRun: !apply, + } + if apply && result.Changed { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + if err := requireFilesystemActivationAllowed(home); err != nil { + return err + } + mount := defaultMountPoint(home, mountPoint) + update, err := service.StageBinaryUpdate(candidate, target) + if err != nil { + return err + } + if err := stopPlatformService(command.Context(), platform, definition); err != nil { + _ = update.Commit() + return fmt.Errorf("stop filesystem service before binary update: %w", err) + } + if err := update.Promote(); err != nil { + rollbackErr := update.Rollback() + var cleanupErr error + var restartErr error + if rollbackErr == nil { + cleanupErr = update.Commit() + restartErr = startPlatformService(command.Context(), platform, definition, mount) + } + return errors.Join(fmt.Errorf("promote filesystem service binary: %w", err), rollbackErr, cleanupErr, restartErr) + } + if err := startPlatformService(command.Context(), platform, definition, mount); err != nil { + _ = stopPlatformService(command.Context(), platform, definition) + rollbackErr := update.Rollback() + var restartErr error + if rollbackErr == nil { + restartErr = startPlatformService(command.Context(), platform, definition, mount) + _ = update.Commit() + } + return errors.Join(fmt.Errorf("start verified filesystem service binary: %w", err), rollbackErr, restartErr) + } + if err := update.Commit(); err != nil { + return fmt.Errorf("remove filesystem binary rollback artifact: %w", err) + } + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "dry_run=%t changed=%t target=%s current=%s candidate=%s\n", result.DryRun, result.Changed, result.Target, result.CurrentSHA256, result.CandidateSHA256) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + addServiceDefinitionFlags(command, &definitionPath) + command.Flags().BoolVar(&apply, "apply", false, "Stop, replace, restart, and verify the service binary") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") return command } func newFSServiceInstallCommand() *cobra.Command { var codexHome, storeDir, mountPoint, binaryPath, plistPath, logDir, nativeRoot, operationTracePath string - var apply, canonicalNamespace, jsonOutput bool + var frontend, fskitResource, fskitAppPath, fskitAppSource, label string + var enrollmentInterval, enrollmentStableFor time.Duration + var enrollmentBatchSize int + var apply, canonicalNamespace, enrollmentCanary, jsonOutput bool command := &cobra.Command{ Use: "install", - Short: "Render and optionally bootstrap a per-user launchd service", + Short: "Render and optionally start the native platform service", Args: cobra.NoArgs, - RunE: func(command *cobra.Command, _ []string) error { + RunE: func(command *cobra.Command, _ []string) (runErr error) { home, store, mount, binary, plist, logs, err := resolveServicePaths(codexHome, storeDir, mountPoint, binaryPath, plistPath, logDir) if err != nil { return err } + if apply { + if err := requireFilesystemActivationAllowed(home); err != nil { + return err + } + } + platform, err := service.CurrentPlatform() + if err != nil { + return err + } + if frontend == "" { + frontend = "fuse" + } + if label == "" { + label = serviceLabel + } + if label != serviceLabel && platform != service.PlatformLaunchd { + return errors.New("custom service labels are supported only for macOS LaunchAgents") + } + hadExistingDefinition := false + if apply { + if info, statErr := os.Stat(plist); statErr == nil { + if !info.Mode().IsRegular() { + return errors.New("installed service definition is not a regular file") + } + hadExistingDefinition = true + } else if !errors.Is(statErr, os.ErrNotExist) { + return statErr + } + } + if frontend == "native-fskit" { + if platform != service.PlatformLaunchd { + return errors.New("native-fskit service frontend is available only on macOS") + } + canonicalNamespace = true + userHome, err := os.UserHomeDir() + if err != nil { + return err + } + if fskitAppPath == "" { + fskitAppPath = service.DefaultFSKitAppPath(userHome) + } + fskitAppPath, err = filepath.Abs(fskitAppPath) + if err != nil { + return err + } + } + var appTransaction fsKitAppTransaction + var definitionUpdates []*service.DefinitionUpdate + rollbackInstall := false + restartPreviousService := false + defer func() { + if !rollbackInstall { + return + } + rollbackContext, cancel := context.WithTimeout(context.Background(), 2*nativeFSKitStartupTimeout+15*time.Second) + defer cancel() + runErr = errors.Join(runErr, rollbackFailedServiceInstall( + rollbackContext, platform, plist, mount, definitionUpdates, appTransaction, + restartPreviousService, stopPlatformService, startPlatformService, + )) + }() + if apply && frontend == "native-fskit" && hadExistingDefinition { + rollbackInstall = true + restartPreviousService = true + stopErr := stopPlatformService(command.Context(), platform, plist) + if err := waitPlatformServiceInactive(command.Context(), platform, plist, mount, 30*time.Second); err != nil { + return errors.Join(stopErr, err) + } + } + if apply && frontend == "native-fskit" { + if fskitAppSource != "" { + fskitAppSource, err = filepath.Abs(fskitAppSource) + if err != nil { + return err + } + } + appTransaction, err = prepareFSKitAppPlatform(command.Context(), fskitAppSource, fskitAppPath) + if err != nil { + return err + } + rollbackInstall = true + restartPreviousService = hadExistingDefinition && appTransaction.Changed() + if fskitResource == "" { + fskitResource = filepath.Join(appTransaction.AppGroupPath(), service.FSKitResourceDirectoryName) + } + } + if frontend == "native-fskit" && fskitResource == "" { + userHome, err := os.UserHomeDir() + if err != nil { + return err + } + fskitResource = service.DefaultFSKitResourcePath(userHome) + } + if frontend == "native-fskit" && apply { + within, err := filepath.Rel(appTransaction.AppGroupPath(), filepath.Clean(fskitResource)) + if err != nil || within == ".." || strings.HasPrefix(within, ".."+string(filepath.Separator)) { + return errors.New("native FSKit resource must remain inside the configured App Group") + } + } + launcherPath := "" + if frontend == "native-fskit" { + launcherPath, err = service.FSKitHostLauncherPath(fskitAppPath) + if err != nil { + return err + } + } if canonicalNamespace { if nativeRoot == "" { nativeRoot = filepath.Join(home, "fold-native") @@ -105,100 +334,288 @@ func newFSServiceInstallCommand() *cobra.Command { } nativeRoot = filepath.Clean(nativeRoot) } - definition, err := service.RenderLaunchd(service.Options{ - Label: serviceLabel, BinaryPath: binary, CodexHome: home, StoreDir: store, MountPoint: mount, + if enrollmentCanary { + userHome, err := os.UserHomeDir() + if err != nil { + return err + } + if err := validateCompatibilityCanary(home, filepath.Join(userHome, ".codex"), store, canonicalNamespace, compatibilityFlags{cliPath: "none", desktopPath: "none"}); err != nil { + return err + } + } + options := service.Options{ + Label: label, BinaryPath: binary, CodexHome: home, StoreDir: store, MountPoint: mount, StdoutPath: filepath.Join(logs, "stdout.log"), StderrPath: filepath.Join(logs, "stderr.log"), CanonicalNamespace: canonicalNamespace, NativeRoot: nativeRoot, OperationTrace: operationTracePath, - }) + EnrollmentInterval: enrollmentInterval, EnrollmentStableFor: enrollmentStableFor, + EnrollmentBatchSize: enrollmentBatchSize, EnrollmentCanary: enrollmentCanary, + Frontend: frontend, FSKitResource: fskitResource, LauncherPath: launcherPath, + } + definition, err := service.RenderDefinition(platform, options) if err != nil { return err } - if apply && (runtime.GOOS != "darwin" || !mountfs.Available()) { - return errors.New("service installation requires a FUSE-enabled macOS build and an authorized host prerequisite") + if apply && frontend == "fuse" && !mountfs.Available() { + return errors.New("service installation requires a platform FUSE build and an authorized host prerequisite") } if apply { if err := os.MkdirAll(logs, 0o700); err != nil { return err } } - result, err := service.WriteDefinition(plist, definition, apply) - if err != nil { + written := service.InstallResult{Path: plist, DryRun: !apply, Bytes: len(definition)} + if apply { + update, err := service.StageDefinitionUpdate(plist, definition) + if err != nil { + return err + } + definitionUpdates = append(definitionUpdates, update) + rollbackInstall = true + } else if _, err := service.WriteDefinition(plist, definition, false); err != nil { return err } - if apply { - manager := service.Manager{} - _ = manager.Bootout(command.Context(), plist) - if err := manager.Bootstrap(command.Context(), plist); err != nil { + result := FSServiceInstallResult{Path: written.Path, DryRun: written.DryRun, Bytes: written.Bytes} + if frontend == "native-fskit" { + result.FSKitAppPath = fskitAppPath + result.FSKitLauncherPath = launcherPath + result.FSKitResourcePath = fskitResource + if appTransaction != nil { + result.FSKitAppChanged = appTransaction.Changed() + } + supervisorDefinition, err := service.RenderLaunchdSupervisor(options) + if err != nil { return err } - if err := manager.Kickstart(command.Context(), serviceLabel); err != nil { + supervisorPath := nativeFSKitSupervisorDefinitionPath(plist) + if apply { + update, err := service.StageDefinitionUpdate(supervisorPath, supervisorDefinition) + if err != nil { + _ = commitDefinitionUpdates(definitionUpdates) + return err + } + definitionUpdates = append(definitionUpdates, update) + } else if _, err := service.WriteDefinition(supervisorPath, supervisorDefinition, false); err != nil { return err } - if _, err := manager.WaitHealthy(command.Context(), serviceLabel, mount, 15*time.Second); err != nil { - _ = manager.Bootout(command.Context(), plist) + result.SupervisorPath = supervisorPath + result.SupervisorBytes = len(supervisorDefinition) + } + if apply { + for _, update := range definitionUpdates { + if err := update.Promote(); err != nil { + return err + } + } + } + if apply { + restartPreviousService = restartPreviousService || hadExistingDefinition + if err := installPlatformService(command.Context(), platform, plist, binary, mount); err != nil { return err } } + rollbackInstall = false + cleanupErr := commitDefinitionUpdates(definitionUpdates) + if appTransaction != nil { + if err := appTransaction.Commit(); err != nil { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("remove FSKit app rollback artifacts: %w", err)) + } + } + if cleanupErr != nil { + return cleanupErr + } if jsonOutput { return writeJSON(command, result) } - _, err = fmt.Fprintf(command.OutOrStdout(), "dry_run=%t path=%s bytes=%d\n", result.DryRun, result.Path, result.Bytes) + _, err = fmt.Fprintf(command.OutOrStdout(), "dry_run=%t path=%s bytes=%d supervisor=%s\n", result.DryRun, result.Path, result.Bytes, result.SupervisorPath) return err }, } addServicePathFlags(command, &codexHome, &storeDir, &mountPoint, &binaryPath, &plistPath, &logDir) command.Flags().BoolVar(&canonicalNamespace, "canonical-namespace", false, "Start the service with the canonical Codex session namespace") + command.Flags().StringVar(&frontend, "frontend", "fuse", "Filesystem frontend: fuse or native-fskit") + command.Flags().StringVar(&label, "label", serviceLabel, "Service label; non-default labels are intended for isolated macOS validation") + command.Flags().StringVar(&fskitResource, "fskit-resource", "", "Native FSKit resource inside the App Group; defaults to /native-fskit") + command.Flags().StringVar(&fskitAppPath, "fskit-app", "", "Installed signed FSKit app; defaults to ~/Applications/CodexFoldFSKit.app") + command.Flags().StringVar(&fskitAppSource, "fskit-app-source", "", "Signed FSKit app candidate to atomically install or update at --fskit-app") command.Flags().StringVar(&nativeRoot, "native-root", "", "Canonical native backing root; defaults to /fold-native") - command.Flags().StringVar(&operationTracePath, "operation-trace", "", "Absolute path for sanitized FUSE operation names") - command.Flags().BoolVar(&apply, "apply", false, "Write, bootstrap, and start the per-user service") + command.Flags().StringVar(&operationTracePath, "operation-trace", "", "Absolute path for sanitized filesystem operation names") + command.Flags().DurationVar(&enrollmentInterval, "enrollment-interval", 0, "Periodic stable-session enrollment interval; zero disables the loop") + command.Flags().DurationVar(&enrollmentStableFor, "enrollment-stable-for", time.Hour, "Required unchanged interval before periodic enrollment") + command.Flags().IntVar(&enrollmentBatchSize, "enrollment-batch-size", 1, "Maximum sessions enrolled per periodic cycle") + command.Flags().BoolVar(&enrollmentCanary, "enrollment-canary", false, "Enable periodic apply only for an explicitly isolated Codex home while capability remains preview") + command.Flags().BoolVar(&apply, "apply", false, "Write, install, and start the native platform service") command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") return command } -func newFSServiceStartCommand() *cobra.Command { - return newFSServiceLifecycleCommand("start", true, func(ctx context.Context, manager service.Manager, plist string) error { - _ = manager.Bootout(ctx, plist) - if err := manager.Bootstrap(ctx, plist); err != nil { +func rollbackDefinitionUpdates(updates []*service.DefinitionUpdate) error { + var result error + for index := len(updates) - 1; index >= 0; index-- { + result = errors.Join(result, updates[index].Rollback()) + } + return errors.Join(result, commitDefinitionUpdates(updates)) +} + +func commitDefinitionUpdates(updates []*service.DefinitionUpdate) error { + var result error + for _, update := range updates { + result = errors.Join(result, update.Commit()) + } + return result +} + +type stopServiceOperation func(context.Context, service.Platform, string) error +type startServiceOperation func(context.Context, service.Platform, string, string) error + +func rollbackFailedServiceInstall( + ctx context.Context, + platform service.Platform, + definitionPath string, + mountPoint string, + definitionUpdates []*service.DefinitionUpdate, + appTransaction fsKitAppTransaction, + restartPreviousService bool, + stop stopServiceOperation, + start startServiceOperation, +) error { + if restartPreviousService && stop != nil { + // A failed start normally booted the jobs out already. This extra stop is + // best effort so rollback can also recover failures before the health gate. + _ = stop(ctx, platform, definitionPath) + } + result := rollbackDefinitionUpdates(definitionUpdates) + if appTransaction != nil { + result = errors.Join(result, appTransaction.Rollback(ctx)) + } + if restartPreviousService { + if start == nil { + result = errors.Join(result, errors.New("service rollback restart operation is unavailable")) + } else { + result = errors.Join(result, start(ctx, platform, definitionPath, mountPoint)) + } + } + return result +} + +func waitPlatformServiceInactive(ctx context.Context, platform service.Platform, definitionPath string, mountPoint string, timeout time.Duration) error { + if timeout <= 0 { + timeout = 30 * time.Second + } + var lockPaths nativeFSKitProcessLockPaths + checkLocks := false + if platform == service.PlatformLaunchd { + frontend, err := service.DefinitionFrontend(platform, definitionPath) + if err != nil { return err } - return manager.Kickstart(ctx, serviceLabel) - }) + if frontend == "native-fskit" { + lockPaths, err = nativeFSKitLaunchdLockPaths(definitionPath) + if err != nil { + return err + } + checkLocks = true + } + } + deadline := time.NewTimer(timeout) + defer deadline.Stop() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + var daemonLock, supervisorLock service.ProcessLockStatus + for { + status, err := platformServiceStatus(ctx, platform, mountPoint, definitionPath) + if err != nil { + return err + } + if checkLocks { + daemonLock, err = service.InspectProcessLock(lockPaths.daemon) + if err != nil { + return fmt.Errorf("inspect daemon process lock: %w", err) + } + supervisorLock, err = service.InspectProcessLock(lockPaths.supervisor) + if err != nil { + return fmt.Errorf("inspect supervisor process lock: %w", err) + } + } + if !status.DaemonRunning && !status.SupervisorRunning && !status.MountHealthy && !daemonLock.Held && !supervisorLock.Held { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-deadline.C: + return fmt.Errorf( + "previous filesystem service did not stop cleanly: daemon=%t supervisor=%t mount=%t daemon_lock=%t daemon_lock_pid=%d supervisor_lock=%t supervisor_lock_pid=%d", + status.DaemonRunning, status.SupervisorRunning, status.MountHealthy, + daemonLock.Held, daemonLock.PID, supervisorLock.Held, supervisorLock.PID, + ) + case <-ticker.C: + } + } +} + +func newFSServiceStartCommand() *cobra.Command { + return newFSServiceLifecycleCommand("start", true) } func newFSServiceStopCommand() *cobra.Command { - return newFSServiceLifecycleCommand("stop", false, func(ctx context.Context, manager service.Manager, plist string) error { - return manager.Bootout(ctx, plist) - }) + return newFSServiceLifecycleCommand("stop", false) +} + +func newFSServiceRestartCommand() *cobra.Command { + return newFSServiceLifecycleCommand("restart", true) } -func newFSServiceLifecycleCommand(action string, waitForMount bool, run func(context.Context, service.Manager, string) error) *cobra.Command { +func newFSServiceLifecycleCommand(action string, waitForMount bool) *cobra.Command { var plistPath, codexHome, mountPoint string var apply, jsonOutput bool command := &cobra.Command{ Use: action, - Short: action + " the per-user filesystem service", + Short: action + " the filesystem service", Args: cobra.NoArgs, RunE: func(command *cobra.Command, _ []string) error { - plist, err := resolvePlistPath(plistPath) + definition, err := resolveServiceDefinitionPath(plistPath) if err != nil { return err } - result := FSServiceActionResult{Action: action, Path: plist, DryRun: !apply} - if apply { - if runtime.GOOS != "darwin" { - return errors.New("launchd service lifecycle is available only on macOS") + var home string + if apply && (action == "start" || action == "restart") { + home, err = codex.ResolveHome(codexHome) + if err != nil { + return err } - manager := service.Manager{} - if err := run(command.Context(), manager, plist); err != nil { + if err := requireFilesystemActivationAllowed(home); err != nil { return err } - if waitForMount { - home, err := codex.ResolveHome(codexHome) - if err != nil { + } + platform, err := service.CurrentPlatform() + if err != nil { + return err + } + result := FSServiceActionResult{Action: action, Path: definition, DryRun: !apply} + if apply { + frontend, err := service.DefinitionFrontend(platform, definition) + if err != nil { + return err + } + if frontend == "native-fskit" { + result.SupervisorPath = nativeFSKitSupervisorDefinitionPath(definition) + } + if waitForMount && frontend == "fuse" && !mountfs.Available() { + return errors.New("service start requires a platform FUSE build and an authorized host prerequisite") + } + if action == "start" { + if err := startPlatformService(command.Context(), platform, definition, defaultMountPoint(home, mountPoint)); err != nil { return err } - if _, err := manager.WaitHealthy(command.Context(), serviceLabel, defaultMountPoint(home, mountPoint), 15*time.Second); err != nil { - _ = manager.Bootout(command.Context(), plist) + } else if action == "restart" { + if err := stopPlatformService(command.Context(), platform, definition); err != nil { + return err + } + if err := startPlatformService(command.Context(), platform, definition, defaultMountPoint(home, mountPoint)); err != nil { + return err + } + } else { + if err := stopPlatformService(command.Context(), platform, definition); err != nil { return err } } @@ -206,22 +623,22 @@ func newFSServiceLifecycleCommand(action string, waitForMount bool, run func(con if jsonOutput { return writeJSON(command, result) } - _, err = fmt.Fprintf(command.OutOrStdout(), "action=%s dry_run=%t path=%s\n", action, result.DryRun, plist) + _, err = fmt.Fprintf(command.OutOrStdout(), "action=%s dry_run=%t path=%s supervisor=%s\n", action, result.DryRun, definition, result.SupervisorPath) return err }, } - command.Flags().StringVar(&plistPath, "plist", "", "LaunchAgent plist path") + addServiceDefinitionFlags(command, &plistPath) if waitForMount { command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") } - command.Flags().BoolVar(&apply, "apply", false, "Execute the launchctl action") + command.Flags().BoolVar(&apply, "apply", false, "Execute the native service-manager action") command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") return command } func newFSServiceStatusCommand() *cobra.Command { - var codexHome, mountPoint string + var codexHome, mountPoint, definitionPath string var jsonOutput bool command := &cobra.Command{ Use: "status", @@ -232,20 +649,329 @@ func newFSServiceStatusCommand() *cobra.Command { if err != nil { return err } - status := service.Manager{}.Status(command.Context(), serviceLabel, defaultMountPoint(home, mountPoint)) + platform, err := service.CurrentPlatform() + if err != nil { + return err + } + definition, err := resolveServiceDefinitionPath(definitionPath) + if err != nil { + return err + } + status, err := platformServiceStatus(command.Context(), platform, defaultMountPoint(home, mountPoint), definition) + if err != nil { + return err + } if jsonOutput { return writeJSON(command, status) } - _, err = fmt.Fprintf(command.OutOrStdout(), "daemon=%t mount=%t daemon_error=%q mount_error=%q\n", status.DaemonRunning, status.MountHealthy, status.DaemonError, status.MountError) + _, err = fmt.Fprintf(command.OutOrStdout(), "daemon=%t supervisor=%t mount=%t build=%t running_build=%s disk_build=%s binary=%s daemon_error=%q supervisor_error=%q mount_error=%q build_error=%q\n", status.DaemonRunning, status.SupervisorRunning, status.MountHealthy, status.Build.Healthy, status.Build.RunningBuildSHA256, status.Build.ConfiguredBuildSHA256, status.Build.ConfiguredBinaryPath, status.DaemonError, status.SupervisorError, status.MountError, status.Build.Error) return err }, } command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + addServiceDefinitionFlags(command, &definitionPath) command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") return command } +func installPlatformService(ctx context.Context, platform service.Platform, definitionPath string, binaryPath string, mountPoint string) error { + label, err := service.DefinitionLabel(platform, definitionPath) + if err != nil { + return err + } + if platform == service.PlatformWindows { + manager := service.WindowsManager{} + _ = manager.Stop(ctx, label) + if err := manager.Install(ctx, label, binaryPath, definitionPath); err != nil { + return err + } + } + return startPlatformService(ctx, platform, definitionPath, mountPoint) +} + +func startPlatformService(ctx context.Context, platform service.Platform, definitionPath string, mountPoint string) error { + frontend, err := service.DefinitionFrontend(platform, definitionPath) + if err != nil { + return err + } + label, err := service.DefinitionLabel(platform, definitionPath) + if err != nil { + return err + } + switch platform { + case service.PlatformLaunchd: + manager := service.Manager{} + if frontend == "native-fskit" { + supervisorLabel := nativeFSKitSupervisorLabel(label) + supervisorDefinition := nativeFSKitSupervisorDefinitionPath(definitionPath) + _ = manager.Bootout(ctx, supervisorDefinition) + _ = manager.Bootout(ctx, definitionPath) + if err := manager.Enable(ctx, label); err != nil { + return err + } + if err := manager.Enable(ctx, supervisorLabel); err != nil { + return err + } + if err := manager.Bootstrap(ctx, definitionPath); err != nil { + return err + } + if err := manager.Kickstart(ctx, label); err != nil { + _ = manager.Bootout(ctx, definitionPath) + return err + } + if err := manager.Bootstrap(ctx, supervisorDefinition); err != nil { + _ = manager.Bootout(ctx, definitionPath) + return err + } + if err := manager.Kickstart(ctx, supervisorLabel); err != nil { + _ = manager.Bootout(ctx, supervisorDefinition) + _ = manager.Bootout(ctx, definitionPath) + return err + } + if err := waitLaunchdNativeFSKitHealthy(ctx, manager, label, definitionPath, mountPoint, nativeFSKitStartupTimeout); err != nil { + _ = manager.Bootout(ctx, supervisorDefinition) + _ = manager.Bootout(ctx, definitionPath) + return err + } + if err := verifyServiceBuild(service.PlatformLaunchd, definitionPath, mountPoint); err != nil { + _ = manager.Bootout(ctx, supervisorDefinition) + _ = manager.Bootout(ctx, definitionPath) + return err + } + return nil + } + _ = manager.Bootout(ctx, definitionPath) + if err := manager.Enable(ctx, label); err != nil { + return err + } + if err := manager.Bootstrap(ctx, definitionPath); err != nil { + return err + } + if err := manager.Kickstart(ctx, label); err != nil { + _ = manager.Bootout(ctx, definitionPath) + return err + } + if _, err := manager.WaitHealthy(ctx, label, mountPoint, 15*time.Second); err != nil { + _ = manager.Bootout(ctx, definitionPath) + return err + } + if err := verifyServiceBuild(service.PlatformLaunchd, definitionPath, mountPoint); err != nil { + _ = manager.Bootout(ctx, definitionPath) + return err + } + return nil + case service.PlatformSystemd: + unit, err := systemdServiceUnit(definitionPath, label) + if err != nil { + return err + } + manager := service.SystemdManager{} + _ = manager.Stop(ctx, unit) + if err := manager.Start(ctx, unit); err != nil { + return err + } + if _, err := manager.WaitHealthy(ctx, unit, mountPoint, 15*time.Second); err != nil { + _ = manager.Stop(ctx, unit) + return err + } + if err := verifyServiceBuild(service.PlatformSystemd, definitionPath, mountPoint); err != nil { + _ = manager.Stop(ctx, unit) + return err + } + return nil + case service.PlatformWindows: + manager := service.WindowsManager{} + _ = manager.Stop(ctx, label) + if err := manager.Start(ctx, label); err != nil { + return err + } + if _, err := manager.WaitHealthy(ctx, label, mountPoint, 15*time.Second); err != nil { + _ = manager.Stop(ctx, label) + return err + } + if err := verifyServiceBuild(service.PlatformWindows, definitionPath, mountPoint); err != nil { + _ = manager.Stop(ctx, label) + return err + } + return nil + default: + return errors.New("unknown service platform") + } +} + +func stopPlatformService(ctx context.Context, platform service.Platform, definitionPath string) error { + frontend, err := service.DefinitionFrontend(platform, definitionPath) + if err != nil { + return err + } + label, err := service.DefinitionLabel(platform, definitionPath) + if err != nil { + return err + } + switch platform { + case service.PlatformLaunchd: + if frontend == "native-fskit" { + manager := service.Manager{} + supervisorErr := manager.Bootout(ctx, nativeFSKitSupervisorDefinitionPath(definitionPath)) + daemonErr := manager.Bootout(ctx, definitionPath) + return errors.Join(supervisorErr, daemonErr) + } + return (service.Manager{}).Bootout(ctx, definitionPath) + case service.PlatformSystemd: + unit, err := systemdServiceUnit(definitionPath, label) + if err != nil { + return err + } + return (service.SystemdManager{}).Stop(ctx, unit) + case service.PlatformWindows: + return (service.WindowsManager{}).Stop(ctx, label) + default: + return errors.New("unknown service platform") + } +} + +func verifyServiceBuild(platform service.Platform, definitionPath string, mountPoint string) error { + status := service.InspectBuild(platform, definitionPath, mountPoint) + if !status.Healthy { + return fmt.Errorf("filesystem service build verification failed: %s", status.Error) + } + return nil +} + +func platformServiceStatus(ctx context.Context, platform service.Platform, mountPoint string, definitionPath string) (service.Status, error) { + frontend, err := service.DefinitionFrontend(platform, definitionPath) + if err != nil { + return service.Status{}, err + } + label, err := service.DefinitionLabel(platform, definitionPath) + if err != nil { + return service.Status{}, err + } + var status service.Status + switch platform { + case service.PlatformLaunchd: + manager := service.Manager{} + status = manager.Status(ctx, label, mountPoint) + if frontend == "native-fskit" { + lockPaths, err := nativeFSKitLaunchdLockPaths(definitionPath) + if err != nil { + return service.Status{}, err + } + status = validateLaunchdChildProcess(status, lockPaths.daemon, "daemon") + supervisor := validateLaunchdChildProcess( + manager.Status(ctx, nativeFSKitSupervisorLabel(label), mountPoint), + lockPaths.supervisor, + "supervisor", + ) + status.SupervisorRunning = supervisor.DaemonRunning + status.SupervisorPID = supervisor.DaemonPID + status.SupervisorError = supervisor.DaemonError + } + case service.PlatformSystemd: + unit, err := service.SystemdUnitName(label) + if err != nil { + return service.Status{}, err + } + status = (service.SystemdManager{}).Status(ctx, unit, mountPoint) + case service.PlatformWindows: + status = (service.WindowsManager{}).Status(ctx, label, mountPoint) + default: + return service.Status{}, errors.New("unknown service platform") + } + status.Build = service.InspectBuild(platform, definitionPath, mountPoint) + return status, nil +} + +type nativeFSKitProcessLockPaths struct { + daemon string + supervisor string +} + +func nativeFSKitLaunchdLockPaths(definitionPath string) (nativeFSKitProcessLockPaths, error) { + store, err := service.DefinitionStore(service.PlatformLaunchd, definitionPath) + if err != nil { + return nativeFSKitProcessLockPaths{}, err + } + resource, err := service.DefinitionFSKitResource(service.PlatformLaunchd, definitionPath) + if err != nil { + return nativeFSKitProcessLockPaths{}, err + } + return nativeFSKitProcessLockPaths{ + daemon: filepath.Join(store, "fs", "service.lock"), + supervisor: filepath.Join(resource, service.NativeFSKitSupervisorLockName), + }, nil +} + +func validateLaunchdChildProcess(status service.Status, lockPath string, role string) service.Status { + if !status.DaemonRunning { + return status + } + lockStatus, err := service.InspectProcessLock(lockPath) + if err != nil { + status.DaemonRunning = false + status.DaemonError = fmt.Sprintf("inspect %s process lock: %v", role, err) + return status + } + if !lockStatus.Held { + status.DaemonRunning = false + status.DaemonError = fmt.Sprintf("%s host is running without an active child process lock", role) + return status + } + parentPID, err := service.ProcessParentPID(lockStatus.PID) + if err != nil { + status.DaemonRunning = false + status.DaemonError = fmt.Sprintf("inspect %s child process %d: %v", role, lockStatus.PID, err) + return status + } + if parentPID != status.DaemonPID { + status.DaemonRunning = false + status.DaemonError = fmt.Sprintf("%s process lock owner %d belongs to host %d, not launchd host %d", role, lockStatus.PID, parentPID, status.DaemonPID) + } + return status +} + +func waitLaunchdNativeFSKitHealthy(ctx context.Context, manager service.Manager, label string, definitionPath string, mountPoint string, timeout time.Duration) error { + if timeout <= 0 { + timeout = 15 * time.Second + } + lockPaths, err := nativeFSKitLaunchdLockPaths(definitionPath) + if err != nil { + return err + } + deadline := time.NewTimer(timeout) + defer deadline.Stop() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + var daemon, supervisor service.Status + supervisorLabel := nativeFSKitSupervisorLabel(label) + for { + daemon = validateLaunchdChildProcess(manager.Status(ctx, label, mountPoint), lockPaths.daemon, "daemon") + supervisor = validateLaunchdChildProcess(manager.Status(ctx, supervisorLabel, mountPoint), lockPaths.supervisor, "supervisor") + if daemon.DaemonRunning && supervisor.DaemonRunning && daemon.MountHealthy { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-deadline.C: + return fmt.Errorf("native FSKit service did not become healthy: daemon=%t supervisor=%t mount=%t daemon_error=%q supervisor_error=%q mount_error=%q", daemon.DaemonRunning, supervisor.DaemonRunning, daemon.MountHealthy, daemon.DaemonError, supervisor.DaemonError, daemon.MountError) + case <-ticker.C: + } + } +} + +func systemdServiceUnit(definitionPath string, label string) (string, error) { + unit, err := service.SystemdUnitName(label) + if err != nil { + return "", err + } + if filepath.Base(definitionPath) != unit { + return "", fmt.Errorf("systemd definition filename must be %s", unit) + } + return unit, nil +} + func newFSServiceUpdatePreflightCommand() *cobra.Command { var codexHome, storeDir string var compatibility compatibilityFlags @@ -428,7 +1154,7 @@ func restoreManagedState(store string, sessionID string, retiredPath string) err return nil } -func retainCanonicalSnapshot(store string, sessionID string, source vfs.NativeFile) (vfs.NativeFile, error) { +func retainCanonicalSnapshot(ctx context.Context, store string, sessionID string, source vfs.NativeFile, budget storage.Checker) (vfs.NativeFile, error) { if store == "" || !validSessionID(sessionID) || source.Path == "" { return vfs.NativeFile{}, errors.New("store, session ID, and source snapshot are required") } @@ -440,6 +1166,16 @@ func retainCanonicalSnapshot(store string, sessionID string, source vfs.NativeFi if verified.Bytes != source.Bytes || verified.SHA256 != source.SHA256 { return vfs.NativeFile{}, errors.New("canonical native snapshot changed during migration") } + if budget == nil { + guard, err := storage.DefaultGuard(store) + if err != nil { + return vfs.NativeFile{}, err + } + budget = guard + } + if _, err := budget.Check(ctx, storage.Projection{Operation: "retain-migration-snapshot", AdditionalPersistentBytes: source.Bytes}); err != nil { + return vfs.NativeFile{}, err + } retainedDir := filepath.Join(filepath.Clean(store), "fs", "snapshots", sessionID) retainedPath := filepath.Join(retainedDir, "native.jsonl") if err := os.MkdirAll(retainedDir, 0o700); err != nil { @@ -891,10 +1627,15 @@ func addServicePathFlags(command *cobra.Command, codexHome, storeDir, mountPoint command.Flags().StringVar(storeDir, "store", "", "Fold store directory; defaults to /fold-store") command.Flags().StringVar(mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") command.Flags().StringVar(binaryPath, "binary", "", "Absolute CodexFold binary path; defaults to the current executable") - command.Flags().StringVar(plistPath, "plist", "", "LaunchAgent plist path") + addServiceDefinitionFlags(command, plistPath) command.Flags().StringVar(logDir, "log-dir", "", "Service log directory; defaults to /service/logs") } +func addServiceDefinitionFlags(command *cobra.Command, definitionPath *string) { + command.Flags().StringVar(definitionPath, "definition", "", "Native service definition path") + command.Flags().StringVar(definitionPath, "plist", "", "LaunchAgent plist path (macOS compatibility alias)") +} + func resolveServicePaths(codexHome, storeDir, mountPoint, binaryPath, plistPath, logDir string) (string, string, string, string, string, string, error) { home, err := codex.ResolveHome(codexHome) if err != nil { @@ -913,7 +1654,7 @@ func resolveServicePaths(codexHome, storeDir, mountPoint, binaryPath, plistPath, if err != nil { return "", "", "", "", "", "", err } - plist, err := resolvePlistPath(plistPath) + plist, err := resolveServiceDefinitionPath(plistPath) if err != nil { return "", "", "", "", "", "", err } @@ -938,3 +1679,53 @@ func resolvePlistPath(explicit string) (string, error) { } return filepath.Join(home, "Library", "LaunchAgents", serviceLabel+".plist"), nil } + +func nativeFSKitSupervisorDefinitionPath(definitionPath string) string { + definitionPath = filepath.Clean(definitionPath) + extension := filepath.Ext(definitionPath) + base := strings.TrimSuffix(filepath.Base(definitionPath), extension) + if extension == "" { + extension = ".plist" + } + return filepath.Join(filepath.Dir(definitionPath), base+".supervisor"+extension) +} + +func nativeFSKitSupervisorLabel(label string) string { + return label + ".supervisor" +} + +func resolveServiceDefinitionPath(explicit string) (string, error) { + if explicit != "" { + return filepath.Abs(explicit) + } + platform, err := service.CurrentPlatform() + if err != nil { + return "", err + } + switch platform { + case service.PlatformLaunchd: + return resolvePlistPath("") + case service.PlatformSystemd: + configHome := os.Getenv("XDG_CONFIG_HOME") + if !filepath.IsAbs(configHome) { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + configHome = filepath.Join(home, ".config") + } + unit, err := service.SystemdUnitName(serviceLabel) + if err != nil { + return "", err + } + return filepath.Join(configHome, "systemd", "user", unit), nil + case service.PlatformWindows: + programData := os.Getenv("ProgramData") + if programData == "" { + return "", errors.New("ProgramData is not set") + } + return filepath.Join(programData, "CodexFold", "service.json"), nil + default: + return "", errors.New("unknown service platform") + } +} diff --git a/internal/cli/fs_service_fskit.go b/internal/cli/fs_service_fskit.go new file mode 100644 index 0000000..3acc8be --- /dev/null +++ b/internal/cli/fs_service_fskit.go @@ -0,0 +1,10 @@ +package cli + +import "context" + +type fsKitAppTransaction interface { + AppGroupPath() string + Changed() bool + Rollback(context.Context) error + Commit() error +} diff --git a/internal/cli/fs_service_fskit_darwin.go b/internal/cli/fs_service_fskit_darwin.go new file mode 100644 index 0000000..434a38b --- /dev/null +++ b/internal/cli/fs_service_fskit_darwin.go @@ -0,0 +1,388 @@ +//go:build darwin + +package cli + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/jstar0/codexfold/internal/service" +) + +const launchServicesRegister = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister" + +type darwinFSKitAppTransaction struct { + target string + source string + stageRoot string + backupRoot string + backupPath string + appGroupPath string + changed bool + hadTarget bool +} + +func prepareFSKitAppPlatform(ctx context.Context, source string, target string) (fsKitAppTransaction, error) { + if !filepath.IsAbs(target) { + return nil, errors.New("installed FSKit app path must be absolute") + } + target = filepath.Clean(target) + if source == "" { + appGroup, err := validateAndEnableFSKitApp(ctx, target) + if err != nil { + return nil, err + } + return &darwinFSKitAppTransaction{target: target, appGroupPath: appGroup}, nil + } + if !filepath.IsAbs(source) { + return nil, errors.New("FSKit app source path must be absolute") + } + source = filepath.Clean(source) + if source == target { + appGroup, err := validateAndEnableFSKitApp(ctx, target) + if err != nil { + return nil, err + } + return &darwinFSKitAppTransaction{target: target, appGroupPath: appGroup}, nil + } + if err := validateFSKitApp(ctx, source); err != nil { + return nil, fmt.Errorf("validate FSKit app source: %w", err) + } + sourceDigest, err := hashAppBundle(source) + if err != nil { + return nil, err + } + if targetDigest, targetErr := hashAppBundle(target); targetErr == nil && targetDigest == sourceDigest { + unregisterFSKitApp(ctx, source) + appGroup, err := validateAndEnableFSKitApp(ctx, target) + if err != nil { + return nil, err + } + return &darwinFSKitAppTransaction{target: target, appGroupPath: appGroup}, nil + } else if targetErr != nil && !errors.Is(targetErr, os.ErrNotExist) { + return nil, targetErr + } + + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + return nil, err + } + stageRoot, err := os.MkdirTemp(filepath.Dir(target), ".codexfold-fskit-stage-*") + if err != nil { + return nil, err + } + transaction := &darwinFSKitAppTransaction{target: target, source: source, stageRoot: stageRoot, changed: true} + stagePath := filepath.Join(stageRoot, filepath.Base(target)) + if output, err := exec.CommandContext(ctx, "/usr/bin/ditto", source, stagePath).CombinedOutput(); err != nil { + _ = transaction.Commit() + return nil, commandOutputError("stage FSKit app", output, err) + } + if err := validateFSKitApp(ctx, stagePath); err != nil { + _ = transaction.Commit() + return nil, fmt.Errorf("validate staged FSKit app: %w", err) + } + if stagedDigest, err := hashAppBundle(stagePath); err != nil || stagedDigest != sourceDigest { + _ = transaction.Commit() + if err != nil { + return nil, err + } + return nil, errors.New("staged FSKit app does not match the source bundle") + } + if _, err := os.Stat(target); err == nil { + transaction.hadTarget = true + transaction.backupRoot, err = os.MkdirTemp(filepath.Dir(target), ".codexfold-fskit-backup-*") + if err != nil { + _ = transaction.Commit() + return nil, err + } + transaction.backupPath = filepath.Join(transaction.backupRoot, filepath.Base(target)) + if err := os.Rename(target, transaction.backupPath); err != nil { + _ = transaction.Commit() + return nil, err + } + } else if !errors.Is(err, os.ErrNotExist) { + _ = transaction.Commit() + return nil, err + } + if err := os.Rename(stagePath, target); err != nil { + rollbackErr := transaction.Rollback(ctx) + return nil, errors.Join(err, rollbackErr) + } + if err := syncDirectory(filepath.Dir(target)); err != nil { + rollbackErr := transaction.Rollback(ctx) + return nil, errors.Join(err, rollbackErr) + } + unregisterFSKitApp(ctx, source) + appGroup, err := validateAndEnableFSKitApp(ctx, target) + if err != nil { + rollbackErr := transaction.Rollback(ctx) + return nil, errors.Join(err, rollbackErr) + } + transaction.appGroupPath = appGroup + return transaction, nil +} + +func unregisterFSKitApp(ctx context.Context, appPath string) { + if module, err := service.FSKitModulePath(appPath); err == nil { + _, _ = exec.CommandContext(ctx, "/usr/bin/pluginkit", "-r", module).CombinedOutput() + } + _, _ = exec.CommandContext(ctx, launchServicesRegister, "-u", appPath).CombinedOutput() +} + +func (t *darwinFSKitAppTransaction) AppGroupPath() string { return t.appGroupPath } +func (t *darwinFSKitAppTransaction) Changed() bool { return t.changed } + +func (t *darwinFSKitAppTransaction) Rollback(ctx context.Context) error { + if t == nil || !t.changed { + return nil + } + var result error + if t.source != "" { + unregisterFSKitApp(ctx, t.source) + } + _, _ = exec.CommandContext(ctx, launchServicesRegister, "-u", t.target).CombinedOutput() + if err := os.RemoveAll(t.target); err != nil { + result = errors.Join(result, err) + } + if t.hadTarget && t.backupPath != "" { + if err := os.Rename(t.backupPath, t.target); err != nil { + result = errors.Join(result, err) + } else if _, err := validateAndEnableFSKitApp(ctx, t.target); err != nil { + result = errors.Join(result, err) + } + } + result = errors.Join(result, syncDirectory(filepath.Dir(t.target))) + t.changed = false + return errors.Join(result, t.Commit()) +} + +func (t *darwinFSKitAppTransaction) Commit() error { + if t == nil { + return nil + } + var result error + for _, path := range []string{t.stageRoot, t.backupRoot} { + if path == "" { + continue + } + if err := os.RemoveAll(path); err != nil && !errors.Is(err, os.ErrNotExist) { + result = errors.Join(result, err) + } + } + t.stageRoot = "" + t.backupRoot = "" + t.backupPath = "" + return errors.Join(result, syncDirectory(filepath.Dir(t.target))) +} + +func validateAndEnableFSKitApp(ctx context.Context, appPath string) (string, error) { + if err := validateFSKitApp(ctx, appPath); err != nil { + return "", err + } + if output, err := exec.CommandContext(ctx, launchServicesRegister, "-f", "-R", "-trusted", appPath).CombinedOutput(); err != nil { + return "", commandOutputError("register FSKit app", output, err) + } + if output, err := exec.CommandContext(ctx, "/usr/bin/pluginkit", "-e", "use", "-p", "com.apple.fskit.fsmodule", "-i", service.FSKitModuleIdentifier).CombinedOutput(); err != nil { + return "", commandOutputError("enable FSKit extension election", output, err) + } + if _, err := ensureFSKitModuleEnabled(service.FSKitModuleIdentifier); err != nil { + return "", err + } + // FSKit may retain the previous extension endpoint across a same-bundle-ID + // app replacement. Stop the agent first so it cannot immediately respawn the + // stale module while the LaunchServices and preferences caches are refreshed. + killUserProcess("fskit_agent") + killUserProcess("CodexFoldFSKitModule") + killUserProcess("cfprefsd") + time.Sleep(time.Second) + launcher, err := service.FSKitHostLauncherPath(appPath) + if err != nil { + return "", err + } + output, err := exec.CommandContext(ctx, launcher, "--app-group-path").CombinedOutput() + if err != nil { + return "", commandOutputError("resolve FSKit App Group path", output, err) + } + appGroup := filepath.Clean(strings.TrimSpace(string(output))) + if !filepath.IsAbs(appGroup) || filepath.Base(appGroup) != service.FSKitAppGroupIdentifier { + return "", fmt.Errorf("FSKit host returned invalid App Group path %q", appGroup) + } + return appGroup, nil +} + +func validateFSKitApp(ctx context.Context, appPath string) error { + launcher, err := service.FSKitHostLauncherPath(appPath) + if err != nil { + return err + } + module, err := service.FSKitModulePath(appPath) + if err != nil { + return err + } + if info, err := os.Stat(launcher); err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 { + if err != nil { + return err + } + return errors.New("FSKit host launcher is not executable") + } + if info, err := os.Stat(module); err != nil || !info.IsDir() { + if err != nil { + return err + } + return errors.New("FSKit module bundle is missing") + } + checks := []struct { + path string + key string + want string + }{ + {filepath.Join(appPath, "Contents", "Info.plist"), "CFBundleIdentifier", service.FSKitHostBundleIdentifier}, + {filepath.Join(module, "Contents", "Info.plist"), "CFBundleIdentifier", service.FSKitModuleIdentifier}, + } + for _, check := range checks { + output, err := exec.CommandContext(ctx, "/usr/bin/plutil", "-extract", check.key, "raw", check.path).CombinedOutput() + if err != nil { + return commandOutputError("read FSKit bundle identity", output, err) + } + if strings.TrimSpace(string(output)) != check.want { + return fmt.Errorf("FSKit bundle identifier=%q expected=%q", strings.TrimSpace(string(output)), check.want) + } + } + if output, err := exec.CommandContext(ctx, "/usr/bin/codesign", "--verify", "--deep", "--strict", "--verbose=2", appPath).CombinedOutput(); err != nil { + return commandOutputError("verify FSKit app signature", output, err) + } + entitlements, err := exec.CommandContext(ctx, "/usr/bin/codesign", "-d", "--entitlements", ":-", "--xml", module).CombinedOutput() + if err != nil { + return commandOutputError("read FSKit module entitlements", entitlements, err) + } + for _, required := range []string{"com.apple.developer.fskit.fsmodule", "com.apple.security.app-sandbox", "com.apple.security.application-groups", service.FSKitAppGroupIdentifier} { + if !strings.Contains(string(entitlements), required) { + return fmt.Errorf("FSKit module signature is missing %s", required) + } + } + profile := filepath.Join(module, "Contents", "embedded.provisionprofile") + profileData, err := exec.CommandContext(ctx, "/usr/bin/security", "cms", "-D", "-i", profile).CombinedOutput() + if err != nil { + return commandOutputError("read FSKit module provisioning profile", profileData, err) + } + if !strings.Contains(string(profileData), service.FSKitAppGroupIdentifier) { + return errors.New("FSKit module provisioning profile does not authorize the App Group") + } + return nil +} + +func ensureFSKitModuleEnabled(moduleID string) (bool, error) { + home, err := os.UserHomeDir() + if err != nil { + return false, err + } + path := filepath.Join(home, "Library", "Group Containers", "group.com.apple.fskit.settings", "enabledModules.plist") + output, err := exec.Command("/usr/bin/plutil", "-convert", "json", "-o", "-", path).CombinedOutput() + if err != nil { + return false, commandOutputError("read enabled FSKit modules", output, err) + } + var modules []string + if err := json.Unmarshal(output, &modules); err != nil { + return false, err + } + for _, current := range modules { + if current == moduleID { + return false, nil + } + } + command := fmt.Sprintf("Add :%d string %s", len(modules), moduleID) + if output, err := exec.Command("/usr/libexec/PlistBuddy", "-c", command, path).CombinedOutput(); err != nil { + return false, commandOutputError("enable FSKit module", output, err) + } + if output, err := exec.Command("/usr/bin/plutil", "-lint", path).CombinedOutput(); err != nil { + return false, commandOutputError("validate enabled FSKit modules", output, err) + } + return true, nil +} + +func killUserProcess(name string) { + output, err := exec.Command("/usr/bin/pgrep", "-u", strconv.Itoa(os.Getuid()), "-x", name).Output() + if err != nil { + return + } + for _, field := range strings.Fields(string(output)) { + pid, err := strconv.Atoi(field) + if err != nil || pid <= 1 { + continue + } + if process, err := os.FindProcess(pid); err == nil { + _ = process.Kill() + } + } +} + +func hashAppBundle(root string) (string, error) { + if !filepath.IsAbs(root) { + return "", errors.New("app bundle path must be absolute") + } + hash := sha256.New() + err := filepath.WalkDir(filepath.Clean(root), func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + info, err := entry.Info() + if err != nil { + return err + } + _, _ = io.WriteString(hash, relative) + _, _ = io.WriteString(hash, "\x00"+info.Mode().String()+"\x00") + if entry.Type()&os.ModeSymlink != 0 { + target, err := os.Readlink(path) + if err != nil { + return err + } + _, _ = io.WriteString(hash, target) + return nil + } + if !entry.Type().IsRegular() { + return nil + } + file, err := os.Open(path) + if err != nil { + return err + } + _, copyErr := io.Copy(hash, file) + closeErr := file.Close() + return errors.Join(copyErr, closeErr) + }) + if err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func syncDirectory(path string) error { + directory, err := os.Open(filepath.Clean(path)) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} + +func commandOutputError(action string, output []byte, err error) error { + detail := strings.TrimSpace(string(output)) + if detail == "" { + return fmt.Errorf("%s: %w", action, err) + } + return fmt.Errorf("%s: %w: %s", action, err, detail) +} diff --git a/internal/cli/fs_service_fskit_other.go b/internal/cli/fs_service_fskit_other.go new file mode 100644 index 0000000..871cbae --- /dev/null +++ b/internal/cli/fs_service_fskit_other.go @@ -0,0 +1,12 @@ +//go:build !darwin + +package cli + +import ( + "context" + "errors" +) + +func prepareFSKitAppPlatform(context.Context, string, string) (fsKitAppTransaction, error) { + return nil, errors.New("native FSKit app installation is available only on macOS") +} diff --git a/internal/cli/fs_service_linux_integration_test.go b/internal/cli/fs_service_linux_integration_test.go new file mode 100644 index 0000000..c5cb8d2 --- /dev/null +++ b/internal/cli/fs_service_linux_integration_test.go @@ -0,0 +1,170 @@ +//go:build linux && fuse && fuse3 && cgo + +package cli + +import ( + "bytes" + "context" + "errors" + "os" + "os/exec" + "os/signal" + "path/filepath" + "sync" + "syscall" + "testing" + "time" + + "github.com/jstar0/codexfold/internal/mountfs" + "github.com/jstar0/codexfold/internal/service" +) + +func TestRealLinuxFSServeRecoversAfterHostSIGKILL(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE3_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE3_TEST=1 to run the real Linux fs serve crash test") + } + if !mountfs.Available() { + t.Fatal("FUSE3 host is unavailable") + } + root := t.TempDir() + home := filepath.Join(root, "codex") + store := filepath.Join(root, "store") + mount := filepath.Join(root, "mount") + native := filepath.Join(root, "native") + for _, directory := range []string{home, store, native} { + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { + _ = exec.Command("fusermount3", "-uz", mount).Run() + _ = os.Chmod(mount, 0o500) + }) + + first, firstDone, firstOutput := startLinuxFSServeHelper(t, home, store, mount, native) + waitForLinuxFSServeMount(t, mount, firstDone, firstOutput) + firstPID := first.Process.Pid + if err := first.Process.Kill(); err != nil { + t.Fatal(err) + } + if err := <-firstDone; err == nil { + t.Fatal("SIGKILLed fs serve helper exited successfully") + } + + second, secondDone, secondOutput := startLinuxFSServeHelper(t, home, store, mount, native) + waitForLinuxFSServeMount(t, mount, secondDone, secondOutput) + if second.Process.Pid == firstPID { + t.Fatal("replacement fs serve reused the killed process ID") + } + if err := second.Process.Signal(syscall.SIGTERM); err != nil { + t.Fatal(err) + } + select { + case err := <-secondDone: + if err != nil { + t.Fatalf("replacement fs serve shutdown: %v\n%s", err, secondOutput.String()) + } + case <-time.After(15 * time.Second): + _ = second.Process.Kill() + t.Fatal("replacement fs serve did not stop after SIGTERM") + } + waitForLinuxFSServeUnmount(t, mount) + info, err := os.Stat(mount) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o500 { + t.Fatalf("unmounted fs serve backing mode=%#o", info.Mode().Perm()) + } +} + +func TestRealLinuxFSServeCrashHelper(t *testing.T) { + if os.Getenv("CODEXFOLD_FS_SERVE_CRASH_HELPER") != "1" { + t.Skip("helper process") + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + root := NewRootCommand() + root.SetOut(os.Stdout) + root.SetErr(os.Stderr) + root.SetArgs([]string{ + "fs", "serve", "--apply", "--foreground=true", + "--codex-home", os.Getenv("CODEXFOLD_FS_SERVE_HOME"), + "--store", os.Getenv("CODEXFOLD_FS_SERVE_STORE"), + "--mount", os.Getenv("CODEXFOLD_FS_SERVE_MOUNT"), + "--canonical-namespace", "--native-root", os.Getenv("CODEXFOLD_FS_SERVE_NATIVE"), + }) + if err := root.ExecuteContext(ctx); err != nil && !errors.Is(err, context.Canceled) { + t.Fatal(err) + } +} + +func startLinuxFSServeHelper(t *testing.T, home string, store string, mount string, native string) (*exec.Cmd, <-chan error, *lockedBuffer) { + t.Helper() + command := exec.Command(os.Args[0], "-test.run=^TestRealLinuxFSServeCrashHelper$", "-test.v") + command.Env = append(os.Environ(), + "CODEXFOLD_FS_SERVE_CRASH_HELPER=1", + "CODEXFOLD_FS_SERVE_HOME="+home, + "CODEXFOLD_FS_SERVE_STORE="+store, + "CODEXFOLD_FS_SERVE_MOUNT="+mount, + "CODEXFOLD_FS_SERVE_NATIVE="+native, + ) + output := &lockedBuffer{} + command.Stdout = output + command.Stderr = output + if err := command.Start(); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- command.Wait() }() + t.Cleanup(func() { _ = command.Process.Kill() }) + return command, done, output +} + +func waitForLinuxFSServeMount(t *testing.T, mount string, done <-chan error, output *lockedBuffer) { + t.Helper() + deadline := time.Now().Add(20 * time.Second) + var lastErr error + for time.Now().Before(deadline) { + if err := service.ProbeMount(mount); err == nil { + return + } else { + lastErr = err + } + select { + case err := <-done: + t.Fatalf("fs serve exited before mount health: %v\n%s", err, output.String()) + case <-time.After(50 * time.Millisecond): + } + } + t.Fatalf("fs serve mount did not become healthy: %v\n%s", lastErr, output.String()) +} + +func waitForLinuxFSServeUnmount(t *testing.T, mount string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if err := service.ProbeMount(mount); err != nil { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatal("fs serve mount remained healthy after shutdown") +} + +type lockedBuffer struct { + mu sync.Mutex + b bytes.Buffer +} + +func (b *lockedBuffer) Write(value []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.b.Write(value) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.b.String() +} diff --git a/internal/cli/fs_service_runtime_other.go b/internal/cli/fs_service_runtime_other.go new file mode 100644 index 0000000..42ba33a --- /dev/null +++ b/internal/cli/fs_service_runtime_other.go @@ -0,0 +1,7 @@ +//go:build !windows + +package cli + +import "github.com/spf13/cobra" + +func addPlatformServiceCommands(*cobra.Command) {} diff --git a/internal/cli/fs_service_runtime_windows.go b/internal/cli/fs_service_runtime_windows.go new file mode 100644 index 0000000..c4b26fc --- /dev/null +++ b/internal/cli/fs_service_runtime_windows.go @@ -0,0 +1,147 @@ +//go:build windows + +package cli + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/jstar0/codexfold/internal/mountfs" + "github.com/jstar0/codexfold/internal/service" + "github.com/spf13/cobra" + "golang.org/x/sys/windows/svc" +) + +func addPlatformServiceCommands(parent *cobra.Command) { + parent.AddCommand(newFSServiceRunCommand()) +} + +func newFSServiceRunCommand() *cobra.Command { + var definitionPath string + command := &cobra.Command{ + Use: "run", + Short: "Run the Windows SCM service host", + Hidden: true, + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + if !mountfs.Available() { + return errors.New("Windows service runtime requires a WinFsp-enabled build") + } + if !filepath.IsAbs(definitionPath) { + return errors.New("absolute Windows service definition path is required") + } + definition, err := os.ReadFile(filepath.Clean(definitionPath)) + if err != nil { + return err + } + config, err := service.ParseWindowsConfig(definition) + if err != nil { + return err + } + if config.ServiceName != serviceLabel { + return errors.New("Windows service definition name does not match this binary") + } + isService, err := svc.IsWindowsService() + if err != nil { + return err + } + if !isService { + return errors.New("Windows service run must be started by the Service Control Manager") + } + stdout, stderr, closeLogs, err := openWindowsServiceLogs(config) + if err != nil { + return err + } + defer closeLogs() + handler := &windowsFSService{ + log: stderr, + run: func(ctx context.Context) error { + serve := newFSServeCommand() + serve.SetArgs(config.Arguments[2:]) + serve.SetOut(stdout) + serve.SetErr(stderr) + serve.SilenceErrors = true + serve.SilenceUsage = true + return serve.ExecuteContext(ctx) + }, + } + return svc.Run(config.ServiceName, handler) + }, + } + command.Flags().StringVar(&definitionPath, "definition", "", "Absolute Windows service definition path") + return command +} + +type windowsFSService struct { + run func(context.Context) error + log io.Writer +} + +func (s *windowsFSService) Execute(_ []string, requests <-chan svc.ChangeRequest, changes chan<- svc.Status) (bool, uint32) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + changes <- svc.Status{State: svc.StartPending, CheckPoint: 1, WaitHint: 15000} + go func() { done <- s.run(ctx) }() + running := svc.Status{State: svc.Running, Accepts: svc.AcceptStop | svc.AcceptShutdown} + changes <- running + + for { + select { + case err := <-done: + if err != nil && !errors.Is(err, context.Canceled) { + _, _ = fmt.Fprintf(s.log, "filesystem service exited: %v\n", err) + return false, 1 + } + return false, 0 + case request := <-requests: + switch request.Cmd { + case svc.Interrogate: + changes <- running + case svc.Stop, svc.Shutdown: + changes <- svc.Status{State: svc.StopPending, CheckPoint: 1, WaitHint: 30000} + cancel() + select { + case err := <-done: + if err != nil && !errors.Is(err, context.Canceled) { + _, _ = fmt.Fprintf(s.log, "filesystem service shutdown failed: %v\n", err) + return false, 1 + } + return false, 0 + case <-time.After(30 * time.Second): + _, _ = fmt.Fprintln(s.log, "filesystem service shutdown timed out") + return false, 1 + } + } + } + } +} + +func openWindowsServiceLogs(config service.WindowsConfig) (io.Writer, io.Writer, func(), error) { + for _, path := range []string{config.StdoutPath, config.StderrPath} { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, nil, nil, err + } + } + stdout, err := os.OpenFile(config.StdoutPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return nil, nil, nil, err + } + stderr, err := os.OpenFile(config.StderrPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + _ = stdout.Close() + return nil, nil, nil, err + } + closeLogs := func() { + _ = stdout.Sync() + _ = stderr.Sync() + _ = stdout.Close() + _ = stderr.Close() + } + return stdout, stderr, closeLogs, nil +} diff --git a/internal/cli/fs_service_transaction_test.go b/internal/cli/fs_service_transaction_test.go new file mode 100644 index 0000000..b87b474 --- /dev/null +++ b/internal/cli/fs_service_transaction_test.go @@ -0,0 +1,174 @@ +package cli + +import ( + "context" + "errors" + "os" + "path/filepath" + "reflect" + "runtime" + "testing" + + "github.com/jstar0/codexfold/internal/service" +) + +type rollbackTestApp struct { + changed bool + rollback func() error +} + +func (a *rollbackTestApp) AppGroupPath() string { return "/tmp/group.vip.jstar.codexfold" } +func (a *rollbackTestApp) Changed() bool { return a.changed } +func (a *rollbackTestApp) Commit() error { return nil } +func (a *rollbackTestApp) Rollback(context.Context) error { + if a.rollback == nil { + return nil + } + return a.rollback() +} + +func TestRollbackFailedServiceInstallRestoresDefinitionAndAppBeforeRestart(t *testing.T) { + root := t.TempDir() + definition := filepath.Join(root, "com.codexfold.test.plist") + if err := os.WriteFile(definition, []byte("old-definition"), 0o600); err != nil { + t.Fatal(err) + } + update, err := service.StageDefinitionUpdate(definition, []byte("new-definition")) + if err != nil { + t.Fatal(err) + } + if err := update.Promote(); err != nil { + t.Fatal(err) + } + + var order []string + appRolledBack := false + app := &rollbackTestApp{changed: true, rollback: func() error { + current, err := os.ReadFile(definition) + if err != nil { + return err + } + if string(current) != "old-definition" { + return errors.New("app rollback ran before the service definition was restored") + } + order = append(order, "app-rollback") + appRolledBack = true + return nil + }} + stop := func(context.Context, service.Platform, string) error { + order = append(order, "stop") + return errors.New("already stopped") + } + start := func(context.Context, service.Platform, string, string) error { + if !appRolledBack { + return errors.New("service restarted before app rollback") + } + current, err := os.ReadFile(definition) + if err != nil { + return err + } + if string(current) != "old-definition" { + return errors.New("service restarted before definition rollback") + } + order = append(order, "start") + return nil + } + + if err := rollbackFailedServiceInstall( + context.Background(), service.PlatformLaunchd, definition, filepath.Join(root, "mount"), + []*service.DefinitionUpdate{update}, app, true, stop, start, + ); err != nil { + t.Fatal(err) + } + if want := []string{"stop", "app-rollback", "start"}; !reflect.DeepEqual(order, want) { + t.Fatalf("rollback order = %v, want %v", order, want) + } + artifacts, err := filepath.Glob(filepath.Join(root, ".codexfold-definition-*")) + if err != nil { + t.Fatal(err) + } + if len(artifacts) != 0 { + t.Fatalf("definition rollback artifacts remain: %v", artifacts) + } +} + +func TestRollbackFailedFirstInstallDoesNotStartAService(t *testing.T) { + root := t.TempDir() + definition := filepath.Join(root, "com.codexfold.test.plist") + update, err := service.StageDefinitionUpdate(definition, []byte("new-definition")) + if err != nil { + t.Fatal(err) + } + if err := update.Promote(); err != nil { + t.Fatal(err) + } + started := false + if err := rollbackFailedServiceInstall( + context.Background(), service.PlatformLaunchd, definition, filepath.Join(root, "mount"), + []*service.DefinitionUpdate{update}, nil, false, + func(context.Context, service.Platform, string) error { return nil }, + func(context.Context, service.Platform, string, string) error { started = true; return nil }, + ); err != nil { + t.Fatal(err) + } + if started { + t.Fatal("failed first install restarted a service that did not previously exist") + } + if _, err := os.Stat(definition); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("failed first install left definition behind: %v", err) + } +} + +func TestValidateLaunchdChildProcessRequiresLockOwnerToBelongToHost(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("launchd child ancestry is macOS-only") + } + lockPath := filepath.Join(t.TempDir(), "service.lock") + lock, err := service.AcquireProcessLock(lockPath) + if err != nil { + t.Fatal(err) + } + defer lock.Close() + + status := validateLaunchdChildProcess(service.Status{DaemonRunning: true, DaemonPID: os.Getppid()}, lockPath, "test") + if !status.DaemonRunning || status.DaemonError != "" { + t.Fatalf("valid child process was rejected: %#v", status) + } + status = validateLaunchdChildProcess(service.Status{DaemonRunning: true, DaemonPID: os.Getppid() + 1}, lockPath, "test") + if status.DaemonRunning || status.DaemonError == "" { + t.Fatalf("foreign child process was accepted: %#v", status) + } +} + +func TestNativeFSKitProcessLocksReportHeldOwners(t *testing.T) { + root := t.TempDir() + paths := nativeFSKitProcessLockPaths{ + daemon: filepath.Join(root, "service.lock"), + supervisor: filepath.Join(root, "supervisor.lock"), + } + daemon, err := service.AcquireProcessLock(paths.daemon) + if err != nil { + t.Fatal(err) + } + defer daemon.Close() + supervisor, err := service.AcquireProcessLock(paths.supervisor) + if err != nil { + t.Fatal(err) + } + defer supervisor.Close() + + daemonStatus, err := service.InspectProcessLock(paths.daemon) + if err != nil { + t.Fatal(err) + } + supervisorStatus, err := service.InspectProcessLock(paths.supervisor) + if err != nil { + t.Fatal(err) + } + if !daemonStatus.Held || daemonStatus.PID != os.Getpid() { + t.Fatalf("daemon lock = %#v", daemonStatus) + } + if !supervisorStatus.Held || supervisorStatus.PID != os.Getpid() { + t.Fatalf("supervisor lock = %#v", supervisorStatus) + } +} diff --git a/internal/cli/fs_supervisor.go b/internal/cli/fs_supervisor.go new file mode 100644 index 0000000..14534fb --- /dev/null +++ b/internal/cli/fs_supervisor.go @@ -0,0 +1,74 @@ +package cli + +import ( + "errors" + "fmt" + "path/filepath" + "runtime" + "time" + + "github.com/jstar0/codexfold/internal/service" + "github.com/spf13/cobra" +) + +type FSNativeSupervisorResult struct { + ResourcePath string `json:"resource_path"` + MountPoint string `json:"mount_point"` + Interval time.Duration `json:"interval"` + ProbeTimeout time.Duration `json:"probe_timeout"` + Recovery time.Duration `json:"recovery_timeout"` + DryRun bool `json:"dry_run"` +} + +func newFSNativeSupervisorCommand() *cobra.Command { + var resourcePath, mountPoint string + var interval, probeTimeout, recoveryTimeout time.Duration + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "supervise", + Short: "Keep the native FSKit mount healthy and remount it after daemon or extension failure", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + if !filepath.IsAbs(resourcePath) || !filepath.IsAbs(mountPoint) { + return errors.New("absolute FSKit resource and mount paths are required") + } + if interval <= 0 || probeTimeout <= 0 || recoveryTimeout <= 0 { + return errors.New("supervisor timing values must be positive") + } + result := FSNativeSupervisorResult{ + ResourcePath: filepath.Clean(resourcePath), MountPoint: filepath.Clean(mountPoint), + Interval: interval, ProbeTimeout: probeTimeout, Recovery: recoveryTimeout, DryRun: !apply, + } + if !apply { + if jsonOutput { + return writeJSON(command, result) + } + _, err := fmt.Fprintf(command.OutOrStdout(), "dry_run=true resource=%s mount=%s interval=%s probe_timeout=%s recovery_timeout=%s\n", result.ResourcePath, result.MountPoint, result.Interval, result.ProbeTimeout, result.Recovery) + return err + } + if runtime.GOOS != "darwin" { + return errors.New("native FSKit supervision is available only on macOS") + } + processLock, err := service.AcquireProcessLock(filepath.Join(result.ResourcePath, service.NativeFSKitSupervisorLockName)) + if err != nil { + return err + } + defer processLock.Close() + return service.RunNativeFSKitSupervisor(command.Context(), service.NativeFSKitSupervisorOptions{ + ResourcePath: result.ResourcePath, MountPoint: result.MountPoint, + Interval: result.Interval, ProbeTimeout: result.ProbeTimeout, RecoveryTimeout: result.Recovery, + Event: func(message string) { + _, _ = fmt.Fprintf(command.ErrOrStderr(), "native-fskit supervisor: %s\n", message) + }, + }) + }, + } + command.Flags().StringVar(&resourcePath, "resource", "", "Absolute native FSKit resource descriptor path") + command.Flags().StringVar(&mountPoint, "mount", "", "Absolute native FSKit mount point") + command.Flags().DurationVar(&interval, "interval", time.Second, "Health reconciliation interval") + command.Flags().DurationVar(&probeTimeout, "probe-timeout", 2*time.Second, "Maximum duration of one mount health probe") + command.Flags().DurationVar(&recoveryTimeout, "recovery-timeout", 15*time.Second, "Maximum duration of one mount or unmount recovery") + command.Flags().BoolVar(&apply, "apply", false, "Run the native FSKit mount supervisor") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output for dry-run") + return command +} diff --git a/internal/cli/fs_supervisor_test.go b/internal/cli/fs_supervisor_test.go new file mode 100644 index 0000000..cb1160e --- /dev/null +++ b/internal/cli/fs_supervisor_test.go @@ -0,0 +1,32 @@ +package cli + +import ( + "bytes" + "encoding/json" + "path/filepath" + "testing" +) + +func TestFSNativeSupervisorDryRunReportsAbsoluteRuntimePaths(t *testing.T) { + root := t.TempDir() + command := NewRootCommand() + var output bytes.Buffer + command.SetOut(&output) + command.SetErr(&output) + command.SetArgs([]string{ + "fs", "supervise", + "--resource", filepath.Join(root, "resource.bin"), + "--mount", filepath.Join(root, "mount"), + "--json", + }) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + var result FSNativeSupervisorResult + if err := json.Unmarshal(output.Bytes(), &result); err != nil { + t.Fatalf("decode supervisor dry-run: %v\n%s", err, output.String()) + } + if !result.DryRun || result.ResourcePath == "" || result.MountPoint == "" { + t.Fatalf("unexpected supervisor dry-run: %#v", result) + } +} diff --git a/internal/cli/fs_test.go b/internal/cli/fs_test.go index 18e2065..2b1bcb8 100644 --- a/internal/cli/fs_test.go +++ b/internal/cli/fs_test.go @@ -17,23 +17,37 @@ import ( "github.com/jstar0/codexfold/internal/codex" "github.com/jstar0/codexfold/internal/compat" + "github.com/jstar0/codexfold/internal/enroll" "github.com/jstar0/codexfold/internal/fold" "github.com/jstar0/codexfold/internal/fsctl" "github.com/jstar0/codexfold/internal/mountfs" "github.com/jstar0/codexfold/internal/pack" + "github.com/jstar0/codexfold/internal/storage" "github.com/jstar0/codexfold/internal/vfs" ) +type cliRejectingChecker struct { + Calls int + Projection storage.Projection +} + +func (c *cliRejectingChecker) Check(_ context.Context, projection storage.Projection) (storage.Assessment, error) { + c.Calls++ + c.Projection = projection + return storage.Assessment{}, storage.ErrBudgetExceeded +} + func TestRootExposesPackAndFilesystemCommands(t *testing.T) { root := NewRootCommand() for _, commandPath := range [][]string{ {"pack", "build"}, {"pack", "doctor"}, - {"fs", "status"}, {"fs", "doctor"}, {"fs", "compatibility"}, {"fs", "compatibility-import"}, {"fs", "benchmark"}, + {"fs", "status"}, {"fs", "doctor"}, {"fs", "validate-native"}, {"fs", "compatibility"}, {"fs", "compatibility-import"}, {"fs", "benchmark"}, {"fs", "serve"}, {"fs", "migrate"}, {"fs", "rollback"}, {"fs", "compact"}, {"fs", "recover"}, + {"fs", "enroll", "plan"}, {"fs", "enroll", "apply"}, {"fs", "namespace", "status"}, {"fs", "namespace", "activate"}, {"fs", "namespace", "deactivate"}, {"fs", "namespace", "recover"}, {"fs", "service", "install"}, {"fs", "service", "start"}, {"fs", "service", "stop"}, - {"fs", "service", "status"}, {"fs", "service", "update-preflight"}, + {"fs", "service", "status"}, {"fs", "service", "update-binary"}, {"fs", "service", "update-preflight"}, } { if _, _, err := root.Find(commandPath); err != nil { t.Fatalf("command %v should be exposed: %v", commandPath, err) @@ -41,6 +55,29 @@ func TestRootExposesPackAndFilesystemCommands(t *testing.T) { } } +func TestRetainCanonicalSnapshotBudgetRejectsBeforeCreatingSnapshot(t *testing.T) { + root := t.TempDir() + store := filepath.Join(root, "store") + sourcePath := filepath.Join(root, "source.jsonl") + if err := os.WriteFile(sourcePath, []byte("budgeted snapshot"), 0o600); err != nil { + t.Fatal(err) + } + source, err := hashPath(sourcePath) + if err != nil { + t.Fatal(err) + } + checker := &cliRejectingChecker{} + if _, err := retainCanonicalSnapshot(context.Background(), store, "session", source, checker); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("retainCanonicalSnapshot error = %v, want storage budget rejection", err) + } + if checker.Calls != 1 || checker.Projection.Operation != "retain-migration-snapshot" || checker.Projection.AdditionalPersistentBytes != source.Bytes { + t.Fatalf("unexpected snapshot budget projection: %#v", checker) + } + if _, err := os.Stat(filepath.Join(store, "fs", "snapshots")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("snapshot directory exists after preflight rejection: %v", err) + } +} + func TestFSCompatibilityImportPersistsOnlySanitizedContract(t *testing.T) { home := t.TempDir() store := filepath.Join(home, "fold-store") @@ -330,6 +367,64 @@ func TestFSServiceInstallIsDryRunByDefaultAndApplyRequiresFuseBuild(t *testing.T } } +func TestFSServiceInstallRendersNativeFSKitDaemonAndSupervisor(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("native FSKit services are macOS-only") + } + home, storeDir, _ := fsFixture(t, true) + definition := filepath.Join(home, "LaunchAgents", "com.codexfold.fs.plist") + fskitApp := filepath.Join(home, "Applications", "CodexFoldFSKit.app") + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "service", "install", "--frontend", "native-fskit", + "--codex-home", home, "--store", storeDir, "--definition", definition, + "--fskit-app", fskitApp, "--json", + }) + if err := root.Execute(); err != nil { + t.Fatalf("native FSKit service dry-run: %v", err) + } + var result FSServiceInstallResult + if err := json.Unmarshal(output.Bytes(), &result); err != nil { + t.Fatal(err) + } + wantSupervisor := filepath.Join(filepath.Dir(definition), "com.codexfold.fs.supervisor.plist") + if !result.DryRun || result.Path != definition || result.SupervisorPath != wantSupervisor || result.SupervisorBytes == 0 { + t.Fatalf("native FSKit install result = %#v", result) + } + for _, path := range []string{definition, wantSupervisor} { + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("dry-run wrote %s: %v", path, err) + } + } + launcher := filepath.Join(fskitApp, "Contents", "MacOS", "CodexFoldFSKit") + if result.FSKitAppPath != fskitApp || result.FSKitLauncherPath != launcher || result.FSKitResourcePath == "" { + t.Fatalf("native FSKit resolved paths = %#v", result) + } +} + +func TestFSServiceRestartIsExposedAsDryRun(t *testing.T) { + home := t.TempDir() + definition := filepath.Join(home, "com.codexfold.fs.plist") + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "service", "restart", "--definition", definition, "--json"}) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + var result FSServiceActionResult + if err := json.Unmarshal(output.Bytes(), &result); err != nil { + t.Fatal(err) + } + if result.Action != "restart" || !result.DryRun || result.Path != definition { + t.Fatalf("restart dry-run = %#v", result) + } +} + func TestFSUpdatePreflightQuarantineRoutesLatestVisibleBytesNative(t *testing.T) { allowFixtureMount(t) home, storeDir, nativePath := fsFixture(t, true) @@ -444,6 +539,206 @@ func TestFSMigrateIsDryRunByDefaultAndDoesNotChangeRoute(t *testing.T) { } } +func TestFSEnrollmentPlanRequiresTwoStableObservationsAndCanaryGate(t *testing.T) { + home, storeDir, _ := fsFixture(t, true) + allowEnrollmentWriterProbe(t) + oldProbe := mountHealthProbe + mountHealthProbe = func(string) error { return nil } + t.Cleanup(func() { mountHealthProbe = oldProbe }) + mount := filepath.Join(home, "mount") + nativeRoot := filepath.Join(home, "fold-native") + args := []string{ + "fs", "enroll", "plan", "--codex-home", home, "--store", storeDir, "--mount", mount, + "--canonical-namespace", "--native-root", nativeRoot, "--enrollment-canary", "--stable-for", "1ns", "--record-observations", "--json", + } + runPlan := func() enroll.Plan { + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs(args) + if err := root.Execute(); err != nil { + t.Fatalf("enrollment plan: %v", err) + } + var plan enroll.Plan + if err := json.Unmarshal(output.Bytes(), &plan); err != nil { + t.Fatalf("decode enrollment plan: %v\n%s", err, output.String()) + } + return plan + } + first := runPlan() + if len(first.Selected) != 0 { + t.Fatalf("first observation selected enrollment: %#v", first) + } + time.Sleep(time.Millisecond) + second := runPlan() + if len(second.Selected) != 1 || second.Selected[0].SessionID != "session" { + t.Fatalf("second stable observation was not selected: %#v", second) + } +} + +func TestFSEnrollmentApplyRunsFoldPackMigrateAndStopsBeforeRouteOnFailure(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + allowEnrollmentWriterProbe(t) + oldProbe := mountHealthProbe + mountHealthProbe = func(string) error { return nil } + t.Cleanup(func() { mountHealthProbe = oldProbe }) + observationPath := enrollmentObservationPath(storeDir) + info, err := os.Stat(nativePath) + if err != nil { + t.Fatal(err) + } + if err := enroll.SaveObservations(observationPath, enroll.Observations{"session": { + Path: nativePath, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), StableSinceUnixNano: time.Now().Add(-time.Hour).UnixNano(), + }}); err != nil { + t.Fatal(err) + } + oldRunner := runEnrollmentCommand + defer func() { runEnrollmentCommand = oldRunner }() + var calls [][]string + runEnrollmentCommand = func(_ context.Context, args []string) error { + calls = append(calls, append([]string(nil), args...)) + if len(calls) == 2 { + return errors.New("pack failed") + } + return nil + } + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "enroll", "apply", "--codex-home", home, "--store", storeDir, "--mount", filepath.Join(home, "mount"), + "--canonical-namespace", "--native-root", filepath.Join(home, "fold-native"), "--enrollment-canary", "--stable-for", "1ns", "--apply", + }) + if err := root.Execute(); err == nil || !strings.Contains(err.Error(), "pack failed") { + t.Fatalf("enrollment apply error = %v, want pack failure", err) + } + if len(calls) != 2 || len(calls[0]) < 2 || calls[0][0] != "fold" || calls[1][0] != "pack" { + t.Fatalf("enrollment command sequence = %#v", calls) + } + sessions, err := codex.LoadSessions(home) + if err != nil || len(sessions) != 1 || filepath.Clean(sessions[0].RolloutPath) != filepath.Clean(nativePath) { + t.Fatalf("failed enrollment changed route: sessions=%#v err=%v", sessions, err) + } + if data, err := os.ReadFile(nativePath); err != nil || len(data) == 0 { + t.Fatalf("failed enrollment changed source: bytes=%d err=%v", len(data), err) + } +} + +func TestPeriodicEnrollmentLoopRunsSerialCyclesAndStopsWithContext(t *testing.T) { + oldRunner := runServiceEnrollmentCycle + defer func() { runServiceEnrollmentCycle = oldRunner }() + + started := make(chan struct{}, 2) + release := make(chan struct{}, 2) + flagErrors := make(chan error, 1) + runServiceEnrollmentCycle = func(ctx context.Context, flags enrollmentFlags) (FSEnrollmentApplyResult, error) { + if flags.batchSize != 3 || flags.stableFor != 2*time.Hour || !flags.canonicalNamespace { + select { + case flagErrors <- fmt.Errorf("unexpected enrollment flags: %#v", flags): + default: + } + } + started <- struct{}{} + select { + case <-ctx.Done(): + return FSEnrollmentApplyResult{}, ctx.Err() + case <-release: + return FSEnrollmentApplyResult{}, nil + } + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + runPeriodicEnrollment(ctx, enrollmentFlags{batchSize: 3, stableFor: 2 * time.Hour, canonicalNamespace: true}, time.Millisecond, nil) + close(done) + }() + + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("periodic enrollment did not run") + } + select { + case <-started: + t.Fatal("periodic enrollment overlapped a running cycle") + case <-time.After(10 * time.Millisecond): + } + release <- struct{}{} + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("periodic enrollment did not schedule the next cycle") + } + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("periodic enrollment did not stop with its context") + } + select { + case err := <-flagErrors: + t.Fatal(err) + default: + } +} + +func TestEnrollmentCycleInPreviewRecordsObservationsWithoutMutating(t *testing.T) { + home, storeDir, _ := fsFixture(t, true) + allowEnrollmentWriterProbe(t) + oldProbe := mountHealthProbe + mountHealthProbe = func(string) error { return nil } + t.Cleanup(func() { mountHealthProbe = oldProbe }) + oldRunner := runEnrollmentCommand + defer func() { runEnrollmentCommand = oldRunner }() + runEnrollmentCommand = func(context.Context, []string) error { + t.Fatal("preview enrollment cycle attempted a mutation") + return nil + } + + result, err := runEnrollmentCycle(context.Background(), enrollmentFlags{ + codexHome: home, storeDir: storeDir, mountPoint: filepath.Join(home, "mount"), + nativeRoot: filepath.Join(home, "fold-native"), canonicalNamespace: true, + stableFor: time.Nanosecond, batchSize: 1, + }) + if err != nil { + t.Fatalf("runEnrollmentCycle: %v", err) + } + if result.Apply.Applied != 0 || len(result.Plan.Selected) != 0 { + t.Fatalf("preview cycle selected or applied sessions: %#v", result) + } + observations, err := enroll.LoadObservations(enrollmentObservationPath(storeDir)) + if err != nil || len(observations) != 1 { + t.Fatalf("preview cycle did not persist observations: observations=%#v err=%v", observations, err) + } +} + +func allowEnrollmentWriterProbe(t *testing.T) { + t.Helper() + oldProbe := enrollmentWriterProbe + enrollmentWriterProbe = func(context.Context, []codex.Session) (map[string]bool, error) { + return map[string]bool{}, nil + } + t.Cleanup(func() { enrollmentWriterProbe = oldProbe }) +} + +func TestEnrollmentStorageHealthAllowsBootstrapButRejectsBrokenCommittedPack(t *testing.T) { + storeDir := t.TempDir() + if err := requireEnrollmentStorageHealth(context.Background(), storeDir); err != nil { + t.Fatalf("empty enrollment store should be a valid bootstrap state: %v", err) + } + if err := os.MkdirAll(filepath.Join(storeDir, "packs"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(storeDir, "packs", "CURRENT"), []byte("missing-generation\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := requireEnrollmentStorageHealth(context.Background(), storeDir); err == nil { + t.Fatal("a broken committed pack generation passed enrollment health") + } +} + func TestFSMigrateApplyFailsClosedWithoutMountedTarget(t *testing.T) { home, storeDir, nativePath := fsFixture(t, true) root := NewRootCommand() @@ -642,7 +937,7 @@ func TestFSMigrateCanonicalKeepsCodexRouteAndHidesRetainedSnapshot(t *testing.T) t.Fatal(err) } _ = db.Close() - if _, err := fold.Fold(context.Background(), codex.Session{ID: "session", RolloutPath: route, Archived: true}, fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { + if _, err := fold.Fold(context.Background(), fold.Session{ID: "session", RolloutPath: route, Archived: true}, fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { t.Fatal(err) } if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { @@ -827,7 +1122,7 @@ func interruptedCanonicalMigrationFixture(t *testing.T) interruptedCanonicalFixt t.Fatal(err) } _ = db.Close() - if _, err := fold.Fold(context.Background(), codex.Session{ID: "session", RolloutPath: route, Archived: true}, fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { + if _, err := fold.Fold(context.Background(), fold.Session{ID: "session", RolloutPath: route, Archived: true}, fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { t.Fatal(err) } if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { @@ -845,7 +1140,7 @@ func interruptedCanonicalMigrationFixture(t *testing.T) interruptedCanonicalFixt if err != nil { t.Fatal(err) } - retained, err := retainCanonicalSnapshot(storeDir, "session", native) + retained, err := retainCanonicalSnapshot(context.Background(), storeDir, "session", native, nil) if err != nil { t.Fatal(err) } @@ -893,7 +1188,7 @@ func TestFSMigrateCanonicalReservesWriterDuringCutover(t *testing.T) { t.Fatal(err) } _ = db.Close() - if _, err := fold.Fold(context.Background(), codex.Session{ID: "session", RolloutPath: route, Archived: true}, fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { + if _, err := fold.Fold(context.Background(), fold.Session{ID: "session", RolloutPath: route, Archived: true}, fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { t.Fatal(err) } if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { @@ -1983,6 +2278,104 @@ func TestFSReadOnlyCommandsRunWithoutClaimingMountHealth(t *testing.T) { } } +func TestFSStatusAndDoctorExposePhysicalStorageAccounting(t *testing.T) { + home, storeDir, _ := fsFixture(t, true) + for _, args := range [][]string{ + {"fs", "status", "--codex-home", home, "--store", storeDir, "--json"}, + {"fs", "doctor", "--codex-home", home, "--store", storeDir, "--json"}, + } { + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs(args) + if err := root.Execute(); err != nil { + t.Fatalf("%v: %v", args, err) + } + var payload struct { + Storage struct { + LogicalSessionBytes int64 `json:"logical_session_bytes"` + TotalPhysicalBytes int64 `json:"total_physical_bytes"` + } `json:"storage"` + StorageLimits storage.Limits `json:"storage_limits"` + AvailableBytes int64 `json:"available_bytes"` + } + if err := json.Unmarshal(output.Bytes(), &payload); err != nil { + t.Fatalf("decode %v: %v\n%s", args, err, output.String()) + } + if payload.Storage.LogicalSessionBytes <= 0 || payload.Storage.TotalPhysicalBytes <= 0 || payload.StorageLimits.MaxPhysicalBytes <= 0 || payload.AvailableBytes <= 0 { + t.Fatalf("incomplete storage accounting for %v: %#v", args, payload) + } + } +} + +func TestStartupStorageGCRunsOnlyAfterHealthyStoreVerification(t *testing.T) { + _, storeDir, _ := fsFixture(t, true) + if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { + t.Fatal(err) + } + if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { + t.Fatal(err) + } + before := countPackGenerationDirectories(t, storeDir) + result, ran, err := startupStorageGC(context.Background(), storeDir) + if err != nil { + t.Fatalf("startupStorageGC: %v", err) + } + if !ran || before != 3 || result.RemovedCount != 1 || countPackGenerationDirectories(t, storeDir) != 2 { + t.Fatalf("healthy startup GC result: before=%d ran=%t result=%#v after=%d", before, ran, result, countPackGenerationDirectories(t, storeDir)) + } + + if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { + t.Fatal(err) + } + current, err := os.ReadFile(filepath.Join(storeDir, "packs", "CURRENT")) + if err != nil { + t.Fatal(err) + } + index, err := os.ReadFile(filepath.Join(storeDir, "packs", strings.TrimSpace(string(current)), "index.json")) + if err != nil { + t.Fatal(err) + } + var decoded struct { + Objects []struct { + Blocks []struct { + Pack string `json:"pack"` + } `json:"blocks"` + } `json:"objects"` + } + if err := json.Unmarshal(index, &decoded); err != nil || len(decoded.Objects) == 0 || len(decoded.Objects[0].Blocks) == 0 { + t.Fatalf("decode current pack index: %#v err=%v", decoded, err) + } + packPath := filepath.Join(storeDir, "packs", strings.TrimSpace(string(current)), decoded.Objects[0].Blocks[0].Pack) + if err := os.WriteFile(packPath, []byte("corrupt"), 0o600); err != nil { + t.Fatal(err) + } + before = countPackGenerationDirectories(t, storeDir) + _, ran, err = startupStorageGC(context.Background(), storeDir) + if err != nil { + t.Fatalf("unhealthy startupStorageGC: %v", err) + } + if ran || countPackGenerationDirectories(t, storeDir) != before { + t.Fatalf("unhealthy store was mutated: ran=%t before=%d after=%d", ran, before, countPackGenerationDirectories(t, storeDir)) + } +} + +func countPackGenerationDirectories(t *testing.T, store string) int { + t.Helper() + entries, err := os.ReadDir(filepath.Join(store, "packs")) + if err != nil { + t.Fatal(err) + } + count := 0 + for _, entry := range entries { + if entry.IsDir() && !strings.HasPrefix(entry.Name(), ".") { + count++ + } + } + return count +} + func fsFixture(t *testing.T, archived bool) (string, string, string) { t.Helper() home := t.TempDir() @@ -2006,7 +2399,7 @@ func fsFixture(t *testing.T, archived bool) (string, string, string) { } } session := codex.Session{ID: "session", RolloutPath: nativePath, Archived: archived} - if _, err := fold.Fold(context.Background(), session, fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { + if _, err := fold.Fold(context.Background(), toFoldSession(session), fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { t.Fatalf("fold fixture: %v", err) } if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { diff --git a/internal/cli/root.go b/internal/cli/root.go index fdb819d..7c611a0 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -23,6 +23,8 @@ func NewRootCommand() *cobra.Command { Version: resolvedVersion(), } root.AddCommand(newScanCommand()) + root.AddCommand(newForkFamilyCommand()) + root.AddCommand(newArchiveCommand()) root.AddCommand(newContainsCommand()) root.AddCommand(newRemoveContainedCommand()) root.AddCommand(newFoldCommand()) diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 3106231..42f925e 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -8,19 +8,88 @@ import ( "path/filepath" "testing" + "github.com/jstar0/codexfold/internal/family" "github.com/jstar0/codexfold/internal/scan" _ "modernc.org/sqlite" ) func TestRootExposesScanCommand(t *testing.T) { root := NewRootCommand() - for _, name := range []string{"scan", "contains", "remove-contained", "fold", "unfold", "materialize", "doctor", "gc"} { + for _, name := range []string{"scan", "fork-family", "archive", "contains", "remove-contained", "fold", "unfold", "materialize", "doctor", "gc"} { if _, _, err := root.Find([]string{name}); err != nil { t.Fatalf("%s command should be exposed: %v", name, err) } } } +func TestForkFamilyShowAndCompareUseExplicitSessions(t *testing.T) { + home := t.TempDir() + leftPath := filepath.Join(home, "left.jsonl") + rightPath := filepath.Join(home, "right.jsonl") + if err := os.WriteFile(leftPath, []byte("{\"type\":\"session_meta\",\"id\":\"left\"}\n{\"v\":1}\n{\"left\":true}\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(rightPath, []byte("{\"type\":\"session_meta\",\"id\":\"right\"}\n{\"v\":1}\n{\"right\":true}\n"), 0o600); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(` + create table threads ( + id text primary key, title text, cwd text, rollout_path text, + model_provider text, model text, updated_at integer, + archived integer, git_branch text + ); + create table thread_spawn_edges ( + parent_thread_id text not null, + child_thread_id text not null primary key, + status text not null + ); + `); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`insert into threads values ('left', 'Left', '/workspace', ?, 'provider', 'model', 2, 0, 'main')`, leftPath); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`insert into threads values ('right', 'Right', '/workspace', ?, 'provider', 'model', 1, 1, 'main')`, rightPath); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`insert into thread_spawn_edges values ('left', 'right', 'closed')`); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fork-family", "show", "left", "--codex-home", home, "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("fork-family show: %v", err) + } + var report family.Report + if err := json.Unmarshal(output.Bytes(), &report); err != nil || len(report.Members) != 2 || len(report.Edges) != 1 { + t.Fatalf("family show = %#v err=%v output=%s", report, err, output.String()) + } + + output.Reset() + root = NewRootCommand() + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fork-family", "compare", "left", "right", "--codex-home", home, "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("fork-family compare: %v", err) + } + var comparison family.Comparison + if err := json.Unmarshal(output.Bytes(), &comparison); err != nil || comparison.Relation != family.RelationIndependentTails || comparison.GraphRelation != family.GraphAncestor { + t.Fatalf("family comparison = %#v err=%v output=%s", comparison, err, output.String()) + } +} + func TestRootUsesExplicitBuildVersion(t *testing.T) { previous := Version Version = "v-test" diff --git a/internal/codex/edges.go b/internal/codex/edges.go new file mode 100644 index 0000000..4d22023 --- /dev/null +++ b/internal/codex/edges.go @@ -0,0 +1,55 @@ +package codex + +import ( + "database/sql" + "fmt" + "path/filepath" + + _ "modernc.org/sqlite" +) + +type SpawnEdge struct { + ParentID string `json:"parent_id"` + ChildID string `json:"child_id"` + Status string `json:"status"` +} + +func LoadSpawnEdges(home string) ([]SpawnEdge, error) { + dbPath := filepath.Join(home, "state_5.sqlite") + db, err := sql.Open("sqlite", sqliteReadOnlyDSN(dbPath)) + if err != nil { + return nil, fmt.Errorf("open Codex spawn-edge database: %w", err) + } + defer func() { _ = db.Close() }() + if _, err := db.Exec(`pragma busy_timeout = 5000`); err != nil { + return nil, fmt.Errorf("configure Codex spawn-edge database: %w", err) + } + var exists int + if err := db.QueryRow(`select count(*) from sqlite_master where type = 'table' and name = 'thread_spawn_edges'`).Scan(&exists); err != nil { + return nil, fmt.Errorf("inspect Codex spawn-edge table: %w", err) + } + if exists == 0 { + return []SpawnEdge{}, nil + } + rows, err := db.Query(` + select parent_thread_id, child_thread_id, status + from thread_spawn_edges + order by parent_thread_id, child_thread_id, status + `) + if err != nil { + return nil, fmt.Errorf("query Codex spawn edges: %w", err) + } + defer func() { _ = rows.Close() }() + edges := make([]SpawnEdge, 0) + for rows.Next() { + var edge SpawnEdge + if err := rows.Scan(&edge.ParentID, &edge.ChildID, &edge.Status); err != nil { + return nil, fmt.Errorf("scan Codex spawn edge: %w", err) + } + edges = append(edges, edge) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate Codex spawn edges: %w", err) + } + return edges, nil +} diff --git a/internal/codex/edges_test.go b/internal/codex/edges_test.go new file mode 100644 index 0000000..46fb1f3 --- /dev/null +++ b/internal/codex/edges_test.go @@ -0,0 +1,54 @@ +package codex + +import ( + "database/sql" + "path/filepath" + "testing" + + _ "modernc.org/sqlite" +) + +func TestLoadSpawnEdgesReturnsCurrentGraphAndAllowsMissingTable(t *testing.T) { + home := t.TempDir() + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(` + create table thread_spawn_edges ( + parent_thread_id text not null, + child_thread_id text not null primary key, + status text not null + ); + insert into thread_spawn_edges values ('parent', 'child-b', 'closed'); + insert into thread_spawn_edges values ('parent', 'child-a', 'open'); + `); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + edges, err := LoadSpawnEdges(home) + if err != nil { + t.Fatal(err) + } + if len(edges) != 2 || edges[0].ChildID != "child-a" || edges[1].Status != "closed" { + t.Fatalf("spawn edges = %#v", edges) + } + + emptyHome := t.TempDir() + empty, err := sql.Open("sqlite", filepath.Join(emptyHome, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := empty.Exec(`create table threads (id text primary key);`); err != nil { + t.Fatal(err) + } + if err := empty.Close(); err != nil { + t.Fatal(err) + } + edges, err = LoadSpawnEdges(emptyHome) + if err != nil || len(edges) != 0 { + t.Fatalf("missing edge table = %#v err=%v", edges, err) + } +} diff --git a/internal/enroll/apply.go b/internal/enroll/apply.go new file mode 100644 index 0000000..a77650b --- /dev/null +++ b/internal/enroll/apply.go @@ -0,0 +1,55 @@ +package enroll + +import ( + "context" + "errors" + "fmt" + "os" +) + +type ApplyOptions struct { + Limit int + IsManaged func(context.Context, string) (bool, error) + Apply func(context.Context, Decision) error +} + +type ApplyResult struct { + Selected int `json:"selected"` + Applied int `json:"applied"` + SkippedChanged int `json:"skipped_changed"` + SkippedManaged int `json:"skipped_managed"` +} + +func Apply(ctx context.Context, plan Plan, options ApplyOptions) (ApplyResult, error) { + if options.IsManaged == nil || options.Apply == nil { + return ApplyResult{}, errors.New("enrollment managed-state and apply callbacks are required") + } + limit := options.Limit + if limit <= 0 || limit > len(plan.Selected) { + limit = len(plan.Selected) + } + result := ApplyResult{Selected: min(limit, len(plan.Selected))} + for _, decision := range plan.Selected[:limit] { + if err := ctx.Err(); err != nil { + return result, err + } + managed, err := options.IsManaged(ctx, decision.SessionID) + if err != nil { + return result, err + } + if managed { + result.SkippedManaged++ + continue + } + info, err := os.Lstat(decision.RolloutPath) + if err != nil || !info.Mode().IsRegular() || info.Size() != decision.Fingerprint.Size || info.ModTime().UnixNano() != decision.Fingerprint.ModTimeUnixNano { + result.SkippedChanged++ + continue + } + if err := options.Apply(ctx, decision); err != nil { + return result, fmt.Errorf("apply enrollment for %s: %w", decision.SessionID, err) + } + result.Applied++ + } + return result, nil +} diff --git a/internal/enroll/observations.go b/internal/enroll/observations.go new file mode 100644 index 0000000..bf79bec --- /dev/null +++ b/internal/enroll/observations.go @@ -0,0 +1,88 @@ +package enroll + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +type observationFile struct { + Version int `json:"version"` + Observations Observations `json:"observations"` +} + +func LoadObservations(path string) (Observations, error) { + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return make(Observations), nil + } + if err != nil { + return nil, err + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + var stored observationFile + if err := decoder.Decode(&stored); err != nil { + return nil, fmt.Errorf("decode enrollment observations: %w", err) + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + if err == nil { + err = errors.New("multiple JSON values") + } + return nil, fmt.Errorf("decode enrollment observations: %w", err) + } + if stored.Version != 1 { + return nil, fmt.Errorf("unsupported enrollment observation version %d", stored.Version) + } + if stored.Observations == nil { + stored.Observations = make(Observations) + } + return stored.Observations, nil +} + +func SaveObservations(path string, observations Observations) error { + if !filepath.IsAbs(path) { + return errors.New("enrollment observation path must be absolute") + } + if observations == nil { + observations = make(Observations) + } + data, err := json.MarshalIndent(observationFile{Version: 1, Observations: observations}, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + directory := filepath.Dir(path) + if err := os.MkdirAll(directory, 0o700); err != nil { + return err + } + temporary, err := os.CreateTemp(directory, ".observations-*.tmp") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return err + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + if err := os.Rename(temporaryPath, path); err != nil { + return err + } + return syncObservationDirectory(directory) +} diff --git a/internal/enroll/observations_sync_unix.go b/internal/enroll/observations_sync_unix.go new file mode 100644 index 0000000..81e3d0d --- /dev/null +++ b/internal/enroll/observations_sync_unix.go @@ -0,0 +1,14 @@ +//go:build !windows + +package enroll + +import "os" + +func syncObservationDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} diff --git a/internal/enroll/observations_sync_windows.go b/internal/enroll/observations_sync_windows.go new file mode 100644 index 0000000..24670f7 --- /dev/null +++ b/internal/enroll/observations_sync_windows.go @@ -0,0 +1,5 @@ +//go:build windows + +package enroll + +func syncObservationDirectory(string) error { return nil } diff --git a/internal/enroll/observations_test.go b/internal/enroll/observations_test.go new file mode 100644 index 0000000..0385bb1 --- /dev/null +++ b/internal/enroll/observations_test.go @@ -0,0 +1,35 @@ +package enroll + +import ( + "os" + "path/filepath" + "testing" +) + +func TestObservationStoreRoundTripsAndRejectsUnknownVersion(t *testing.T) { + path := filepath.Join(t.TempDir(), "enrollment", "observations.json") + want := Observations{"session": {Path: "/tmp/session.jsonl", Size: 12, ModTimeUnixNano: 34, StableSinceUnixNano: 56}} + if err := SaveObservations(path, want); err != nil { + t.Fatalf("SaveObservations: %v", err) + } + got, err := LoadObservations(path) + if err != nil { + t.Fatalf("LoadObservations: %v", err) + } + if got["session"] != want["session"] { + t.Fatalf("observations = %#v, want %#v", got, want) + } + if err := os.WriteFile(path, []byte(`{"version":2,"observations":{}}`), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadObservations(path); err == nil { + t.Fatal("unknown observation version should fail") + } +} + +func TestLoadObservationsReturnsEmptyWhenMissing(t *testing.T) { + got, err := LoadObservations(filepath.Join(t.TempDir(), "missing.json")) + if err != nil || len(got) != 0 { + t.Fatalf("missing observations = %#v err=%v", got, err) + } +} diff --git a/internal/enroll/planner.go b/internal/enroll/planner.go new file mode 100644 index 0000000..5895ed2 --- /dev/null +++ b/internal/enroll/planner.go @@ -0,0 +1,243 @@ +package enroll + +import ( + "context" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/jstar0/codexfold/internal/codex" + "github.com/jstar0/codexfold/internal/storage" +) + +type Reason string + +const ( + ReasonAlreadyManaged Reason = "already-managed" + ReasonNotArchived Reason = "not-archived" + ReasonInvalidPath Reason = "invalid-rollout-path" + ReasonStabilityPending Reason = "stability-observation-pending" + ReasonFileChanged Reason = "rollout-changed" + ReasonWriterActive Reason = "writer-active" + ReasonDoctorUnhealthy Reason = "doctor-unhealthy" + ReasonCompatibility Reason = "client-compatibility-unapproved" + ReasonMountUnhealthy Reason = "mount-unhealthy" + ReasonNamespaceDisabled Reason = "canonical-namespace-disabled" + ReasonPromotionStage Reason = "promotion-stage-blocked" + ReasonInsufficientBudget Reason = "insufficient-storage-budget" + ReasonBatchLimit Reason = "batch-limit" +) + +type Policy struct { + StableFor time.Duration `json:"stable_for"` + BatchSize int `json:"batch_size"` + ArchivedOnly bool `json:"archived_only"` +} + +type Gates struct { + DoctorHealthy bool `json:"doctor_healthy"` + CompatibilityApproved bool `json:"compatibility_approved"` + MountHealthy bool `json:"mount_healthy"` + CanonicalNamespace bool `json:"canonical_namespace"` + EnrollmentAllowed bool `json:"enrollment_allowed"` +} + +type Observation struct { + Path string `json:"path"` + Size int64 `json:"size"` + ModTimeUnixNano int64 `json:"mod_time_unix_nano"` + StableSinceUnixNano int64 `json:"stable_since_unix_nano"` +} + +type Observations map[string]Observation + +type Fingerprint struct { + Size int64 `json:"size"` + ModTimeUnixNano int64 `json:"mod_time_unix_nano"` +} + +type WriterProbe func(context.Context, codex.Session) (bool, error) + +type Input struct { + Sessions []codex.Session + Managed map[string]struct{} + Previous Observations + Now time.Time + Policy Policy + Gates Gates + WriterActive WriterProbe + Budget storage.Checker +} + +type Decision struct { + SessionID string `json:"session_id"` + RolloutPath string `json:"rollout_path"` + Archived bool `json:"archived"` + Eligible bool `json:"eligible"` + Selected bool `json:"selected"` + Reasons []Reason `json:"reasons,omitempty"` + Fingerprint Fingerprint `json:"fingerprint"` +} + +type Plan struct { + GeneratedAt string `json:"generated_at"` + Decisions []Decision `json:"decisions"` + Selected []Decision `json:"selected"` + Observations Observations `json:"observations"` +} + +func Build(ctx context.Context, input Input) (Plan, error) { + if input.Now.IsZero() { + input.Now = time.Now() + } + if input.Policy.StableFor <= 0 { + input.Policy.StableFor = time.Hour + } + if input.Policy.BatchSize <= 0 { + input.Policy.BatchSize = 1 + } + if input.Previous == nil { + input.Previous = make(Observations) + } + plan := Plan{GeneratedAt: input.Now.UTC().Format(time.RFC3339Nano), Observations: make(Observations)} + sessions := append([]codex.Session(nil), input.Sessions...) + sort.Slice(sessions, func(i, j int) bool { + if sessions[i].UpdatedAt == sessions[j].UpdatedAt { + return sessions[i].ID < sessions[j].ID + } + return sessions[i].UpdatedAt < sessions[j].UpdatedAt + }) + var selectedPersistentBytes int64 + for _, session := range sessions { + if err := ctx.Err(); err != nil { + return Plan{}, err + } + decision := Decision{SessionID: session.ID, RolloutPath: filepath.Clean(session.RolloutPath), Archived: session.Archived} + if _, managed := input.Managed[session.ID]; managed { + decision.Reasons = append(decision.Reasons, ReasonAlreadyManaged) + plan.Decisions = append(plan.Decisions, decision) + continue + } + if !safeSessionID(session.ID) || !filepath.IsAbs(session.RolloutPath) { + decision.Reasons = append(decision.Reasons, ReasonInvalidPath) + plan.Decisions = append(plan.Decisions, decision) + continue + } + info, err := os.Lstat(session.RolloutPath) + if err != nil || !info.Mode().IsRegular() { + decision.Reasons = append(decision.Reasons, ReasonInvalidPath) + plan.Decisions = append(plan.Decisions, decision) + continue + } + decision.Fingerprint = Fingerprint{Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano()} + previous, observed := input.Previous[session.ID] + unchanged := observed && filepath.Clean(previous.Path) == decision.RolloutPath && previous.Size == info.Size() && previous.ModTimeUnixNano == info.ModTime().UnixNano() + observation := Observation{Path: decision.RolloutPath, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), StableSinceUnixNano: input.Now.UnixNano()} + if unchanged { + observation.StableSinceUnixNano = previous.StableSinceUnixNano + } + plan.Observations[session.ID] = observation + + if input.Policy.ArchivedOnly && !session.Archived { + decision.Reasons = append(decision.Reasons, ReasonNotArchived) + } + if !input.Gates.DoctorHealthy { + decision.Reasons = append(decision.Reasons, ReasonDoctorUnhealthy) + } + if !input.Gates.CompatibilityApproved { + decision.Reasons = append(decision.Reasons, ReasonCompatibility) + } + if !input.Gates.MountHealthy { + decision.Reasons = append(decision.Reasons, ReasonMountUnhealthy) + } + if !input.Gates.CanonicalNamespace { + decision.Reasons = append(decision.Reasons, ReasonNamespaceDisabled) + } + if !input.Gates.EnrollmentAllowed { + decision.Reasons = append(decision.Reasons, ReasonPromotionStage) + } + if input.WriterActive != nil { + active, err := input.WriterActive(ctx, session) + if err != nil { + return Plan{}, fmt.Errorf("probe writer for %s: %w", session.ID, err) + } + if active { + decision.Reasons = append(decision.Reasons, ReasonWriterActive) + } + } + switch { + case !observed: + decision.Reasons = append(decision.Reasons, ReasonStabilityPending) + case !unchanged: + decision.Reasons = append(decision.Reasons, ReasonFileChanged) + case input.Now.Sub(time.Unix(0, observation.StableSinceUnixNano)) < input.Policy.StableFor: + decision.Reasons = append(decision.Reasons, ReasonStabilityPending) + case input.Now.Sub(info.ModTime()) < input.Policy.StableFor: + decision.Reasons = append(decision.Reasons, ReasonStabilityPending) + } + if len(decision.Reasons) != 0 { + plan.Decisions = append(plan.Decisions, decision) + continue + } + projectedPersistent, err := enrollmentPersistentBytes(info.Size()) + if err != nil { + return Plan{}, err + } + cumulative, overflow := addBytes(selectedPersistentBytes, projectedPersistent) + if overflow { + return Plan{}, errors.New("enrollment batch byte estimate overflow") + } + if input.Budget == nil { + decision.Reasons = append(decision.Reasons, ReasonInsufficientBudget) + } else if _, err := input.Budget.Check(ctx, storage.Projection{ + Operation: "enroll:" + session.ID, AdditionalPersistentBytes: cumulative, TemporaryBytes: info.Size(), + }); err != nil { + if !errors.Is(err, storage.ErrBudgetExceeded) { + return Plan{}, err + } + decision.Reasons = append(decision.Reasons, ReasonInsufficientBudget) + } + if len(decision.Reasons) == 0 && len(plan.Selected) >= input.Policy.BatchSize { + decision.Reasons = append(decision.Reasons, ReasonBatchLimit) + } + if len(decision.Reasons) == 0 { + decision.Eligible = true + decision.Selected = true + selectedPersistentBytes = cumulative + plan.Selected = append(plan.Selected, decision) + } + plan.Decisions = append(plan.Decisions, decision) + } + return plan, nil +} + +func safeSessionID(sessionID string) bool { + return sessionID != "" && sessionID != "." && sessionID != ".." && !strings.ContainsAny(sessionID, "/\\\x00") +} + +func enrollmentPersistentBytes(rawBytes int64) (int64, error) { + if rawBytes < 0 { + return 0, errors.New("enrollment byte estimate cannot be negative") + } + overhead := rawBytes/16 + 1<<20 + if rawBytes > math.MaxInt64-overhead { + return 0, errors.New("enrollment byte estimate overflow") + } + estimated := rawBytes + overhead + if estimated > math.MaxInt64/3 { + return 0, errors.New("enrollment byte estimate overflow") + } + return estimated * 3, nil +} + +func addBytes(left int64, right int64) (int64, bool) { + if right > math.MaxInt64-left { + return 0, true + } + return left + right, false +} diff --git a/internal/enroll/planner_test.go b/internal/enroll/planner_test.go new file mode 100644 index 0000000..c1937f5 --- /dev/null +++ b/internal/enroll/planner_test.go @@ -0,0 +1,243 @@ +package enroll + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/jstar0/codexfold/internal/codex" + "github.com/jstar0/codexfold/internal/storage" +) + +func TestPlannerRequiresStableArchivedSessionAndAllGlobalGates(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "session.jsonl") + if err := os.WriteFile(path, []byte("stable-session\n"), 0o600); err != nil { + t.Fatal(err) + } + now := time.Unix(10_000, 0) + if err := os.Chtimes(path, now.Add(-2*time.Hour), now.Add(-2*time.Hour)); err != nil { + t.Fatal(err) + } + session := codex.Session{ID: "session", RolloutPath: path, Archived: true, UpdatedAt: now.Add(-2 * time.Hour).Unix()} + input := Input{ + Sessions: []codex.Session{session}, Now: now, + Policy: Policy{StableFor: time.Hour, BatchSize: 1, ArchivedOnly: true}, + Gates: Gates{DoctorHealthy: true, CompatibilityApproved: true, MountHealthy: true, CanonicalNamespace: true, EnrollmentAllowed: true}, + Budget: allowingBudget{}, + } + first, err := Build(context.Background(), input) + if err != nil { + t.Fatal(err) + } + assertDecisionReason(t, first, "session", ReasonStabilityPending) + if len(first.Selected) != 0 { + t.Fatalf("first observation selected a session: %#v", first) + } + + input.Now = now.Add(2 * time.Hour) + input.Previous = first.Observations + second, err := Build(context.Background(), input) + if err != nil { + t.Fatal(err) + } + if len(second.Selected) != 1 || second.Selected[0].SessionID != "session" { + t.Fatalf("stable archived session was not selected: %#v", second) + } + + for name, test := range map[string]struct { + mutate func(*Input) + reason Reason + }{ + "doctor": {mutate: func(input *Input) { input.Gates.DoctorHealthy = false }, reason: ReasonDoctorUnhealthy}, + "client": {mutate: func(input *Input) { input.Gates.CompatibilityApproved = false }, reason: ReasonCompatibility}, + "mount": {mutate: func(input *Input) { input.Gates.MountHealthy = false }, reason: ReasonMountUnhealthy}, + "namespace": {mutate: func(input *Input) { input.Gates.CanonicalNamespace = false }, reason: ReasonNamespaceDisabled}, + "stage": {mutate: func(input *Input) { input.Gates.EnrollmentAllowed = false }, reason: ReasonPromotionStage}, + } { + t.Run(name, func(t *testing.T) { + candidate := input + test.mutate(&candidate) + plan, err := Build(context.Background(), candidate) + if err != nil { + t.Fatal(err) + } + assertDecisionReason(t, plan, "session", test.reason) + }) + } +} + +func TestPlannerSeparatesActiveChangingManagedWriterBudgetAndBatchCases(t *testing.T) { + root := t.TempDir() + now := time.Unix(20_000, 0) + makeSession := func(id string, archived bool) codex.Session { + path := filepath.Join(root, id+".jsonl") + if err := os.WriteFile(path, []byte(id+"\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(path, now.Add(-2*time.Hour), now.Add(-2*time.Hour)); err != nil { + t.Fatal(err) + } + return codex.Session{ID: id, RolloutPath: path, Archived: archived, UpdatedAt: now.Add(-2 * time.Hour).Unix()} + } + active := makeSession("active", false) + changing := makeSession("changing", true) + managed := makeSession("managed", true) + writer := makeSession("writer", true) + firstBatch := makeSession("batch-a", true) + secondBatch := makeSession("batch-b", true) + budgeted := makeSession("budgeted", true) + previous := make(Observations) + for _, session := range []codex.Session{changing, managed, writer, firstBatch, secondBatch, budgeted} { + info, err := os.Stat(session.RolloutPath) + if err != nil { + t.Fatal(err) + } + previous[session.ID] = Observation{Path: session.RolloutPath, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), StableSinceUnixNano: now.Add(-2 * time.Hour).UnixNano()} + } + if err := os.WriteFile(changing.RolloutPath, []byte("changed\n"), 0o600); err != nil { + t.Fatal(err) + } + input := Input{ + Sessions: []codex.Session{active, changing, managed, writer, firstBatch, secondBatch, budgeted}, + Managed: map[string]struct{}{"managed": {}}, Previous: previous, Now: now, + Policy: Policy{StableFor: time.Hour, BatchSize: 1, ArchivedOnly: true}, + Gates: Gates{DoctorHealthy: true, CompatibilityApproved: true, MountHealthy: true, CanonicalNamespace: true, EnrollmentAllowed: true}, + WriterActive: func(_ context.Context, session codex.Session) (bool, error) { return session.ID == "writer", nil }, + Budget: rejectingSessionBudget{sessionID: "budgeted"}, + } + plan, err := Build(context.Background(), input) + if err != nil { + t.Fatal(err) + } + assertDecisionReason(t, plan, "active", ReasonNotArchived) + assertDecisionReason(t, plan, "changing", ReasonFileChanged) + assertDecisionReason(t, plan, "managed", ReasonAlreadyManaged) + assertDecisionReason(t, plan, "writer", ReasonWriterActive) + assertDecisionReason(t, plan, "batch-b", ReasonBatchLimit) + assertDecisionReason(t, plan, "budgeted", ReasonInsufficientBudget) + if len(plan.Selected) != 1 || plan.Selected[0].SessionID != "batch-a" { + t.Fatalf("bounded selection = %#v", plan.Selected) + } +} + +func TestPlannerDiscoversExistingNewAndForkedSessionsAcrossCycles(t *testing.T) { + root := t.TempDir() + now := time.Unix(30_000, 0) + makeSession := func(id string) codex.Session { + path := filepath.Join(root, id+".jsonl") + if err := os.WriteFile(path, []byte(id+"\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(path, now.Add(-2*time.Hour), now.Add(-2*time.Hour)); err != nil { + t.Fatal(err) + } + return codex.Session{ID: id, RolloutPath: path, Archived: true, UpdatedAt: now.Add(-2 * time.Hour).Unix()} + } + existing := makeSession("existing") + input := Input{ + Sessions: []codex.Session{existing}, Now: now, + Policy: Policy{StableFor: time.Hour, BatchSize: 3, ArchivedOnly: true}, + Gates: Gates{DoctorHealthy: true, CompatibilityApproved: true, MountHealthy: true, CanonicalNamespace: true, EnrollmentAllowed: true}, + Budget: allowingBudget{}, + } + first, err := Build(context.Background(), input) + if err != nil { + t.Fatal(err) + } + newSession := makeSession("new") + fork := makeSession("fork") + input.Sessions = []codex.Session{existing, newSession, fork} + input.Previous = first.Observations + input.Now = now.Add(2 * time.Hour) + second, err := Build(context.Background(), input) + if err != nil { + t.Fatal(err) + } + if len(second.Selected) != 1 || second.Selected[0].SessionID != "existing" { + t.Fatalf("existing session was not selected while new discoveries observed: %#v", second) + } + assertDecisionReason(t, second, "new", ReasonStabilityPending) + assertDecisionReason(t, second, "fork", ReasonStabilityPending) + + input.Managed = map[string]struct{}{"existing": {}} + input.Previous = second.Observations + input.Now = now.Add(4 * time.Hour) + third, err := Build(context.Background(), input) + if err != nil { + t.Fatal(err) + } + if len(third.Selected) != 2 || third.Selected[0].SessionID != "fork" || third.Selected[1].SessionID != "new" { + t.Fatalf("new and forked sessions were not selected after becoming stable: %#v", third.Selected) + } +} + +func TestApplyRevalidatesFingerprintAndSkipsAlreadyManagedSessions(t *testing.T) { + root := t.TempDir() + first := filepath.Join(root, "first.jsonl") + second := filepath.Join(root, "second.jsonl") + if err := os.WriteFile(first, []byte("first\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(second, []byte("second\n"), 0o600); err != nil { + t.Fatal(err) + } + firstInfo, _ := os.Stat(first) + secondInfo, _ := os.Stat(second) + plan := Plan{Selected: []Decision{ + {SessionID: "first", RolloutPath: first, Selected: true, Fingerprint: Fingerprint{Size: firstInfo.Size(), ModTimeUnixNano: firstInfo.ModTime().UnixNano()}}, + {SessionID: "second", RolloutPath: second, Selected: true, Fingerprint: Fingerprint{Size: secondInfo.Size(), ModTimeUnixNano: secondInfo.ModTime().UnixNano()}}, + }} + if err := os.WriteFile(first, []byte("first changed\n"), 0o600); err != nil { + t.Fatal(err) + } + applied := make([]string, 0) + result, err := Apply(context.Background(), plan, ApplyOptions{ + IsManaged: func(_ context.Context, sessionID string) (bool, error) { return sessionID == "second", nil }, + Apply: func(_ context.Context, decision Decision) error { + applied = append(applied, decision.SessionID) + return nil + }, + }) + if err != nil { + t.Fatal(err) + } + if len(applied) != 0 || result.Applied != 0 || result.SkippedChanged != 1 || result.SkippedManaged != 1 { + t.Fatalf("apply revalidation result = %#v applied=%v", result, applied) + } +} + +type allowingBudget struct{} + +func (allowingBudget) Check(_ context.Context, projection storage.Projection) (storage.Assessment, error) { + return storage.Assessment{Budget: storage.BudgetReport{Operation: projection.Operation, Allowed: true}}, nil +} + +type rejectingSessionBudget struct { + sessionID string +} + +func (b rejectingSessionBudget) Check(_ context.Context, projection storage.Projection) (storage.Assessment, error) { + if projection.Operation == "enroll:"+b.sessionID { + return storage.Assessment{}, storage.ErrBudgetExceeded + } + return storage.Assessment{Budget: storage.BudgetReport{Operation: projection.Operation, Allowed: true}}, nil +} + +func assertDecisionReason(t *testing.T, plan Plan, sessionID string, reason Reason) { + t.Helper() + for _, decision := range plan.Decisions { + if decision.SessionID != sessionID { + continue + } + for _, found := range decision.Reasons { + if found == reason { + return + } + } + t.Fatalf("decision %s reasons = %v, want %s", sessionID, decision.Reasons, reason) + } + t.Fatalf("decision not found: %s", sessionID) +} diff --git a/internal/family/family.go b/internal/family/family.go new file mode 100644 index 0000000..8a57c87 --- /dev/null +++ b/internal/family/family.go @@ -0,0 +1,507 @@ +package family + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "sort" + + "github.com/jstar0/codexfold/internal/codex" + "github.com/jstar0/codexfold/internal/contain" +) + +type GraphRelation string + +const ( + GraphSeed GraphRelation = "seed" + GraphAncestor GraphRelation = "ancestor" + GraphDescendant GraphRelation = "descendant" + GraphCollateral GraphRelation = "collateral" + GraphNone GraphRelation = "none" +) + +type Relation string + +const ( + RelationIdentical Relation = "identical-applicable-records" + RelationLeftContained Relation = "left-contained-in-right" + RelationRightContained Relation = "right-contained-in-left" + RelationIndependentTails Relation = "shared-prefix-independent-tails" + RelationSharedRecords Relation = "shared-exact-records" + RelationUnknown Relation = "unknown" +) + +type Member struct { + ID string `json:"id"` + Title string `json:"title"` + CWD string `json:"cwd"` + RolloutPath string `json:"rollout_path"` + Archived bool `json:"archived"` + GitBranch string `json:"git_branch,omitempty"` + RelationToSeed GraphRelation `json:"relation_to_seed"` +} + +type Report struct { + SeedID string `json:"seed_id"` + Members []Member `json:"members"` + Edges []codex.SpawnEdge `json:"edges"` + MissingSessionIDs []string `json:"missing_session_ids,omitempty"` +} + +type Comparison struct { + LeftID string `json:"left_id"` + RightID string `json:"right_id"` + LeftArchived bool `json:"left_archived"` + RightArchived bool `json:"right_archived"` + GraphRelation GraphRelation `json:"graph_relation"` + Relation Relation `json:"relation"` + VerifiedExact bool `json:"verified_exact"` + LeftRecords int64 `json:"left_records"` + RightRecords int64 `json:"right_records"` + SharedPrefixRecords int64 `json:"shared_prefix_records"` + SharedRecords int64 `json:"shared_records"` + LeftContainedInRight bool `json:"left_contained_in_right"` + RightContainedInLeft bool `json:"right_contained_in_left"` +} + +type sourceSnapshot struct { + FileInfo os.FileInfo +} + +var beforeComparisonSourceValidation = func() {} + +func Build(seedID string, sessions []codex.Session, edges []codex.SpawnEdge) (Report, error) { + byID := make(map[string]codex.Session, len(sessions)) + for _, session := range sessions { + byID[session.ID] = session + } + if _, exists := byID[seedID]; !exists { + return Report{}, fmt.Errorf("session not found: %s", seedID) + } + adjacent := make(map[string][]string) + for _, edge := range edges { + adjacent[edge.ParentID] = append(adjacent[edge.ParentID], edge.ChildID) + adjacent[edge.ChildID] = append(adjacent[edge.ChildID], edge.ParentID) + } + component := map[string]struct{}{seedID: {}} + queue := []string{seedID} + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + for _, next := range adjacent[current] { + if _, seen := component[next]; seen { + continue + } + component[next] = struct{}{} + queue = append(queue, next) + } + } + report := Report{SeedID: seedID} + for sessionID := range component { + session, exists := byID[sessionID] + if !exists { + report.MissingSessionIDs = append(report.MissingSessionIDs, sessionID) + continue + } + report.Members = append(report.Members, Member{ + ID: session.ID, Title: session.Title, CWD: session.CWD, RolloutPath: session.RolloutPath, + Archived: session.Archived, GitBranch: session.GitBranch, + RelationToSeed: graphRelation(session.ID, seedID, edges), + }) + } + for _, edge := range edges { + if _, parent := component[edge.ParentID]; !parent { + continue + } + if _, child := component[edge.ChildID]; child { + report.Edges = append(report.Edges, edge) + } + } + sort.Slice(report.Members, func(i, j int) bool { return report.Members[i].ID < report.Members[j].ID }) + sort.Slice(report.Edges, func(i, j int) bool { + if report.Edges[i].ParentID != report.Edges[j].ParentID { + return report.Edges[i].ParentID < report.Edges[j].ParentID + } + if report.Edges[i].ChildID != report.Edges[j].ChildID { + return report.Edges[i].ChildID < report.Edges[j].ChildID + } + return report.Edges[i].Status < report.Edges[j].Status + }) + sort.Strings(report.MissingSessionIDs) + return report, nil +} + +func Compare(ctx context.Context, left codex.Session, right codex.Session, edges []codex.SpawnEdge) (Comparison, error) { + if left.ID == "" || right.ID == "" || left.ID == right.ID || left.RolloutPath == "" || right.RolloutPath == "" { + return Comparison{}, errors.New("distinct sessions with rollout paths are required") + } + leftFile, err := os.Open(left.RolloutPath) + if err != nil { + return Comparison{}, fmt.Errorf("open left rollout: %w", err) + } + defer func() { _ = leftFile.Close() }() + rightFile, err := os.Open(right.RolloutPath) + if err != nil { + return Comparison{}, fmt.Errorf("open right rollout: %w", err) + } + defer func() { _ = rightFile.Close() }() + leftSnapshot, err := snapshotFile(leftFile) + if err != nil { + return Comparison{}, fmt.Errorf("stat left rollout: %w", err) + } + rightSnapshot, err := snapshotFile(rightFile) + if err != nil { + return Comparison{}, fmt.Errorf("stat right rollout: %w", err) + } + leftRecords, err := scanFile(ctx, leftFile) + if err != nil { + return Comparison{}, fmt.Errorf("scan left rollout: %w", err) + } + rightRecords, err := scanFile(ctx, rightFile) + if err != nil { + return Comparison{}, fmt.Errorf("scan right rollout: %w", err) + } + if len(leftRecords) == 0 || len(rightRecords) == 0 { + return Comparison{}, errors.New("both rollouts must contain comparable records") + } + result := Comparison{ + LeftID: left.ID, RightID: right.ID, LeftArchived: left.Archived, RightArchived: right.Archived, + GraphRelation: graphRelation(left.ID, right.ID, edges), Relation: RelationUnknown, + LeftRecords: int64(len(leftRecords)), RightRecords: int64(len(rightRecords)), + } + result.SharedPrefixRecords, err = sharedPrefix(ctx, leftFile, leftRecords, rightFile, rightRecords) + if err != nil { + return Comparison{}, err + } + result.SharedRecords, err = sharedRecordCount(ctx, leftFile, leftRecords, rightFile, rightRecords) + if err != nil { + return Comparison{}, err + } + leftContained, err := contain.Check(ctx, + contain.Input{ID: left.ID, Path: left.RolloutPath}, + contain.Input{ID: right.ID, Path: right.RolloutPath}, + contain.Options{IgnoreSessionMeta: true}, + ) + if err != nil { + return Comparison{}, err + } + beforeComparisonSourceValidation() + if err := verifySourceUnchanged(left.RolloutPath, leftFile, leftSnapshot); err != nil { + return Comparison{}, fmt.Errorf("left rollout changed during comparison: %w", err) + } + if err := verifySourceUnchanged(right.RolloutPath, rightFile, rightSnapshot); err != nil { + return Comparison{}, fmt.Errorf("right rollout changed during comparison: %w", err) + } + rightContained, err := contain.Check(ctx, + contain.Input{ID: right.ID, Path: right.RolloutPath}, + contain.Input{ID: left.ID, Path: left.RolloutPath}, + contain.Options{IgnoreSessionMeta: true}, + ) + if err != nil { + return Comparison{}, err + } + result.LeftContainedInRight = leftContained.Contained && leftContained.VerifiedExact + result.RightContainedInLeft = rightContained.Contained && rightContained.VerifiedExact + switch { + case result.LeftContainedInRight && result.RightContainedInLeft: + result.Relation = RelationIdentical + result.VerifiedExact = true + case result.LeftContainedInRight: + result.Relation = RelationLeftContained + result.VerifiedExact = true + case result.RightContainedInLeft: + result.Relation = RelationRightContained + result.VerifiedExact = true + case result.SharedPrefixRecords > 0: + result.Relation = RelationIndependentTails + result.VerifiedExact = true + case result.SharedRecords > 0: + result.Relation = RelationSharedRecords + result.VerifiedExact = true + } + return result, nil +} + +func graphRelation(leftID string, rightID string, edges []codex.SpawnEdge) GraphRelation { + if leftID == rightID { + return GraphSeed + } + if reachable(leftID, rightID, edges) { + return GraphAncestor + } + if reachable(rightID, leftID, edges) { + return GraphDescendant + } + if connected(leftID, rightID, edges) { + return GraphCollateral + } + return GraphNone +} + +func reachable(start string, target string, edges []codex.SpawnEdge) bool { + children := make(map[string][]string) + for _, edge := range edges { + children[edge.ParentID] = append(children[edge.ParentID], edge.ChildID) + } + seen := map[string]struct{}{start: {}} + queue := []string{start} + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + for _, child := range children[current] { + if child == target { + return true + } + if _, exists := seen[child]; exists { + continue + } + seen[child] = struct{}{} + queue = append(queue, child) + } + } + return false +} + +func connected(start string, target string, edges []codex.SpawnEdge) bool { + adjacent := make(map[string][]string) + for _, edge := range edges { + adjacent[edge.ParentID] = append(adjacent[edge.ParentID], edge.ChildID) + adjacent[edge.ChildID] = append(adjacent[edge.ChildID], edge.ParentID) + } + seen := map[string]struct{}{start: {}} + queue := []string{start} + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + for _, next := range adjacent[current] { + if next == target { + return true + } + if _, exists := seen[next]; exists { + continue + } + seen[next] = struct{}{} + queue = append(queue, next) + } + } + return false +} + +type record struct { + digest [sha256.Size]byte + size int64 + start int64 + end int64 +} + +func scanFile(ctx context.Context, file *os.File) ([]record, error) { + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, err + } + reader := bufio.NewReaderSize(file, 1024*1024) + records := make([]record, 0, 1024) + var offset int64 + var physicalIndex int64 + for { + if err := ctx.Err(); err != nil { + return nil, err + } + hasher := sha256.New() + start := offset + var size int64 + var firstCapture []byte + captureComplete := true + hasData := false + reachedEOF := false + for { + fragment, readErr := reader.ReadSlice('\n') + if len(fragment) > 0 { + hasData = true + _, _ = hasher.Write(fragment) + size += int64(len(fragment)) + offset += int64(len(fragment)) + if physicalIndex == 0 && captureComplete { + if len(firstCapture)+len(fragment) <= 8*1024*1024 { + firstCapture = append(firstCapture, fragment...) + } else { + firstCapture = nil + captureComplete = false + } + } + } + switch { + case readErr == nil: + goto complete + case errors.Is(readErr, bufio.ErrBufferFull): + continue + case errors.Is(readErr, io.EOF): + reachedEOF = true + goto complete + default: + return nil, readErr + } + } + complete: + if !hasData { + return records, nil + } + skip := physicalIndex == 0 && captureComplete && isSessionMeta(firstCapture) + if !skip { + var digest [sha256.Size]byte + copy(digest[:], hasher.Sum(nil)) + records = append(records, record{digest: digest, size: size, start: start, end: offset}) + } + physicalIndex++ + if reachedEOF { + return records, nil + } + } +} + +func isSessionMeta(data []byte) bool { + var envelope struct { + Type string `json:"type"` + } + return json.Unmarshal(bytes.TrimSuffix(data, []byte{'\n'}), &envelope) == nil && envelope.Type == "session_meta" +} + +func sharedPrefix(ctx context.Context, leftFile *os.File, left []record, rightFile *os.File, right []record) (int64, error) { + limit := min(len(left), len(right)) + var count int64 + for index := 0; index < limit; index++ { + if left[index].size != right[index].size || left[index].digest != right[index].digest { + break + } + exact, err := equalRanges(ctx, leftFile, left[index], rightFile, right[index]) + if err != nil { + return 0, err + } + if !exact { + break + } + count++ + } + return count, nil +} + +func sharedRecordCount(ctx context.Context, leftFile *os.File, left []record, rightFile *os.File, right []record) (int64, error) { + type key struct { + digest [sha256.Size]byte + size int64 + } + candidates := make(map[key][]int) + for index, item := range right { + candidates[key{digest: item.digest, size: item.size}] = append(candidates[key{digest: item.digest, size: item.size}], index) + } + used := make([]bool, len(right)) + next := make(map[key]int, len(candidates)) + collisions := make(map[key][]int) + var shared int64 + for _, leftRecord := range left { + fingerprint := key{digest: leftRecord.digest, size: leftRecord.size} + matched := false + for _, index := range collisions[fingerprint] { + if used[index] { + continue + } + exact, err := equalRanges(ctx, leftFile, leftRecord, rightFile, right[index]) + if err != nil { + return 0, err + } + if exact { + used[index] = true + shared++ + matched = true + break + } + } + if matched { + continue + } + positions := candidates[fingerprint] + for next[fingerprint] < len(positions) { + index := positions[next[fingerprint]] + next[fingerprint]++ + exact, err := equalRanges(ctx, leftFile, leftRecord, rightFile, right[index]) + if err != nil { + return 0, err + } + if exact { + used[index] = true + shared++ + matched = true + break + } + collisions[fingerprint] = append(collisions[fingerprint], index) + } + } + return shared, nil +} + +func equalRanges(ctx context.Context, leftFile *os.File, left record, rightFile *os.File, right record) (bool, error) { + if left.size != right.size { + return false, nil + } + leftReader := io.NewSectionReader(leftFile, left.start, left.size) + rightReader := io.NewSectionReader(rightFile, right.start, right.size) + leftBuffer := make([]byte, 128*1024) + rightBuffer := make([]byte, len(leftBuffer)) + for remaining := left.size; remaining > 0; { + if err := ctx.Err(); err != nil { + return false, err + } + chunk := int64(len(leftBuffer)) + if remaining < chunk { + chunk = remaining + } + if _, err := io.ReadFull(leftReader, leftBuffer[:chunk]); err != nil { + return false, err + } + if _, err := io.ReadFull(rightReader, rightBuffer[:chunk]); err != nil { + return false, err + } + if !bytes.Equal(leftBuffer[:chunk], rightBuffer[:chunk]) { + return false, nil + } + remaining -= chunk + } + return true, nil +} + +func snapshotFile(file *os.File) (sourceSnapshot, error) { + info, err := file.Stat() + if err != nil { + return sourceSnapshot{}, err + } + if !info.Mode().IsRegular() { + return sourceSnapshot{}, errors.New("rollout path is not a regular file") + } + return sourceSnapshot{FileInfo: info}, nil +} + +func verifySourceUnchanged(path string, file *os.File, before sourceSnapshot) error { + afterHandle, err := file.Stat() + if err != nil { + return err + } + afterPath, err := os.Stat(path) + if err != nil { + return err + } + if !os.SameFile(before.FileInfo, afterHandle) || !os.SameFile(before.FileInfo, afterPath) { + return errors.New("rollout identity changed") + } + if before.FileInfo.Size() != afterHandle.Size() || !before.FileInfo.ModTime().Equal(afterHandle.ModTime()) { + return errors.New("rollout size or modification time changed") + } + if afterHandle.Size() != afterPath.Size() || !afterHandle.ModTime().Equal(afterPath.ModTime()) { + return errors.New("rollout path state differs from open file") + } + return nil +} diff --git a/internal/family/family_test.go b/internal/family/family_test.go new file mode 100644 index 0000000..8a4c661 --- /dev/null +++ b/internal/family/family_test.go @@ -0,0 +1,175 @@ +package family + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jstar0/codexfold/internal/codex" +) + +func TestBuildReportsConnectedFamilyStateWithoutInferringUsefulness(t *testing.T) { + sessions := []codex.Session{ + {ID: "root", Title: "Root", RolloutPath: "/rollouts/root.jsonl", Archived: false}, + {ID: "child", Title: "Child", RolloutPath: "/rollouts/child.jsonl", Archived: true}, + {ID: "grandchild", Title: "Grandchild", RolloutPath: "/rollouts/grandchild.jsonl", Archived: false}, + {ID: "unrelated", Title: "Unrelated", RolloutPath: "/rollouts/unrelated.jsonl", Archived: true}, + } + edges := []codex.SpawnEdge{ + {ParentID: "root", ChildID: "child", Status: "closed"}, + {ParentID: "child", ChildID: "grandchild", Status: "open"}, + {ParentID: "root", ChildID: "missing", Status: "closed"}, + } + report, err := Build("child", sessions, edges) + if err != nil { + t.Fatal(err) + } + if len(report.Members) != 3 || len(report.Edges) != 3 || len(report.MissingSessionIDs) != 1 || report.MissingSessionIDs[0] != "missing" { + t.Fatalf("family report = %#v", report) + } + byID := make(map[string]Member) + for _, member := range report.Members { + byID[member.ID] = member + } + if byID["child"].RelationToSeed != GraphSeed || !byID["child"].Archived { + t.Fatalf("seed member = %#v", byID["child"]) + } + if byID["root"].RelationToSeed != GraphAncestor || byID["grandchild"].RelationToSeed != GraphDescendant { + t.Fatalf("graph relations = %#v", byID) + } +} + +func TestCompareClassifiesExactContainmentIndependentTailsAndUnknown(t *testing.T) { + root := t.TempDir() + write := func(name string, records ...string) codex.Session { + path := filepath.Join(root, name+".jsonl") + data := "" + for _, record := range records { + data += record + "\n" + } + if err := os.WriteFile(path, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + return codex.Session{ID: name, RolloutPath: path, Archived: name == "left"} + } + metaLeft := `{"type":"session_meta","id":"left"}` + metaRight := `{"type":"session_meta","id":"right"}` + a := `{"value":"a"}` + b := `{"value":"b"}` + c := `{"value":"c"}` + x := `{"value":"x"}` + y := `{"value":"y"}` + + identical, err := Compare(context.Background(), + write("identical-left", metaLeft, a, b), write("identical-right", metaRight, a, b), nil, + ) + if err != nil || identical.Relation != RelationIdentical || !identical.VerifiedExact { + t.Fatalf("identical comparison = %#v err=%v", identical, err) + } + + contained, err := Compare(context.Background(), + write("left", metaLeft, a, b), write("container", metaRight, x, a, b, y), nil, + ) + if err != nil || contained.Relation != RelationLeftContained || !contained.LeftContainedInRight || !contained.VerifiedExact { + t.Fatalf("contained comparison = %#v err=%v", contained, err) + } + + tails, err := Compare(context.Background(), + write("tail-left", metaLeft, a, b), write("tail-right", metaRight, a, c), nil, + ) + if err != nil || tails.Relation != RelationIndependentTails || tails.SharedPrefixRecords != 1 || tails.SharedRecords != 1 { + t.Fatalf("independent-tail comparison = %#v err=%v", tails, err) + } + + unknown, err := Compare(context.Background(), + write("unknown-left", metaLeft, x), write("unknown-right", metaRight, y), nil, + ) + if err != nil || unknown.Relation != RelationUnknown || unknown.SharedRecords != 0 { + t.Fatalf("unknown comparison = %#v err=%v", unknown, err) + } +} + +func TestCompareReportsGraphEvidenceSeparatelyFromContent(t *testing.T) { + root := t.TempDir() + leftPath := filepath.Join(root, "left.jsonl") + rightPath := filepath.Join(root, "right.jsonl") + if err := os.WriteFile(leftPath, []byte("{\"v\":1}\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(rightPath, []byte("{\"v\":2}\n"), 0o600); err != nil { + t.Fatal(err) + } + comparison, err := Compare(context.Background(), + codex.Session{ID: "left", RolloutPath: leftPath}, + codex.Session{ID: "right", RolloutPath: rightPath}, + []codex.SpawnEdge{{ParentID: "left", ChildID: "right", Status: "open"}}, + ) + if err != nil || comparison.GraphRelation != GraphAncestor || comparison.Relation != RelationUnknown { + t.Fatalf("graph/content evidence was conflated: %#v err=%v", comparison, err) + } +} + +func TestCompareRejectsSourceMutationBeforeReturningEvidence(t *testing.T) { + root := t.TempDir() + leftPath := filepath.Join(root, "left.jsonl") + rightPath := filepath.Join(root, "right.jsonl") + data := []byte("{\"type\":\"session_meta\"}\n{\"value\":1}\n") + if err := os.WriteFile(leftPath, data, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(rightPath, data, 0o600); err != nil { + t.Fatal(err) + } + previous := beforeComparisonSourceValidation + beforeComparisonSourceValidation = func() { + file, err := os.OpenFile(rightPath, os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteString("{\"mutated\":true}\n"); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { beforeComparisonSourceValidation = previous }) + _, err := Compare(context.Background(), + codex.Session{ID: "left", RolloutPath: leftPath}, + codex.Session{ID: "right", RolloutPath: rightPath}, nil, + ) + if err == nil || !strings.Contains(err.Error(), "changed during comparison") { + t.Fatalf("source mutation error = %v", err) + } +} + +func TestCompareRepeatedRecordsAvoidsQuadraticFileReopens(t *testing.T) { + root := t.TempDir() + writeRepeated := func(name string) codex.Session { + path := filepath.Join(root, name+".jsonl") + var data strings.Builder + data.WriteString("{\"type\":\"session_meta\",\"id\":\"") + data.WriteString(name) + data.WriteString("\"}\n") + for range 1000 { + data.WriteString("{\"value\":\"same repeated record\"}\n") + } + if err := os.WriteFile(path, []byte(data.String()), 0o600); err != nil { + t.Fatal(err) + } + return codex.Session{ID: name, RolloutPath: path} + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + comparison, err := Compare(ctx, writeRepeated("left"), writeRepeated("right"), nil) + if err != nil { + t.Fatal(err) + } + if comparison.Relation != RelationIdentical || comparison.SharedRecords != 1000 { + t.Fatalf("repeated comparison = %#v", comparison) + } +} diff --git a/internal/fold/doctor.go b/internal/fold/doctor.go index e1188ce..b8986ff 100644 --- a/internal/fold/doctor.go +++ b/internal/fold/doctor.go @@ -7,6 +7,8 @@ import ( "io/fs" "os" "path/filepath" + + "github.com/jstar0/codexfold/internal/storage" ) type DoctorIssue struct { @@ -16,13 +18,16 @@ type DoctorIssue struct { } type DoctorResult struct { - StoreDir string `json:"store_dir"` - ManifestCount int `json:"manifest_count"` - VerifiedManifestCount int `json:"verified_manifest_count"` - ObjectReferenceCount int `json:"object_reference_count"` - UniqueObjectCount int `json:"unique_object_count"` - IssueCount int `json:"issue_count"` - Issues []DoctorIssue `json:"issues"` + StoreDir string `json:"store_dir"` + ManifestCount int `json:"manifest_count"` + VerifiedManifestCount int `json:"verified_manifest_count"` + ObjectReferenceCount int `json:"object_reference_count"` + UniqueObjectCount int `json:"unique_object_count"` + IssueCount int `json:"issue_count"` + Issues []DoctorIssue `json:"issues"` + Storage storage.Inventory `json:"storage"` + StorageLimits storage.Limits `json:"storage_limits"` + AvailableBytes int64 `json:"available_bytes"` } type loadedManifest struct { @@ -68,6 +73,22 @@ func Doctor(ctx context.Context, storeDir string) (DoctorResult, error) { }) } } + result.Storage, err = storage.Scan(ctx, storage.Options{StoreDir: storeDir, AllowMetadataIssues: true}) + if err != nil { + result.Issues = append(result.Issues, DoctorIssue{Scope: "storage", Path: storeDir, Error: err.Error()}) + } else { + for _, issue := range result.Storage.Issues { + result.Issues = append(result.Issues, DoctorIssue{Scope: "storage", Path: storeDir, Error: issue}) + } + } + result.StorageLimits, err = storage.LoadLimits(storeDir) + if err != nil { + result.Issues = append(result.Issues, DoctorIssue{Scope: "storage-policy", Path: filepath.Join(storeDir, storage.PolicyFilename), Error: err.Error()}) + } + result.AvailableBytes, err = storage.AvailableBytes(storeDir) + if err != nil { + result.Issues = append(result.Issues, DoctorIssue{Scope: "storage-space", Path: storeDir, Error: err.Error()}) + } result.IssueCount = len(result.Issues) return result, nil } diff --git a/internal/fold/doctor_gc_test.go b/internal/fold/doctor_gc_test.go index be40dba..02f4dc9 100644 --- a/internal/fold/doctor_gc_test.go +++ b/internal/fold/doctor_gc_test.go @@ -7,8 +7,7 @@ import ( "path/filepath" "strings" "testing" - - "github.com/jstar0/codexfold/internal/codex" + "time" ) func TestDoctorDetectsReferencedObjectCorruption(t *testing.T) { @@ -18,7 +17,7 @@ func TestDoctorDetectsReferencedObjectCorruption(t *testing.T) { if err := os.WriteFile(sourcePath, []byte("{\"value\":\"large-field-value\"}\n"), 0o644); err != nil { t.Fatalf("write source: %v", err) } - if _, err := Fold(context.Background(), codex.Session{ID: "doctor", RolloutPath: sourcePath, Archived: true}, FoldOptions{ + if _, err := Fold(context.Background(), Session{ID: "doctor", RolloutPath: sourcePath, Archived: true}, FoldOptions{ StoreDir: storeDir, Apply: true, FieldThreshold: 4, }); err != nil { t.Fatalf("Fold returned error: %v", err) @@ -30,6 +29,9 @@ func TestDoctorDetectsReferencedObjectCorruption(t *testing.T) { if clean.IssueCount != 0 || clean.ManifestCount != 1 { t.Fatalf("unexpected clean doctor result: %#v", clean) } + if clean.Storage.LogicalSessionBytes == 0 || clean.Storage.TotalPhysicalBytes == 0 || clean.StorageLimits.MaxPhysicalBytes == 0 || clean.AvailableBytes == 0 { + t.Fatalf("doctor storage accounting is incomplete: %#v", clean) + } manifest, err := LoadManifest(storeDir, "doctor") if err != nil { t.Fatalf("load manifest: %v", err) @@ -54,7 +56,7 @@ func TestGCDryRunAndApplyRemoveOnlyUnreferencedObjects(t *testing.T) { if err := os.WriteFile(sourcePath, []byte("{\"value\":\"large-field-value\"}\n"), 0o644); err != nil { t.Fatalf("write source: %v", err) } - if _, err := Fold(context.Background(), codex.Session{ID: "gc", RolloutPath: sourcePath, Archived: true}, FoldOptions{ + if _, err := Fold(context.Background(), Session{ID: "gc", RolloutPath: sourcePath, Archived: true}, FoldOptions{ StoreDir: storeDir, Apply: true, FieldThreshold: 4, }); err != nil { t.Fatalf("Fold returned error: %v", err) @@ -90,6 +92,61 @@ func TestGCDryRunAndApplyRemoveOnlyUnreferencedObjects(t *testing.T) { } } +func TestGCIncludesBoundedGenerationAndTemporaryCleanup(t *testing.T) { + root := t.TempDir() + storeDir := filepath.Join(root, "store") + sourcePath := filepath.Join(root, "rollout.jsonl") + if err := os.WriteFile(sourcePath, []byte("{\"value\":\"storage-gc\"}\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Fold(context.Background(), Session{ID: "session", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 4}); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(storeDir, "packs"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(storeDir, "packs", "CURRENT"), []byte("gen-3\n"), 0o600); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-2 * time.Hour) + for _, generation := range []string{"gen-1", "gen-2", "gen-3"} { + directory := filepath.Join(storeDir, "packs", generation) + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(directory, "pack-000001.pack"), []byte(generation), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(directory, old.Add(time.Duration(generation[len(generation)-1]-'0')*time.Minute), old.Add(time.Duration(generation[len(generation)-1]-'0')*time.Minute)); err != nil { + t.Fatal(err) + } + } + temporary := filepath.Join(storeDir, ".backing-abandoned.tmp") + if err := os.WriteFile(temporary, []byte("temporary"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(temporary, old, old); err != nil { + t.Fatal(err) + } + + result, err := GC(context.Background(), storeDir, true) + if err != nil { + t.Fatalf("GC: %v", err) + } + if result.Storage.RemovedCount != 2 || result.ActualReclaimedBytes <= 0 { + t.Fatalf("bounded storage was not collected: %#v", result) + } + if _, err := os.Stat(filepath.Join(storeDir, "packs", "gen-1")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("old pack generation remains: %v", err) + } + if _, err := os.Stat(filepath.Join(storeDir, "packs", "gen-2")); err != nil { + t.Fatalf("previous pack generation was removed: %v", err) + } + if _, err := os.Stat(temporary); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("abandoned temporary remains: %v", err) + } +} + func TestDoctorAndGCKeepGenerationManifestObjects(t *testing.T) { root := t.TempDir() storeDir := filepath.Join(root, "store") @@ -98,7 +155,7 @@ func TestDoctorAndGCKeepGenerationManifestObjects(t *testing.T) { t.Fatal(err) } manifestPath := filepath.Join(storeDir, "manifests", "generations", "session", "2.json") - if _, err := Fold(context.Background(), codex.Session{ID: "session", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, ManifestPathOverride: manifestPath, Apply: true, FieldThreshold: 4}); err != nil { + if _, err := Fold(context.Background(), Session{ID: "session", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, ManifestPathOverride: manifestPath, Apply: true, FieldThreshold: 4}); err != nil { t.Fatalf("Fold generation: %v", err) } doctor, err := Doctor(context.Background(), storeDir) @@ -119,13 +176,13 @@ func TestRemoveSourceRequiresGuardAndCanMaterializeAgain(t *testing.T) { if err := os.WriteFile(sourcePath, source, 0o644); err != nil { t.Fatalf("write source: %v", err) } - _, err := Fold(context.Background(), codex.Session{ID: "active", RolloutPath: sourcePath}, FoldOptions{ + _, err := Fold(context.Background(), Session{ID: "active", RolloutPath: sourcePath}, FoldOptions{ StoreDir: storeDir, Apply: true, RemoveSource: true, FieldThreshold: 4, }) if err == nil { t.Fatalf("non-archived source removal should require --allow-active") } - result, err := Fold(context.Background(), codex.Session{ID: "archived", RolloutPath: sourcePath, Archived: true}, FoldOptions{ + result, err := Fold(context.Background(), Session{ID: "archived", RolloutPath: sourcePath, Archived: true}, FoldOptions{ StoreDir: storeDir, Apply: true, RemoveSource: true, FieldThreshold: 4, }) if err != nil { diff --git a/internal/fold/fold.go b/internal/fold/fold.go index 1a8982d..865c8ee 100644 --- a/internal/fold/fold.go +++ b/internal/fold/fold.go @@ -10,13 +10,14 @@ import ( "fmt" "hash" "io" + "math" "os" "path/filepath" "strings" "github.com/jstar0/codexfold/internal/cdc" - "github.com/jstar0/codexfold/internal/codex" "github.com/jstar0/codexfold/internal/jsonraw" + "github.com/jstar0/codexfold/internal/storage" ) type FoldOptions struct { @@ -29,29 +30,39 @@ type FoldOptions struct { FieldThreshold int64 MaxJSONLineBytes int64 CDC cdc.Options + Budget storage.Checker beforeCommit func() error } +type Session struct { + ID string + Title string + CWD string + RolloutPath string + Archived bool +} + type FoldResult struct { - SessionID string `json:"session_id"` - SourcePath string `json:"source_path"` - ManifestPath string `json:"manifest_path"` - SourceBytes int64 `json:"source_bytes"` - SourceSHA256 string `json:"source_sha256"` - PartCount int `json:"part_count"` - FieldParts int `json:"field_parts"` - ResidualParts int `json:"residual_parts"` - UniqueObjects int `json:"unique_objects"` - ReusedObjects int `json:"reused_objects"` - NewStoredBytes int64 `json:"new_stored_bytes"` - OversizedLines int64 `json:"oversized_lines"` - InvalidJSONLines int64 `json:"invalid_json_lines"` - Verified bool `json:"verified"` - DryRun bool `json:"dry_run"` - RemovedSource bool `json:"removed_source"` + SessionID string `json:"session_id"` + SourcePath string `json:"source_path"` + ManifestPath string `json:"manifest_path"` + SourceBytes int64 `json:"source_bytes"` + SourceSHA256 string `json:"source_sha256"` + PartCount int `json:"part_count"` + FieldParts int `json:"field_parts"` + ResidualParts int `json:"residual_parts"` + UniqueObjects int `json:"unique_objects"` + ReusedObjects int `json:"reused_objects"` + NewStoredBytes int64 `json:"new_stored_bytes"` + OversizedLines int64 `json:"oversized_lines"` + InvalidJSONLines int64 `json:"invalid_json_lines"` + Verified bool `json:"verified"` + DryRun bool `json:"dry_run"` + RemovedSource bool `json:"removed_source"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` } -func Fold(ctx context.Context, session codex.Session, options FoldOptions) (FoldResult, error) { +func Fold(ctx context.Context, session Session, options FoldOptions) (FoldResult, error) { if options.StoreDir == "" { return FoldResult{}, errors.New("fold store directory is required") } @@ -101,6 +112,38 @@ func Fold(ctx context.Context, session codex.Session, options FoldOptions) (Fold if err != nil { return FoldResult{}, fmt.Errorf("stat rollout: %w", err) } + var storageAssessment storage.Assessment + if options.Apply { + budget := options.Budget + if budget == nil { + guard, err := storage.DefaultGuard(options.StoreDir) + if err != nil { + return FoldResult{}, err + } + budget = guard + } + persistentBytes, err := estimateFoldStorageBytes(before.Size()) + if err != nil { + return FoldResult{}, err + } + maximumObjectBytes := min(before.Size(), max(options.CDC.MaxBytes, options.MaxJSONLineBytes)) + temporaryBytes, err := estimateFoldStorageBytes(maximumObjectBytes) + if err != nil { + return FoldResult{}, err + } + reclaimableBytes := int64(0) + if options.RemoveSource { + reclaimableBytes = before.Size() + } + storageAssessment, err = budget.Check(ctx, storage.Projection{ + Operation: "fold", AdditionalPersistentBytes: persistentBytes, + TemporaryBytes: temporaryBytes, TemporaryPersistentOverlapBytes: min(temporaryBytes, persistentBytes), + ReclaimableBytes: reclaimableBytes, + }) + if err != nil { + return FoldResult{}, err + } + } manifest := Manifest{ Version: ManifestVersion, @@ -280,9 +323,22 @@ complete: } result.RemovedSource = true } + result.Storage = storage.CompleteAccounting(ctx, storageAssessment, options.StoreDir) return result, nil } +func estimateFoldStorageBytes(rawBytes int64) (int64, error) { + const fixedOverhead = int64(1 << 20) + if rawBytes < 0 { + return 0, errors.New("fold byte estimate cannot be negative") + } + overhead := rawBytes/16 + fixedOverhead + if rawBytes > math.MaxInt64-overhead { + return 0, errors.New("fold byte estimate overflow") + } + return rawBytes + overhead, nil +} + func verifyCurrentSource(path string, initial os.FileInfo, source ManifestSource) error { file, err := os.Open(path) if err != nil { diff --git a/internal/fold/fold_test.go b/internal/fold/fold_test.go index b2da913..4fe8641 100644 --- a/internal/fold/fold_test.go +++ b/internal/fold/fold_test.go @@ -10,7 +10,7 @@ import ( "testing" "github.com/jstar0/codexfold/internal/cdc" - "github.com/jstar0/codexfold/internal/codex" + "github.com/jstar0/codexfold/internal/storage" ) func TestFoldRejectsSourceMutationBeforeManifestCommit(t *testing.T) { @@ -21,7 +21,7 @@ func TestFoldRejectsSourceMutationBeforeManifestCommit(t *testing.T) { t.Fatalf("write source: %v", err) } - _, err := Fold(context.Background(), codex.Session{ + _, err := Fold(context.Background(), Session{ ID: "changing", RolloutPath: sourcePath, Archived: true, }, FoldOptions{ StoreDir: storeDir, Apply: true, FieldThreshold: 4, @@ -55,6 +55,91 @@ func TestFoldRejectsSourceMutationBeforeManifestCommit(t *testing.T) { } } +func TestFoldBudgetRejectsBeforeWritingObjectsOrManifest(t *testing.T) { + root := t.TempDir() + sourcePath := filepath.Join(root, "rollout.jsonl") + storeDir := filepath.Join(root, "store") + if err := os.WriteFile(sourcePath, []byte("{\"value\":\"budgeted-field\"}\n"), 0o600); err != nil { + t.Fatal(err) + } + checker := &foldRejectingChecker{} + _, err := Fold(context.Background(), Session{ID: "budget", RolloutPath: sourcePath, Archived: true}, FoldOptions{ + StoreDir: storeDir, Apply: true, FieldThreshold: 4, Budget: checker, + }) + if !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("Fold error = %v, want storage budget rejection", err) + } + if checker.Calls != 1 || checker.Projection.Operation != "fold" || checker.Projection.AdditionalPersistentBytes <= 0 { + t.Fatalf("unexpected fold budget projection: %#v", checker) + } + if _, err := os.Stat(storeDir); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("fold store exists after preflight rejection: %v", err) + } +} + +func TestFoldReportsProjectedAndActualPhysicalAccounting(t *testing.T) { + root := t.TempDir() + sourcePath := filepath.Join(root, "source.jsonl") + storeDir := filepath.Join(root, "store") + source := []byte("{\"value\":\"physical-accounting\"}\n") + if err := os.WriteFile(sourcePath, source, 0o600); err != nil { + t.Fatal(err) + } + result, err := Fold(context.Background(), Session{ID: "accounting", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 4}) + if err != nil { + t.Fatal(err) + } + if result.Storage == nil || result.Storage.Budget.ProjectedPeakBytes <= 0 || result.Storage.After.LogicalSessionBytes != int64(len(source)) { + t.Fatalf("fold storage accounting is incomplete: %#v", result.Storage) + } + if result.Storage.ActualReclaimedBytes != 0 { + t.Fatalf("fold with retained source claimed reclamation: %#v", result.Storage) + } +} + +func TestUnfoldBudgetRejectsBeforeCreatingRestoreTarget(t *testing.T) { + root := t.TempDir() + sourcePath := filepath.Join(root, "source.jsonl") + storeDir := filepath.Join(root, "store") + if err := os.WriteFile(sourcePath, []byte("{\"value\":\"restore-budget\"}\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Fold(context.Background(), Session{ID: "restore", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 4}); err != nil { + t.Fatal(err) + } + checker := &foldRejectingChecker{} + target := filepath.Join(root, "output", "restored.jsonl") + if _, err := UnfoldWithOptions(context.Background(), storeDir, "restore", UnfoldOptions{TargetPath: target, Budget: checker}); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("UnfoldWithOptions error = %v, want storage budget rejection", err) + } + if checker.Calls != 1 || checker.Projection.Operation != "unfold" { + t.Fatalf("unexpected unfold budget projection: %#v", checker) + } + if _, err := os.Stat(filepath.Dir(target)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("restore target directory exists after preflight rejection: %v", err) + } +} + +func TestUnfoldReportsStorageBudgetWithoutClaimingReclamation(t *testing.T) { + root := t.TempDir() + sourcePath := filepath.Join(root, "source.jsonl") + storeDir := filepath.Join(root, "store") + if err := os.WriteFile(sourcePath, []byte("{\"value\":\"unfold-accounting\"}\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Fold(context.Background(), Session{ID: "unfold-accounting", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 4}); err != nil { + t.Fatal(err) + } + target := filepath.Join(root, "restored.jsonl") + result, err := Unfold(context.Background(), storeDir, "unfold-accounting", target, false) + if err != nil { + t.Fatal(err) + } + if result.Storage == nil || result.Storage.Budget.ProjectedPeakBytes <= result.Storage.Budget.CurrentPhysicalBytes || result.Storage.ActualReclaimedBytes != 0 { + t.Fatalf("unfold storage accounting is incomplete: %#v", result.Storage) + } +} + func TestFoldRejectsSameSizeMutationEvenWhenMtimeIsRestored(t *testing.T) { root := t.TempDir() sourcePath := filepath.Join(root, "rollout.jsonl") @@ -72,7 +157,7 @@ func TestFoldRejectsSameSizeMutationEvenWhenMtimeIsRestored(t *testing.T) { t.Fatalf("stat source: %v", err) } - _, err = Fold(context.Background(), codex.Session{ + _, err = Fold(context.Background(), Session{ ID: "same-size-change", RolloutPath: sourcePath, Archived: true, }, FoldOptions{ StoreDir: storeDir, Apply: true, FieldThreshold: 4, @@ -102,7 +187,7 @@ func TestFoldCreatesVerifiedManifestAndReusesRepeatedField(t *testing.T) { t.Fatalf("write source: %v", err) } - result, err := Fold(context.Background(), codex.Session{ + result, err := Fold(context.Background(), Session{ ID: "fixture", Title: "Fixture", CWD: "/workspace", RolloutPath: sourcePath, Archived: true, }, FoldOptions{ StoreDir: storeDir, @@ -148,7 +233,7 @@ func TestFoldDryRunDoesNotCreateStore(t *testing.T) { if err := os.WriteFile(sourcePath, []byte("{\"value\":\"large-value\"}\n"), 0o644); err != nil { t.Fatalf("write source: %v", err) } - result, err := Fold(context.Background(), codex.Session{ID: "dry", RolloutPath: sourcePath, Archived: true}, FoldOptions{ + result, err := Fold(context.Background(), Session{ID: "dry", RolloutPath: sourcePath, Archived: true}, FoldOptions{ StoreDir: storeDir, FieldThreshold: 4, }) if err != nil { @@ -178,7 +263,7 @@ func TestFoldWritesToExplicitGenerationManifestWithoutReplacingPrimary(t *testin t.Fatal(err) } generationPath := filepath.Join(storeDir, "manifests", "generations", "session", "2.json") - result, err := Fold(context.Background(), codex.Session{ID: "session", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8, ManifestPathOverride: generationPath}) + result, err := Fold(context.Background(), Session{ID: "session", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8, ManifestPathOverride: generationPath}) if err != nil { t.Fatalf("Fold generation manifest: %v", err) } @@ -214,7 +299,7 @@ func TestFoldRoundTripsEmptyInvalidAndOversizedRollouts(t *testing.T) { if err := os.WriteFile(sourcePath, test.source, 0o644); err != nil { t.Fatalf("write source: %v", err) } - result, err := Fold(context.Background(), codex.Session{ID: test.name, RolloutPath: sourcePath, Archived: true}, FoldOptions{ + result, err := Fold(context.Background(), Session{ID: test.name, RolloutPath: sourcePath, Archived: true}, FoldOptions{ StoreDir: storeDir, Apply: true, FieldThreshold: 4, MaxJSONLineBytes: test.maxLineBytes, }) if err != nil { @@ -243,7 +328,7 @@ func TestFoldHonorsCanceledContextWithoutManifest(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := Fold(ctx, codex.Session{ID: "canceled", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true}) + _, err := Fold(ctx, Session{ID: "canceled", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true}) if !errors.Is(err, context.Canceled) { t.Fatalf("Fold error = %v, want context.Canceled", err) } @@ -251,3 +336,14 @@ func TestFoldHonorsCanceledContextWithoutManifest(t *testing.T) { t.Fatalf("manifest committed after cancellation: %v", statErr) } } + +type foldRejectingChecker struct { + Calls int + Projection storage.Projection +} + +func (c *foldRejectingChecker) Check(_ context.Context, projection storage.Projection) (storage.Assessment, error) { + c.Calls++ + c.Projection = projection + return storage.Assessment{}, storage.ErrBudgetExceeded +} diff --git a/internal/fold/gc.go b/internal/fold/gc.go index e0d564b..af55a41 100644 --- a/internal/fold/gc.go +++ b/internal/fold/gc.go @@ -6,27 +6,49 @@ import ( "os" "path/filepath" "strings" + + "github.com/jstar0/codexfold/internal/storage" ) type GCResult struct { - StoreDir string `json:"store_dir"` - DryRun bool `json:"dry_run"` - Referenced int `json:"referenced_objects"` - OrphanCount int `json:"orphan_count"` - OrphanBytes int64 `json:"orphan_bytes"` - RemovedCount int `json:"removed_count"` - RemovedBytes int64 `json:"removed_bytes"` + StoreDir string `json:"store_dir"` + DryRun bool `json:"dry_run"` + Referenced int `json:"referenced_objects"` + OrphanCount int `json:"orphan_count"` + OrphanBytes int64 `json:"orphan_bytes"` + RemovedCount int `json:"removed_count"` + RemovedBytes int64 `json:"removed_bytes"` + Storage storage.StorageGCResult `json:"storage"` + ProjectedReclaimableBytes int64 `json:"projected_reclaimable_bytes"` + ActualReclaimedBytes int64 `json:"actual_reclaimed_bytes"` } func GC(ctx context.Context, storeDir string, apply bool) (GCResult, error) { result := GCResult{StoreDir: storeDir, DryRun: !apply} - manifests, issues, err := loadAllManifests(storeDir) + _, issues, err := loadAllManifests(storeDir) if err != nil { return GCResult{}, err } if len(issues) > 0 { return GCResult{}, fmt.Errorf("refusing GC with %d invalid manifest(s)", len(issues)) } + before, err := storage.Scan(ctx, storage.Options{StoreDir: storeDir, AllowMetadataIssues: true}) + if err != nil { + return GCResult{}, err + } + storageResult, err := storage.Collect(ctx, storage.GCOptions{StoreDir: storeDir, Apply: apply}) + if err != nil { + return GCResult{}, err + } + result.Storage = storageResult + result.ProjectedReclaimableBytes = storageResult.ProjectedReclaimableBytes + manifests, issues, err := loadAllManifests(storeDir) + if err != nil { + return GCResult{}, err + } + if len(issues) > 0 { + return GCResult{}, fmt.Errorf("refusing loose-object GC with %d invalid manifest(s)", len(issues)) + } referenced := make(map[string]struct{}) for _, loaded := range manifests { for _, part := range loaded.Manifest.Parts { @@ -56,5 +78,13 @@ func GC(ctx context.Context, storeDir string, apply bool) (GCResult, error) { if err != nil { return GCResult{}, err } + result.ProjectedReclaimableBytes += result.OrphanBytes + after, err := storage.Scan(ctx, storage.Options{StoreDir: storeDir, AllowMetadataIssues: true}) + if err != nil { + return GCResult{}, err + } + if before.TotalPhysicalBytes > after.TotalPhysicalBytes { + result.ActualReclaimedBytes = before.TotalPhysicalBytes - after.TotalPhysicalBytes + } return result, nil } diff --git a/internal/fold/unfold.go b/internal/fold/unfold.go index 84e1b61..d6bc9dd 100644 --- a/internal/fold/unfold.go +++ b/internal/fold/unfold.go @@ -6,30 +6,62 @@ import ( "fmt" "os" "path/filepath" + + "github.com/jstar0/codexfold/internal/storage" ) type UnfoldResult struct { - SessionID string `json:"session_id"` - ManifestPath string `json:"manifest_path"` - TargetPath string `json:"target_path"` - Bytes int64 `json:"bytes"` - SHA256 string `json:"sha256"` - Verified bool `json:"verified"` + SessionID string `json:"session_id"` + ManifestPath string `json:"manifest_path"` + TargetPath string `json:"target_path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` + Verified bool `json:"verified"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` +} + +type UnfoldOptions struct { + TargetPath string + Overwrite bool + Budget storage.Checker } func Unfold(ctx context.Context, storeDir string, sessionID string, targetPath string, overwrite bool) (UnfoldResult, error) { + return UnfoldWithOptions(ctx, storeDir, sessionID, UnfoldOptions{TargetPath: targetPath, Overwrite: overwrite}) +} + +func UnfoldWithOptions(ctx context.Context, storeDir string, sessionID string, options UnfoldOptions) (UnfoldResult, error) { manifest, err := LoadManifest(storeDir, sessionID) if err != nil { return UnfoldResult{}, err } + targetPath := options.TargetPath if targetPath == "" { targetPath = manifest.Session.RolloutPath } - if _, err := os.Stat(targetPath); err == nil && !overwrite { + reclaimableBytes := int64(0) + if info, err := os.Stat(targetPath); err == nil && !options.Overwrite { return UnfoldResult{}, fmt.Errorf("restore target already exists: %s", targetPath) + } else if err == nil && info.Mode().IsRegular() { + reclaimableBytes = info.Size() } else if err != nil && !os.IsNotExist(err) { return UnfoldResult{}, err } + budget := options.Budget + if budget == nil { + guard, err := storage.DefaultGuard(storeDir) + if err != nil { + return UnfoldResult{}, err + } + budget = guard + } + storageAssessment, err := budget.Check(ctx, storage.Projection{ + Operation: "unfold", AdditionalPersistentBytes: manifest.Source.Bytes, TemporaryBytes: manifest.Source.Bytes, + TemporaryPersistentOverlapBytes: manifest.Source.Bytes, ReclaimableBytes: reclaimableBytes, + }) + if err != nil { + return UnfoldResult{}, err + } if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil { return UnfoldResult{}, fmt.Errorf("create restore directory: %w", err) } @@ -71,7 +103,7 @@ func Unfold(ctx context.Context, storeDir string, sessionID string, targetPath s return UnfoldResult{}, fmt.Errorf("close restored rollout: %w", err) } commit := os.Rename - if overwrite { + if options.Overwrite { commit = replaceFile } if err := commit(temporaryPath, targetPath); err != nil { @@ -83,5 +115,6 @@ func Unfold(ctx context.Context, storeDir string, sessionID string, targetPath s return UnfoldResult{ SessionID: sessionID, ManifestPath: ManifestPath(storeDir, sessionID), TargetPath: targetPath, Bytes: bytesWritten, SHA256: manifest.Source.SHA256, Verified: true, + Storage: storage.CompleteAccounting(ctx, storageAssessment, storeDir), }, nil } diff --git a/internal/fsctl/doctor.go b/internal/fsctl/doctor.go index 1b0454f..4414501 100644 --- a/internal/fsctl/doctor.go +++ b/internal/fsctl/doctor.go @@ -3,6 +3,8 @@ package fsctl import ( "context" "fmt" + + "github.com/jstar0/codexfold/internal/storage" ) const ( @@ -16,9 +18,10 @@ const ( ComponentFallback = "fallback" ComponentJournal = "journal" ComponentClient = "client" + ComponentStorage = "storage" ) -var RequiredComponents = []string{ComponentDaemon, ComponentMount, ComponentPack, ComponentManifest, ComponentDelta, ComponentBacking, ComponentRoute, ComponentFallback, ComponentJournal, ComponentClient} +var RequiredComponents = []string{ComponentDaemon, ComponentMount, ComponentPack, ComponentManifest, ComponentDelta, ComponentBacking, ComponentRoute, ComponentFallback, ComponentJournal, ComponentClient, ComponentStorage} type Check struct { Component string @@ -35,10 +38,13 @@ type Issue struct { } type DoctorReport struct { - Healthy bool `json:"healthy"` - IssueCount int `json:"issue_count"` - Issues []Issue `json:"issues,omitempty"` - ComponentHealth map[string]bool `json:"component_health"` + Healthy bool `json:"healthy"` + IssueCount int `json:"issue_count"` + Issues []Issue `json:"issues,omitempty"` + ComponentHealth map[string]bool `json:"component_health"` + Storage storage.Inventory `json:"storage"` + StorageLimits storage.Limits `json:"storage_limits"` + AvailableBytes int64 `json:"available_bytes"` } func Doctor(ctx context.Context, checks []Check) DoctorReport { diff --git a/internal/fsctl/status.go b/internal/fsctl/status.go index d1a7cb2..9706e1a 100644 --- a/internal/fsctl/status.go +++ b/internal/fsctl/status.go @@ -3,6 +3,8 @@ package fsctl import ( "fmt" "strings" + + "github.com/jstar0/codexfold/internal/storage" ) type Capability string @@ -15,8 +17,11 @@ const ( ) type Status struct { - Capability Capability `json:"capability"` - Platform string `json:"platform"` + Capability Capability `json:"capability"` + Platform string `json:"platform"` + Storage storage.Inventory `json:"storage"` + StorageLimits storage.Limits `json:"storage_limits"` + AvailableBytes int64 `json:"available_bytes"` } func NewStatus(capability Capability, platform string) (Status, error) { diff --git a/internal/fskitproto/client.go b/internal/fskitproto/client.go new file mode 100644 index 0000000..335d0cf --- /dev/null +++ b/internal/fskitproto/client.go @@ -0,0 +1,149 @@ +package fskitproto + +import ( + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "time" +) + +const DescriptorFilename = "descriptor.bin" + +type StatusError struct { + Operation Op + Errno syscall.Errno +} + +func (e StatusError) Error() string { + return fmt.Sprintf("FSKit operation %d failed: %s", e.Operation, e.Errno) +} + +type Client struct { + mu sync.Mutex + connection net.Conn + generation uint64 + maxPayload uint32 + nextID uint64 +} + +func DialResource(resourcePath string, timeout time.Duration) (*Client, error) { + descriptorPath, err := ResourceDescriptorPath(resourcePath) + if err != nil { + return nil, err + } + data, err := os.ReadFile(descriptorPath) + if err != nil { + return nil, fmt.Errorf("read FSKit resource: %w", err) + } + descriptor, err := DecodeDescriptor(data) + if err != nil { + return nil, err + } + return Dial(descriptor, timeout) +} + +func ResourceDescriptorPath(resourcePath string) (string, error) { + if !filepath.IsAbs(resourcePath) { + return "", errors.New("absolute FSKit resource path is required") + } + resourcePath = filepath.Clean(resourcePath) + if UsesDirectoryResource(resourcePath) { + return filepath.Join(resourcePath, DescriptorFilename), nil + } + _, err := os.Stat(resourcePath) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return "", err + } + return resourcePath, nil +} + +func UsesDirectoryResource(resourcePath string) bool { + if info, err := os.Stat(filepath.Clean(resourcePath)); err == nil { + return info.IsDir() + } + return !strings.EqualFold(filepath.Ext(resourcePath), ".bin") +} + +func Dial(descriptor Descriptor, timeout time.Duration) (*Client, error) { + if timeout <= 0 { + timeout = 5 * time.Second + } + connection, err := net.DialTimeout("unix", descriptor.SocketPath, timeout) + if err != nil { + return nil, fmt.Errorf("connect to FSKit daemon: %w", err) + } + client := &Client{connection: connection, generation: descriptor.Generation, maxPayload: DefaultMaxPayload, nextID: 1} + encoder := NewEncoder(len(descriptor.Token) + 4) + encoder.Bytes(descriptor.Token) + response, err := client.callLocked(OpHello, 0, encoder.Data()) + if err != nil { + _ = connection.Close() + return nil, err + } + decoder := NewDecoder(response) + maxPayload, err := decoder.Uint32() + if err != nil || maxPayload < 4096 { + _ = connection.Close() + return nil, errors.New("invalid FSKit daemon hello response") + } + if _, err := decoder.Uint64(); err != nil || decoder.Done() != nil { + _ = connection.Close() + return nil, errors.New("invalid FSKit daemon hello namespace response") + } + client.maxPayload = maxPayload + return client, nil +} + +func (c *Client) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + if c.connection == nil { + return nil + } + err := c.connection.Close() + c.connection = nil + return err +} + +func (c *Client) Call(operation Op, payload []byte) ([]byte, error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.callLocked(operation, c.generation, payload) +} + +func (c *Client) callLocked(operation Op, generation uint64, payload []byte) ([]byte, error) { + if c.connection == nil { + return nil, net.ErrClosed + } + requestID := c.nextID + c.nextID++ + if err := WriteFrame(c.connection, Frame{ + Kind: KindRequest, Op: operation, RequestID: requestID, Generation: generation, Payload: payload, + }, c.maxPayload); err != nil { + return nil, err + } + response, err := ReadFrame(c.connection, c.maxPayload) + if err != nil { + return nil, err + } + if response.Kind != KindResponse || response.Op != operation || response.RequestID != requestID || response.Generation != c.generation { + return nil, errors.New("mismatched FSKit daemon response") + } + if response.Status != 0 { + return nil, StatusError{Operation: operation, Errno: syscall.Errno(response.Status)} + } + return response.Payload, nil +} + +func ErrorNumber(err error) syscall.Errno { + var status StatusError + if errors.As(err, &status) { + return status.Errno + } + return syscall.EIO +} diff --git a/internal/fskitproto/codec.go b/internal/fskitproto/codec.go new file mode 100644 index 0000000..f4948ea --- /dev/null +++ b/internal/fskitproto/codec.go @@ -0,0 +1,268 @@ +package fskitproto + +import ( + "encoding/binary" + "errors" + "fmt" + "time" +) + +type Encoder struct { + data []byte +} + +func NewEncoder(capacity int) *Encoder { + return &Encoder{data: make([]byte, 0, capacity)} +} + +func (e *Encoder) Data() []byte { return e.data } + +func (e *Encoder) Raw(value []byte) { e.data = append(e.data, value...) } + +func (e *Encoder) Uint8(value uint8) { e.data = append(e.data, value) } + +func (e *Encoder) Uint16(value uint16) { + start := len(e.data) + e.data = append(e.data, 0, 0) + binary.LittleEndian.PutUint16(e.data[start:], value) +} + +func (e *Encoder) Uint32(value uint32) { + start := len(e.data) + e.data = append(e.data, 0, 0, 0, 0) + binary.LittleEndian.PutUint32(e.data[start:], value) +} + +func (e *Encoder) Uint64(value uint64) { + start := len(e.data) + e.data = append(e.data, make([]byte, 8)...) + binary.LittleEndian.PutUint64(e.data[start:], value) +} + +func (e *Encoder) Int64(value int64) { e.Uint64(uint64(value)) } + +func (e *Encoder) String(value string) { + e.Bytes([]byte(value)) +} + +func (e *Encoder) Bytes(value []byte) { + e.Uint32(uint32(len(value))) + e.Raw(value) +} + +func (e *Encoder) Time(value time.Time) { + if value.IsZero() { + e.Int64(0) + e.Uint32(0) + return + } + e.Int64(value.Unix()) + e.Uint32(uint32(value.Nanosecond())) +} + +func (e *Encoder) Entry(entry Entry) { + e.String(entry.Path) + e.String(entry.Name) + e.Uint64(entry.NodeID) + e.Uint64(entry.ParentID) + e.Uint8(uint8(entry.Type)) + e.Uint32(entry.Mode) + e.Uint32(entry.UID) + e.Uint32(entry.GID) + e.Uint64(entry.Size) + e.Uint64(entry.AllocSize) + e.Time(entry.ModTime) + e.Time(entry.ChangeTime) + e.Time(entry.AccessTime) + e.Uint64(entry.NamespaceID) +} + +func (e *Encoder) StatFS(stat StatFS) { + e.Uint32(stat.BlockSize) + e.Uint32(stat.IOSize) + e.Uint64(stat.TotalBytes) + e.Uint64(stat.AvailableBytes) + e.Uint64(stat.FreeBytes) + e.Uint64(stat.UsedBytes) + e.Uint64(stat.TotalFiles) + e.Uint64(stat.FreeFiles) +} + +type Decoder struct { + data []byte + offset int +} + +func NewDecoder(data []byte) *Decoder { return &Decoder{data: data} } + +func (d *Decoder) remaining() int { return len(d.data) - d.offset } + +func (d *Decoder) Raw(length int) ([]byte, error) { + if length < 0 || d.remaining() < length { + return nil, errors.New("truncated FSKit protocol payload") + } + value := d.data[d.offset : d.offset+length] + d.offset += length + return value, nil +} + +func (d *Decoder) Uint8() (uint8, error) { + value, err := d.Raw(1) + if err != nil { + return 0, err + } + return value[0], nil +} + +func (d *Decoder) Uint16() (uint16, error) { + value, err := d.Raw(2) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint16(value), nil +} + +func (d *Decoder) Uint32() (uint32, error) { + value, err := d.Raw(4) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint32(value), nil +} + +func (d *Decoder) Uint64() (uint64, error) { + value, err := d.Raw(8) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint64(value), nil +} + +func (d *Decoder) Int64() (int64, error) { + value, err := d.Uint64() + return int64(value), err +} + +func (d *Decoder) Bytes(limit int) ([]byte, error) { + length, err := d.Uint32() + if err != nil { + return nil, err + } + if uint64(length) > uint64(^uint(0)>>1) || (limit > 0 && length > uint32(limit)) { + return nil, fmt.Errorf("FSKit protocol byte field %d exceeds limit %d", length, limit) + } + return d.Raw(int(length)) +} + +func (d *Decoder) String(limit int) (string, error) { + value, err := d.Bytes(limit) + if err != nil { + return "", err + } + return string(value), nil +} + +func (d *Decoder) Time() (time.Time, error) { + seconds, err := d.Int64() + if err != nil { + return time.Time{}, err + } + nanoseconds, err := d.Uint32() + if err != nil { + return time.Time{}, err + } + if seconds == 0 && nanoseconds == 0 { + return time.Time{}, nil + } + if nanoseconds >= 1_000_000_000 { + return time.Time{}, errors.New("invalid FSKit protocol timestamp") + } + return time.Unix(seconds, int64(nanoseconds)), nil +} + +func (d *Decoder) Entry() (Entry, error) { + var entry Entry + var err error + if entry.Path, err = d.String(1 << 20); err != nil { + return Entry{}, err + } + if entry.Name, err = d.String(4096); err != nil { + return Entry{}, err + } + if entry.NodeID, err = d.Uint64(); err != nil { + return Entry{}, err + } + if entry.ParentID, err = d.Uint64(); err != nil { + return Entry{}, err + } + typeValue, err := d.Uint8() + if err != nil { + return Entry{}, err + } + entry.Type = EntryType(typeValue) + if entry.Mode, err = d.Uint32(); err != nil { + return Entry{}, err + } + if entry.UID, err = d.Uint32(); err != nil { + return Entry{}, err + } + if entry.GID, err = d.Uint32(); err != nil { + return Entry{}, err + } + if entry.Size, err = d.Uint64(); err != nil { + return Entry{}, err + } + if entry.AllocSize, err = d.Uint64(); err != nil { + return Entry{}, err + } + if entry.ModTime, err = d.Time(); err != nil { + return Entry{}, err + } + if entry.ChangeTime, err = d.Time(); err != nil { + return Entry{}, err + } + if entry.AccessTime, err = d.Time(); err != nil { + return Entry{}, err + } + if entry.NamespaceID, err = d.Uint64(); err != nil { + return Entry{}, err + } + return entry, nil +} + +func (d *Decoder) StatFS() (StatFS, error) { + var stat StatFS + var err error + if stat.BlockSize, err = d.Uint32(); err != nil { + return StatFS{}, err + } + if stat.IOSize, err = d.Uint32(); err != nil { + return StatFS{}, err + } + if stat.TotalBytes, err = d.Uint64(); err != nil { + return StatFS{}, err + } + if stat.AvailableBytes, err = d.Uint64(); err != nil { + return StatFS{}, err + } + if stat.FreeBytes, err = d.Uint64(); err != nil { + return StatFS{}, err + } + if stat.UsedBytes, err = d.Uint64(); err != nil { + return StatFS{}, err + } + if stat.TotalFiles, err = d.Uint64(); err != nil { + return StatFS{}, err + } + if stat.FreeFiles, err = d.Uint64(); err != nil { + return StatFS{}, err + } + return stat, nil +} + +func (d *Decoder) Done() error { + if d.remaining() != 0 { + return fmt.Errorf("FSKit protocol payload has %d trailing bytes", d.remaining()) + } + return nil +} diff --git a/internal/fskitproto/protocol.go b/internal/fskitproto/protocol.go new file mode 100644 index 0000000..1b177d3 --- /dev/null +++ b/internal/fskitproto/protocol.go @@ -0,0 +1,250 @@ +package fskitproto + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "time" +) + +const ( + Version uint16 = 2 + HeaderSize = 40 + DefaultMaxPayload = 16 << 20 + OpenFlagSnapshot uint32 = 1 << 31 +) + +var ( + frameMagic = [4]byte{'C', 'F', 'S', 'P'} + descriptorMagic = [4]byte{'C', 'F', 'S', 'R'} +) + +type Kind uint8 + +const ( + KindRequest Kind = 1 + KindResponse Kind = 2 +) + +type Op uint8 + +const ( + OpHello Op = iota + 1 + OpPing + OpGetattr + OpReadDir + OpOpen + OpCreate + OpRead + OpWrite + OpFsync + OpFlush + OpRelease + OpTruncate + OpMkdir + OpRename + OpUnlink + OpRmdir + OpStatfs + OpSync + OpNamespaceVersion + OpSetattr + OpGetXattr + OpSetXattr + OpListXattrs +) + +const ( + SetAttrMode uint32 = 1 << iota + SetAttrUID + SetAttrGID + SetAttrAccessTime + SetAttrModifyTime +) + +type XattrPolicy uint32 + +const ( + XattrAlwaysSet XattrPolicy = iota + XattrMustCreate + XattrMustReplace + XattrDelete +) + +type EntryType uint8 + +const ( + EntryUnknown EntryType = iota + EntryFile + EntryDirectory + EntrySymlink +) + +type Frame struct { + Kind Kind + Op Op + Flags uint32 + RequestID uint64 + Generation uint64 + Status int32 + Payload []byte +} + +func ReadFrame(reader io.Reader, maxPayload uint32) (Frame, error) { + if maxPayload == 0 { + maxPayload = DefaultMaxPayload + } + header := make([]byte, HeaderSize) + if _, err := io.ReadFull(reader, header); err != nil { + return Frame{}, err + } + if string(header[:4]) != string(frameMagic[:]) { + return Frame{}, errors.New("invalid FSKit protocol magic") + } + if version := binary.LittleEndian.Uint16(header[4:6]); version != Version { + return Frame{}, fmt.Errorf("unsupported FSKit protocol version %d", version) + } + payloadLength := binary.LittleEndian.Uint32(header[32:36]) + if payloadLength > maxPayload { + return Frame{}, fmt.Errorf("FSKit protocol payload %d exceeds limit %d", payloadLength, maxPayload) + } + payload := make([]byte, payloadLength) + if _, err := io.ReadFull(reader, payload); err != nil { + return Frame{}, err + } + return Frame{ + Kind: Kind(header[6]), + Op: Op(header[7]), + Flags: binary.LittleEndian.Uint32(header[8:12]), + RequestID: binary.LittleEndian.Uint64(header[12:20]), + Generation: binary.LittleEndian.Uint64(header[20:28]), + Status: int32(binary.LittleEndian.Uint32(header[28:32])), + Payload: payload, + }, nil +} + +func WriteFrame(writer io.Writer, frame Frame, maxPayload uint32) error { + if maxPayload == 0 { + maxPayload = DefaultMaxPayload + } + if len(frame.Payload) > int(maxPayload) { + return fmt.Errorf("FSKit protocol payload %d exceeds limit %d", len(frame.Payload), maxPayload) + } + header := make([]byte, HeaderSize) + copy(header[:4], frameMagic[:]) + binary.LittleEndian.PutUint16(header[4:6], Version) + header[6] = byte(frame.Kind) + header[7] = byte(frame.Op) + binary.LittleEndian.PutUint32(header[8:12], frame.Flags) + binary.LittleEndian.PutUint64(header[12:20], frame.RequestID) + binary.LittleEndian.PutUint64(header[20:28], frame.Generation) + binary.LittleEndian.PutUint32(header[28:32], uint32(frame.Status)) + binary.LittleEndian.PutUint32(header[32:36], uint32(len(frame.Payload))) + if err := writeAll(writer, header); err != nil { + return err + } + return writeAll(writer, frame.Payload) +} + +func writeAll(writer io.Writer, data []byte) error { + for len(data) > 0 { + n, err := writer.Write(data) + if err != nil { + return err + } + if n == 0 { + return io.ErrUnexpectedEOF + } + data = data[n:] + } + return nil +} + +type Entry struct { + Path string + Name string + NodeID uint64 + ParentID uint64 + Type EntryType + Mode uint32 + UID uint32 + GID uint32 + Size uint64 + AllocSize uint64 + ModTime time.Time + ChangeTime time.Time + AccessTime time.Time + NamespaceID uint64 +} + +type StatFS struct { + BlockSize uint32 + IOSize uint32 + TotalBytes uint64 + AvailableBytes uint64 + FreeBytes uint64 + UsedBytes uint64 + TotalFiles uint64 + FreeFiles uint64 +} + +type Descriptor struct { + Generation uint64 + SocketPath string + Token []byte +} + +func EncodeDescriptor(descriptor Descriptor) ([]byte, error) { + if descriptor.Generation == 0 { + return nil, errors.New("descriptor generation is required") + } + if descriptor.SocketPath == "" || len(descriptor.SocketPath) > 4096 { + return nil, errors.New("descriptor socket path is invalid") + } + if len(descriptor.Token) < 16 || len(descriptor.Token) > 256 { + return nil, errors.New("descriptor token length is invalid") + } + encoder := NewEncoder(24 + len(descriptor.SocketPath) + len(descriptor.Token)) + encoder.Raw(descriptorMagic[:]) + encoder.Uint16(Version) + encoder.Uint16(0) + encoder.Uint64(descriptor.Generation) + encoder.String(descriptor.SocketPath) + encoder.Bytes(descriptor.Token) + return encoder.Data(), nil +} + +func DecodeDescriptor(data []byte) (Descriptor, error) { + decoder := NewDecoder(data) + magic, err := decoder.Raw(4) + if err != nil || string(magic) != string(descriptorMagic[:]) { + return Descriptor{}, errors.New("invalid FSKit resource descriptor magic") + } + version, err := decoder.Uint16() + if err != nil { + return Descriptor{}, err + } + if version != Version { + return Descriptor{}, fmt.Errorf("unsupported FSKit resource descriptor version %d", version) + } + if _, err := decoder.Uint16(); err != nil { + return Descriptor{}, err + } + generation, err := decoder.Uint64() + if err != nil || generation == 0 { + return Descriptor{}, errors.New("invalid FSKit resource descriptor generation") + } + socketPath, err := decoder.String(4096) + if err != nil || socketPath == "" { + return Descriptor{}, errors.New("invalid FSKit resource descriptor socket path") + } + token, err := decoder.Bytes(256) + if err != nil || len(token) < 16 { + return Descriptor{}, errors.New("invalid FSKit resource descriptor token") + } + if err := decoder.Done(); err != nil { + return Descriptor{}, err + } + return Descriptor{Generation: generation, SocketPath: socketPath, Token: token}, nil +} diff --git a/internal/fskitproto/protocol_test.go b/internal/fskitproto/protocol_test.go new file mode 100644 index 0000000..aeebef7 --- /dev/null +++ b/internal/fskitproto/protocol_test.go @@ -0,0 +1,93 @@ +package fskitproto + +import ( + "bytes" + "os" + "path/filepath" + "reflect" + "testing" + "time" +) + +func TestFrameRoundTrip(t *testing.T) { + want := Frame{Kind: KindRequest, Op: OpWrite, Flags: 7, RequestID: 99, Generation: 42, Status: -5, Payload: []byte("payload")} + var buffer bytes.Buffer + if err := WriteFrame(&buffer, want, 0); err != nil { + t.Fatal(err) + } + got, err := ReadFrame(&buffer, 0) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("frame = %#v, want %#v", got, want) + } +} + +func TestEntryRoundTrip(t *testing.T) { + want := Entry{ + Path: "/sessions/2026/07/session.jsonl", Name: "session.jsonl", NodeID: 9, ParentID: 8, + Type: EntryFile, Mode: 0o600, UID: 501, GID: 20, Size: 1234, AllocSize: 4096, + ModTime: time.Unix(1_700_000_000, 123), ChangeTime: time.Unix(1_700_000_001, 456), + AccessTime: time.Unix(1_700_000_002, 789), NamespaceID: 33, + } + encoder := NewEncoder(256) + encoder.Entry(want) + decoder := NewDecoder(encoder.Data()) + got, err := decoder.Entry() + if err != nil { + t.Fatal(err) + } + if err := decoder.Done(); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("entry = %#v, want %#v", got, want) + } +} + +func TestDescriptorRoundTrip(t *testing.T) { + want := Descriptor{Generation: 17, SocketPath: "/tmp/codexfold.sock", Token: bytes.Repeat([]byte{0x5a}, 32)} + encoded, err := EncodeDescriptor(want) + if err != nil { + t.Fatal(err) + } + got, err := DecodeDescriptor(encoded) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("descriptor = %#v, want %#v", got, want) + } +} + +func TestResourceDescriptorPathSupportsSecurityScopedDirectoryAndLegacyFile(t *testing.T) { + root := t.TempDir() + directoryResource := filepath.Join(root, "native-fskit") + if err := os.Mkdir(directoryResource, 0o700); err != nil { + t.Fatal(err) + } + path, err := ResourceDescriptorPath(directoryResource) + if err != nil || path != filepath.Join(directoryResource, DescriptorFilename) { + t.Fatalf("directory descriptor = %q err=%v", path, err) + } + legacy := filepath.Join(root, "resource.bin") + if err := os.WriteFile(legacy, []byte("legacy"), 0o600); err != nil { + t.Fatal(err) + } + path, err = ResourceDescriptorPath(legacy) + if err != nil || path != legacy { + t.Fatalf("legacy descriptor = %q err=%v", path, err) + } +} + +func TestReadFrameRejectsOversizedPayloadBeforeAllocation(t *testing.T) { + frame := Frame{Kind: KindRequest, Op: OpWrite, Payload: bytes.Repeat([]byte("x"), 64)} + var buffer bytes.Buffer + if err := WriteFrame(&buffer, frame, 128); err != nil { + t.Fatal(err) + } + if _, err := ReadFrame(&buffer, 32); err == nil { + t.Fatal("ReadFrame unexpectedly accepted an oversized payload") + } +} diff --git a/internal/launcher/parent.go b/internal/launcher/parent.go new file mode 100644 index 0000000..8a589ee --- /dev/null +++ b/internal/launcher/parent.go @@ -0,0 +1,50 @@ +package launcher + +import ( + "context" + "errors" + "os" + "strconv" + "time" +) + +const ParentPIDEnvironment = "CODEXFOLD_LAUNCHER_PARENT_PID" + +func MonitorContext(parent context.Context) (context.Context, context.CancelFunc, error) { + return monitorParent(parent, os.Getenv(ParentPIDEnvironment), os.Getppid, 50*time.Millisecond) +} + +func monitorParent(parent context.Context, value string, currentParent func() int, interval time.Duration) (context.Context, context.CancelFunc, error) { + ctx, cancel := context.WithCancel(parent) + if value == "" { + return ctx, cancel, nil + } + expected, err := strconv.Atoi(value) + if err != nil || expected <= 1 { + cancel() + return nil, nil, errors.New("invalid CodexFold launcher parent PID") + } + if currentParent == nil || currentParent() != expected { + cancel() + return nil, nil, errors.New("CodexFold launcher parent is already unavailable") + } + if interval <= 0 { + interval = 50 * time.Millisecond + } + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if currentParent() != expected { + cancel() + return + } + } + } + }() + return ctx, cancel, nil +} diff --git a/internal/launcher/parent_test.go b/internal/launcher/parent_test.go new file mode 100644 index 0000000..c6174a7 --- /dev/null +++ b/internal/launcher/parent_test.go @@ -0,0 +1,46 @@ +package launcher + +import ( + "context" + "sync/atomic" + "testing" + "time" +) + +func TestMonitorParentCancelsWhenLauncherDisappears(t *testing.T) { + var parent atomic.Int64 + parent.Store(42) + ctx, cancel, err := monitorParent(context.Background(), "42", func() int { return int(parent.Load()) }, time.Millisecond) + if err != nil { + t.Fatal(err) + } + defer cancel() + parent.Store(1) + select { + case <-ctx.Done(): + case <-time.After(time.Second): + t.Fatal("launcher parent loss did not cancel the context") + } +} + +func TestMonitorParentRejectsInvalidOrAlreadyLostLauncher(t *testing.T) { + for _, value := range []string{"abc", "0", "1", "43"} { + if _, _, err := monitorParent(context.Background(), value, func() int { return 42 }, time.Millisecond); err == nil { + t.Fatalf("monitorParent(%q) succeeded", value) + } + } +} + +func TestMonitorParentIsDisabledWithoutLauncherEnvironment(t *testing.T) { + parent := context.Background() + ctx, cancel, err := monitorParent(parent, "", func() int { return 1 }, time.Millisecond) + if err != nil { + t.Fatal(err) + } + defer cancel() + select { + case <-ctx.Done(): + t.Fatal("unset launcher environment canceled an ordinary process") + default: + } +} diff --git a/internal/mountfs/dependency_boundary_test.go b/internal/mountfs/dependency_boundary_test.go new file mode 100644 index 0000000..42f00c4 --- /dev/null +++ b/internal/mountfs/dependency_boundary_test.go @@ -0,0 +1,54 @@ +package mountfs + +import ( + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestAdapterCoreDoesNotDependOnCodexDatabase(t *testing.T) { + repoRoot := dependencyRepoRoot(t) + + assertPackageExcludesDependencies(t, repoRoot, "./internal/fold", []string{ + "github.com/jstar0/codexfold/internal/codex", + "modernc.org", + }) + assertPackageExcludesDependencies(t, repoRoot, "./internal/mountfs", []string{ + "github.com/jstar0/codexfold/internal/codex", + "github.com/jstar0/codexfold/internal/service", + "modernc.org", + }) +} + +func dependencyRepoRoot(t *testing.T) string { + t.Helper() + _, currentFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("resolve dependency boundary test source path") + } + return filepath.Clean(filepath.Join(filepath.Dir(currentFile), "../..")) +} + +func assertPackageExcludesDependencies(t *testing.T, repoRoot, packagePath string, forbidden []string) { + t.Helper() + cmd := exec.Command("go", "list", "-deps", packagePath) + cmd.Dir = repoRoot + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("list dependencies for %s: %v\n%s", packagePath, err, output) + } + + dependencies := make(map[string]struct{}) + for _, dependency := range strings.Fields(string(output)) { + dependencies[dependency] = struct{}{} + } + for dependency := range dependencies { + for _, forbiddenPrefix := range forbidden { + if dependency == forbiddenPrefix || strings.HasPrefix(dependency, forbiddenPrefix+"/") { + t.Errorf("%s must not depend on %s", packagePath, dependency) + } + } + } +} diff --git a/internal/mountfs/file_metadata_darwin.go b/internal/mountfs/file_metadata_darwin.go new file mode 100644 index 0000000..a277323 --- /dev/null +++ b/internal/mountfs/file_metadata_darwin.go @@ -0,0 +1,19 @@ +//go:build darwin + +package mountfs + +import ( + "os" + "syscall" + "time" +) + +func fileOwnershipAndTimes(info os.FileInfo) (uint32, uint32, time.Time, time.Time) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return uint32(os.Getuid()), uint32(os.Getgid()), info.ModTime(), info.ModTime() + } + return stat.Uid, stat.Gid, + time.Unix(stat.Atimespec.Sec, stat.Atimespec.Nsec), + time.Unix(stat.Ctimespec.Sec, stat.Ctimespec.Nsec) +} diff --git a/internal/mountfs/file_metadata_linux.go b/internal/mountfs/file_metadata_linux.go new file mode 100644 index 0000000..3d0d8fe --- /dev/null +++ b/internal/mountfs/file_metadata_linux.go @@ -0,0 +1,19 @@ +//go:build linux + +package mountfs + +import ( + "os" + "syscall" + "time" +) + +func fileOwnershipAndTimes(info os.FileInfo) (uint32, uint32, time.Time, time.Time) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return uint32(os.Getuid()), uint32(os.Getgid()), info.ModTime(), info.ModTime() + } + return stat.Uid, stat.Gid, + time.Unix(stat.Atim.Sec, stat.Atim.Nsec), + time.Unix(stat.Ctim.Sec, stat.Ctim.Nsec) +} diff --git a/internal/mountfs/file_metadata_other.go b/internal/mountfs/file_metadata_other.go new file mode 100644 index 0000000..ff6452e --- /dev/null +++ b/internal/mountfs/file_metadata_other.go @@ -0,0 +1,12 @@ +//go:build !darwin && !linux + +package mountfs + +import ( + "os" + "time" +) + +func fileOwnershipAndTimes(info os.FileInfo) (uint32, uint32, time.Time, time.Time) { + return uint32(os.Getuid()), uint32(os.Getgid()), info.ModTime(), info.ModTime() +} diff --git a/internal/mountfs/filesystem.go b/internal/mountfs/filesystem.go index a6360b1..7981071 100644 --- a/internal/mountfs/filesystem.go +++ b/internal/mountfs/filesystem.go @@ -14,22 +14,41 @@ import ( "sort" "strings" "sync" + "sync/atomic" "syscall" "time" + "unicode/utf8" + "github.com/jstar0/codexfold/internal/fskitproto" "github.com/jstar0/codexfold/internal/vfs" ) type Attr struct { - Mode uint32 `json:"mode"` - Size int64 `json:"size"` - ModTime time.Time `json:"mod_time"` + Mode uint32 `json:"mode"` + UID uint32 `json:"uid"` + GID uint32 `json:"gid"` + Size int64 `json:"size"` + ModTime time.Time `json:"mod_time"` + ChangeTime time.Time `json:"change_time"` + AccessTime time.Time `json:"access_time"` +} + +type SetAttrRequest struct { + Valid uint32 + Mode uint32 + UID uint32 + GID uint32 + AccessTime time.Time + ModTime time.Time } type fileHandle struct { mu sync.Mutex + path string session *vfs.Session native *os.File + nativePath string + nativeAppend *nativeAppendState read *vfs.ReadHandle write *vfs.WriteHandle append bool @@ -39,31 +58,47 @@ type fileHandle struct { } type Filesystem struct { - mu sync.RWMutex - loadMu sync.Mutex - sessions map[string]*vfs.Session - paths map[string]string - retained map[string]string - nativeFirst map[string]struct{} - directories map[string]struct{} - handles map[uint64]*fileHandle - next uint64 - loader func(string) (*vfs.Session, error) - canonical bool - nativeRoot string + mu sync.RWMutex + loadMu sync.Mutex + sessions map[string]*vfs.Session + paths map[string]string + retained map[string]string + nativeFirst map[string]struct{} + directories map[string]struct{} + handles map[uint64]*fileHandle + next uint64 + loader func(string) (*vfs.Session, error) + canonical bool + nativeRoot string + nativeJournalRoot string + nativeAppends map[string]*nativeAppendState + namespaceVersion atomic.Uint64 } func New() *Filesystem { - return &Filesystem{sessions: make(map[string]*vfs.Session), handles: make(map[uint64]*fileHandle), next: 1} + filesystem := &Filesystem{sessions: make(map[string]*vfs.Session), handles: make(map[uint64]*fileHandle), next: 1} + filesystem.namespaceVersion.Store(1) + return filesystem } func NewCanonical() *Filesystem { - return &Filesystem{ + filesystem := &Filesystem{ sessions: make(map[string]*vfs.Session), paths: make(map[string]string), retained: make(map[string]string), nativeFirst: make(map[string]struct{}), directories: map[string]struct{}{`/`: {}, `/sessions`: {}, `/archived_sessions`: {}}, - handles: make(map[uint64]*fileHandle), next: 1, canonical: true, + handles: make(map[uint64]*fileHandle), nativeAppends: make(map[string]*nativeAppendState), + next: 1, canonical: true, } + filesystem.namespaceVersion.Store(1) + return filesystem +} + +func (f *Filesystem) NamespaceVersion() uint64 { + return f.namespaceVersion.Load() +} + +func (f *Filesystem) bumpNamespaceVersion() { + f.namespaceVersion.Add(1) } func (f *Filesystem) SetNativeRoot(root string) { @@ -71,7 +106,14 @@ func (f *Filesystem) SetNativeRoot(root string) { root = filepath.Clean(root) } f.mu.Lock() + if f.nativeRoot != root { + f.nativeAppends = make(map[string]*nativeAppendState) + } f.nativeRoot = root + f.nativeJournalRoot = "" + if root != "" { + f.nativeJournalRoot = filepath.Join(root, ".codexfold-native-journal") + } for retained := range f.retained { delete(f.retained, retained) } @@ -81,6 +123,23 @@ func (f *Filesystem) SetNativeRoot(root string) { f.mu.Unlock() } +func (f *Filesystem) RecoverNativeAppendTransactions() error { + f.mu.RLock() + root := f.nativeRoot + journalRoot := f.nativeJournalRoot + f.mu.RUnlock() + if root == "" { + return nil + } + if err := recoverNativeAppendTransactions(root, journalRoot); err != nil { + return err + } + f.mu.Lock() + f.nativeAppends = make(map[string]*nativeAppendState) + f.mu.Unlock() + return nil +} + func (f *Filesystem) AddSession(sessionID string, session *vfs.Session) error { if sessionID == "" || strings.ContainsAny(sessionID, "/\\\x00") || session == nil { return errors.New("safe session ID and session are required") @@ -92,6 +151,7 @@ func (f *Filesystem) AddSession(sessionID string, session *vfs.Session) error { } f.sessions[sessionID] = session f.mu.Unlock() + f.bumpNamespaceVersion() return nil } @@ -102,6 +162,7 @@ func (f *Filesystem) UpsertSession(sessionID string, session *vfs.Session) error f.mu.Lock() f.sessions[sessionID] = session f.mu.Unlock() + f.bumpNamespaceVersion() return nil } @@ -123,6 +184,7 @@ func (f *Filesystem) AddSessionAt(sessionID string, name string, session *vfs.Se f.paths[cleaned] = sessionID delete(f.nativeFirst, sessionID) f.registerRetainedPathLocked(sessionID, session) + f.bumpNamespaceVersion() return nil } @@ -151,6 +213,7 @@ func (f *Filesystem) UpsertSessionAt(sessionID string, name string, session *vfs f.paths[cleaned] = sessionID delete(f.nativeFirst, sessionID) f.registerRetainedPathLocked(sessionID, session) + f.bumpNamespaceVersion() return nil } @@ -179,6 +242,7 @@ func (f *Filesystem) MoveSessionAt(sessionID string, name string) error { return err } f.paths[cleaned] = sessionID + f.bumpNamespaceVersion() return nil } @@ -216,6 +280,7 @@ func (f *Filesystem) RemoveSession(sessionID string) error { delete(f.retained, retained) } } + f.bumpNamespaceVersion() return nil } @@ -298,25 +363,36 @@ func (f *Filesystem) ReadDir(name string) ([]string, syscall.Errno) { func (f *Filesystem) Getattr(name string) (Attr, syscall.Errno) { cleaned := cleanPath(name) if cleaned == "/" { - return Attr{Mode: syscall.S_IFDIR | 0o700}, 0 + if nativePath, ok := f.nativeMetadataPath(cleaned); ok { + if info, err := os.Lstat(nativePath); err == nil { + return attrFromFileInfo(info, 0), 0 + } + } + return syntheticAttr(syscall.S_IFDIR | 0o700), 0 } if f.canonical { f.mu.RLock() _, directory := f.directories[cleaned] f.mu.RUnlock() if directory { - return Attr{Mode: syscall.S_IFDIR | 0o700}, 0 + if nativePath, ok := f.nativeMetadataPath(cleaned); ok { + if info, err := os.Lstat(nativePath); err == nil { + return attrFromFileInfo(info, 0), 0 + } + } + return syntheticAttr(syscall.S_IFDIR | 0o700), 0 } if session, errno := f.sessionForPath(cleaned); errno == 0 { return sessionAttr(session) } if nativePath, ok := f.nativePath(cleaned); ok { - info, err := os.Stat(nativePath) + info, err := os.Lstat(nativePath) if err == nil { - if info.IsDir() { - return Attr{Mode: syscall.S_IFDIR | 0o700, ModTime: info.ModTime()}, 0 + size := info.Size() + if state := f.nativeAppendState(nativePath); state != nil { + size = state.VisibleSize() } - return Attr{Mode: syscall.S_IFREG | 0o600, Size: info.Size(), ModTime: info.ModTime()}, 0 + return attrFromFileInfo(info, size), 0 } if !errors.Is(err, os.ErrNotExist) { return Attr{}, errnoFor(err) @@ -335,7 +411,134 @@ func sessionAttr(session *vfs.Session) (Attr, syscall.Errno) { if err != nil { return Attr{}, errnoFor(err) } - return Attr{Mode: syscall.S_IFREG | 0o600, Size: info.Size, ModTime: info.ModTime}, 0 + metadata, err := os.Lstat(session.MetadataPath()) + if err != nil { + return Attr{}, errnoFor(err) + } + return attrFromFileInfo(metadata, info.Size), 0 +} + +func (f *Filesystem) SetAttributes(name string, request SetAttrRequest) syscall.Errno { + metadataPath, _, errno := f.metadataPath(name) + if errno != 0 { + return errno + } + info, err := os.Lstat(metadataPath) + if err != nil { + return errnoFor(err) + } + if request.Valid&fskitproto.SetAttrUID != 0 || request.Valid&fskitproto.SetAttrGID != 0 { + uid, gid, _, _ := fileOwnershipAndTimes(info) + if request.Valid&fskitproto.SetAttrUID != 0 { + uid = request.UID + } + if request.Valid&fskitproto.SetAttrGID != 0 { + gid = request.GID + } + if err := os.Chown(metadataPath, int(uid), int(gid)); err != nil { + return errnoFor(err) + } + } + if request.Valid&fskitproto.SetAttrMode != 0 { + if err := os.Chmod(metadataPath, os.FileMode(request.Mode)&os.ModePerm); err != nil { + return errnoFor(err) + } + } + if request.Valid&(fskitproto.SetAttrAccessTime|fskitproto.SetAttrModifyTime) != 0 { + _, _, accessTime, _ := fileOwnershipAndTimes(info) + modifyTime := info.ModTime() + if request.Valid&fskitproto.SetAttrAccessTime != 0 { + accessTime = request.AccessTime + } + if request.Valid&fskitproto.SetAttrModifyTime != 0 { + modifyTime = request.ModTime + } + if err := os.Chtimes(metadataPath, accessTime, modifyTime); err != nil { + return errnoFor(err) + } + } + return 0 +} + +func (f *Filesystem) GetXattr(name string, attribute string) ([]byte, syscall.Errno) { + xattrPath, managed, errno := f.xattrPath(name, false) + if errno != 0 { + return nil, errno + } + value, err := platformGetXattr(xattrPath, attribute) + if err != nil { + if managed && errors.Is(err, os.ErrNotExist) { + return nil, xattrMissingErrno() + } + return nil, errnoFor(err) + } + return value, 0 +} + +func (f *Filesystem) SetXattr(name string, attribute string, value []byte, policy fskitproto.XattrPolicy) syscall.Errno { + if attribute == "" || strings.ContainsRune(attribute, '\x00') { + return syscall.EINVAL + } + createCarrier := policy != fskitproto.XattrDelete + xattrPath, managed, errno := f.xattrPath(name, createCarrier) + if errno != 0 { + return errno + } + if policy == fskitproto.XattrDelete { + if managed { + if _, err := os.Lstat(xattrPath); errors.Is(err, os.ErrNotExist) { + return xattrMissingErrno() + } else if err != nil { + return errnoFor(err) + } + } + return errnoFor(platformRemoveXattr(xattrPath, attribute)) + } + return errnoFor(platformSetXattr(xattrPath, attribute, value, policy)) +} + +func (f *Filesystem) ListXattrs(name string) ([]string, syscall.Errno) { + xattrPath, managed, errno := f.xattrPath(name, false) + if errno != 0 { + return nil, errno + } + attributes, err := platformListXattrs(xattrPath) + if err != nil { + if managed && errors.Is(err, os.ErrNotExist) { + return []string{}, 0 + } + return nil, errnoFor(err) + } + sort.Strings(attributes) + return attributes, 0 +} + +func syntheticAttr(mode uint32) Attr { + now := time.Now() + return Attr{ + Mode: mode, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), + ModTime: now, ChangeTime: now, AccessTime: now, + } +} + +func attrFromFileInfo(info os.FileInfo, size int64) Attr { + mode := uint32(info.Mode().Perm()) + switch { + case info.Mode()&os.ModeSymlink != 0: + mode |= syscall.S_IFLNK + case info.IsDir(): + mode |= syscall.S_IFDIR + default: + mode |= syscall.S_IFREG + } + if size == 0 && !info.IsDir() { + size = info.Size() + } + uid, gid, accessTime, changeTime := fileOwnershipAndTimes(info) + return Attr{ + Mode: mode, UID: uid, GID: gid, Size: size, + ModTime: info.ModTime(), ChangeTime: changeTime, AccessTime: accessTime, + } } func (f *Filesystem) Open(name string, flags int) (uint64, syscall.Errno) { @@ -348,18 +551,40 @@ func (f *Filesystem) Open(name string, flags int) (uint64, syscall.Errno) { if !ok { return 0, errno } - native, err := os.OpenFile(nativePath, flags, 0o600) + // FUSE may split or retry one append syscall as positional writes. Keep + // append and truncate semantics in the transaction layer instead of the + // backing descriptor. + nativeFlags := flags &^ (os.O_APPEND | os.O_TRUNC) + native, err := os.OpenFile(nativePath, nativeFlags, 0o600) + if err != nil { + return 0, errnoFor(err) + } + state, err := f.loadNativeAppendState(nativePath) if err != nil { + _ = native.Close() return 0, errnoFor(err) } + access := flags & (os.O_WRONLY | os.O_RDWR) + writable := access == os.O_WRONLY || access == os.O_RDWR + if writable && flags&os.O_TRUNC != 0 { + if err := state.Truncate(0); err != nil { + _ = native.Close() + return 0, errnoFor(err) + } + } f.mu.Lock() handleID := f.next f.next++ - f.handles[handleID] = &fileHandle{native: native, append: flags&os.O_APPEND != 0} + f.handles[handleID] = &fileHandle{ + path: cleanPath(name), native: native, nativePath: nativePath, nativeAppend: state, + // macOS may strip O_APPEND before invoking FUSE. Every writable + // canonical JSONL handle therefore uses positional transaction staging. + append: writable && nativeTransactionPath(cleanPath(name)), + } f.mu.Unlock() return handleID, 0 } - handle := &fileHandle{session: session, append: flags&os.O_APPEND != 0} + handle := &fileHandle{path: cleanPath(name), session: session, append: flags&os.O_APPEND != 0} access := flags & (os.O_WRONLY | os.O_RDWR) if access != os.O_WRONLY { reader, err := session.OpenReader() @@ -403,7 +628,7 @@ func (f *Filesystem) Read(handleID uint64, destination []byte, offset int64) (in handle.mu.Lock() defer handle.mu.Unlock() if handle.native != nil { - n, err := handle.native.ReadAt(destination, offset) + n, err := handle.nativeAppend.ReadAt(handle.native, destination, offset) if err != nil && !errors.Is(err, io.EOF) { return n, errnoFor(err) } @@ -430,9 +655,15 @@ func (f *Filesystem) Write(handleID uint64, data []byte, offset int64) (int, sys var n int var err error if handle.append { - n, err = handle.native.Write(data) + n, err = handle.nativeAppend.Stage(data, offset) } else { + if handle.nativeAppend.HasPending() { + return 0, syscall.EBUSY + } n, err = handle.native.WriteAt(data, offset) + if err == nil { + err = handle.nativeAppend.Refresh() + } } return n, errnoFor(err) } @@ -480,6 +711,23 @@ func (f *Filesystem) Write(handleID uint64, data []byte, offset int64) (int, sys return n, 0 } +func (f *Filesystem) UseRandomWrites(handleID uint64) syscall.Errno { + handle, errno := f.handle(handleID) + if errno != 0 { + return syscall.EBADF + } + handle.mu.Lock() + defer handle.mu.Unlock() + if handle.native != nil && handle.append { + if err := handle.nativeAppend.Commit(); err != nil { + return errnoFor(err) + } + } + handle.append = false + handle.appendStream = false + return 0 +} + func (f *Filesystem) Truncate(handleID uint64, size int64) syscall.Errno { handle, errno := f.handle(handleID) if errno != 0 { @@ -488,7 +736,7 @@ func (f *Filesystem) Truncate(handleID uint64, size int64) syscall.Errno { handle.mu.Lock() defer handle.mu.Unlock() if handle.native != nil { - return errnoFor(handle.native.Truncate(size)) + return errnoFor(handle.nativeAppend.Truncate(size)) } if handle.write == nil { return syscall.EBADF @@ -510,7 +758,11 @@ func (f *Filesystem) TruncatePath(name string, size int64) syscall.Errno { if errno != 0 { if f.canonical { if nativePath, ok := f.nativePath(cleanPath(name)); ok { - return errnoFor(os.Truncate(nativePath, size)) + state, err := f.loadNativeAppendState(nativePath) + if err != nil { + return errnoFor(err) + } + return errnoFor(state.Truncate(size)) } } return errno @@ -541,7 +793,7 @@ func (f *Filesystem) TruncatePath(name string, size int64) syscall.Errno { } func completeJSONL(data []byte) bool { - if len(data) == 0 || data[len(data)-1] != '\n' { + if len(data) == 0 || data[len(data)-1] != '\n' || !utf8.Valid(data) { return false } for len(data) > 0 { @@ -574,6 +826,9 @@ func (f *Filesystem) Fsync(handleID uint64) syscall.Errno { handle.mu.Lock() defer handle.mu.Unlock() if handle.native != nil { + if handle.append { + return errnoFor(handle.nativeAppend.CommitAvailable()) + } return errnoFor(handle.native.Sync()) } if handle.write == nil { @@ -583,14 +838,27 @@ func (f *Filesystem) Fsync(handleID uint64) syscall.Errno { } func (f *Filesystem) Flush(handleID uint64) syscall.Errno { - _, errno := f.handle(handleID) - return errno + handle, errno := f.handle(handleID) + if errno != 0 { + return errno + } + handle.mu.Lock() + defer handle.mu.Unlock() + if handle.native != nil && handle.append { + if f.nativeAppendIsLastWriter(handleID, handle.nativeAppend) { + return errnoFor(handle.nativeAppend.Commit()) + } + return errnoFor(handle.nativeAppend.CommitAvailable()) + } + return 0 } func (f *Filesystem) Release(handleID uint64) syscall.Errno { f.mu.Lock() handle, ok := f.handles[handleID] + lastNativeWriter := false if ok { + lastNativeWriter = f.nativeAppendIsLastWriterLocked(handleID, handle.nativeAppend) delete(f.handles, handleID) } f.mu.Unlock() @@ -600,7 +868,15 @@ func (f *Filesystem) Release(handleID uint64) syscall.Errno { handle.mu.Lock() defer handle.mu.Unlock() if handle.native != nil { - return errnoFor(handle.native.Close()) + var commitErr error + if handle.append { + if lastNativeWriter { + commitErr = handle.nativeAppend.Commit() + } else { + commitErr = handle.nativeAppend.CommitAvailable() + } + } + return errnoFor(errors.Join(commitErr, handle.native.Close())) } var result syscall.Errno if handle.read != nil { @@ -616,6 +892,24 @@ func (f *Filesystem) Release(handleID uint64) syscall.Errno { return result } +func (f *Filesystem) nativeAppendIsLastWriter(handleID uint64, state *nativeAppendState) bool { + f.mu.RLock() + defer f.mu.RUnlock() + return f.nativeAppendIsLastWriterLocked(handleID, state) +} + +func (f *Filesystem) nativeAppendIsLastWriterLocked(handleID uint64, state *nativeAppendState) bool { + if state == nil { + return false + } + for currentID, current := range f.handles { + if currentID != handleID && current.nativeAppend == state && current.append { + return false + } + } + return true +} + func refreshReader(handle *fileHandle) syscall.Errno { reader, err := handle.session.OpenReader() if err != nil { @@ -659,6 +953,7 @@ func (f *Filesystem) Mkdir(name string, _ uint32) syscall.Errno { f.mu.Lock() defer f.mu.Unlock() f.directories[cleaned] = struct{}{} + f.bumpNamespaceVersion() return 0 } @@ -667,7 +962,8 @@ func (f *Filesystem) Rename(oldName string, newName string) syscall.Errno { return syscall.EPERM } oldPath, newPath := cleanPath(oldName), cleanPath(newName) - if !canonicalSessionPath(oldPath) || !canonicalSessionPath(newPath) { + openUnlinkRename := canonicalSessionPath(oldPath) && fskitOpenUnlinkPath(newPath) && path.Dir(oldPath) == path.Dir(newPath) + if !canonicalSessionPath(oldPath) || (!canonicalSessionPath(newPath) && !openUnlinkRename) { return syscall.EPERM } f.mu.Lock() @@ -677,16 +973,42 @@ func (f *Filesystem) Rename(oldName string, newName string) syscall.Errno { f.mu.Unlock() return syscall.ENOENT } + if f.nativePathBusyLocked(oldPath) && !openUnlinkRename { + f.mu.Unlock() + return syscall.EBUSY + } root := f.nativeRoot + oldNative := nativePathFromRoot(root, oldPath) + newNative := nativePathFromRoot(root, newPath) + appendState := f.nativeAppends[filepath.Clean(oldNative)] f.mu.Unlock() if root == "" { return syscall.ENOENT } - oldNative := nativePathFromRoot(root, oldPath) - newNative := nativePathFromRoot(root, newPath) - if err := os.Rename(oldNative, newNative); err != nil { - return errnoFor(err) + var renameErr error + if appendState != nil { + renameErr = appendState.Relocate(newNative) + } else { + renameErr = os.Rename(oldNative, newNative) + } + if renameErr != nil { + return errnoFor(renameErr) + } + f.mu.Lock() + delete(f.nativeAppends, filepath.Clean(oldNative)) + if appendState != nil { + f.nativeAppends[filepath.Clean(newNative)] = appendState + } else { + delete(f.nativeAppends, filepath.Clean(newNative)) + } + for _, handle := range f.handles { + if handle.path == oldPath { + handle.path = newPath + handle.nativePath = newNative + } } + f.mu.Unlock() + f.bumpNamespaceVersion() return 0 } defer f.mu.Unlock() @@ -705,6 +1027,12 @@ func (f *Filesystem) Rename(oldName string, newName string) syscall.Errno { } delete(f.paths, oldPath) f.paths[newPath] = sessionID + for _, handle := range f.handles { + if handle.path == oldPath { + handle.path = newPath + } + } + f.bumpNamespaceVersion() return 0 } @@ -715,18 +1043,131 @@ func (f *Filesystem) Unlink(name string) syscall.Errno { cleaned := cleanPath(name) f.mu.RLock() _, managed := f.paths[cleaned] + busy := f.nativePathBusyLocked(cleaned) f.mu.RUnlock() if managed { + if fskitOpenUnlinkPath(cleaned) { + f.mu.Lock() + delete(f.paths, cleaned) + delete(f.retained, cleaned) + f.mu.Unlock() + f.bumpNamespaceVersion() + return 0 + } return syscall.EPERM } + if busy { + return syscall.EBUSY + } nativePath, ok := f.nativePath(cleaned) if !ok { return syscall.ENOENT } err := os.Remove(nativePath) + if err == nil { + f.mu.Lock() + delete(f.nativeAppends, filepath.Clean(nativePath)) + f.mu.Unlock() + f.bumpNamespaceVersion() + } return errnoFor(err) } +func (f *Filesystem) Rmdir(name string) syscall.Errno { + if !f.canonical { + return syscall.EPERM + } + cleaned := cleanPath(name) + if cleaned == "/" || cleaned == "/sessions" || cleaned == "/archived_sessions" || !canonicalNamespacePath(cleaned) { + return syscall.EPERM + } + f.mu.RLock() + for route := range f.paths { + if path.Dir(route) == cleaned || strings.HasPrefix(route, cleaned+"/") { + f.mu.RUnlock() + return syscall.ENOTEMPTY + } + } + root := f.nativeRoot + f.mu.RUnlock() + if root == "" { + return syscall.ENOENT + } + if err := os.Remove(nativePathFromRoot(root, cleaned)); err != nil { + return errnoFor(err) + } + f.mu.Lock() + delete(f.directories, cleaned) + f.mu.Unlock() + f.bumpNamespaceVersion() + return 0 +} + +func (f *Filesystem) SyncAll() syscall.Errno { + f.mu.RLock() + handles := make([]uint64, 0, len(f.handles)) + for handleID := range f.handles { + handles = append(handles, handleID) + } + f.mu.RUnlock() + for _, handleID := range handles { + if errno := f.Fsync(handleID); errno != 0 && errno != syscall.EBADF { + return errno + } + } + return 0 +} + +func (f *Filesystem) loadNativeAppendState(nativePath string) (*nativeAppendState, error) { + cleaned := filepath.Clean(nativePath) + f.mu.RLock() + state := f.nativeAppends[cleaned] + journalRoot := f.nativeJournalRoot + f.mu.RUnlock() + if state != nil { + if err := state.RefreshIfIdle(); err != nil { + return nil, err + } + return state, nil + } + created, err := newNativeAppendState(cleaned, journalRoot) + if err != nil { + return nil, err + } + f.mu.Lock() + if state = f.nativeAppends[cleaned]; state == nil { + f.nativeAppends[cleaned] = created + state = created + } + f.mu.Unlock() + if state != created { + if err := state.RefreshIfIdle(); err != nil { + return nil, err + } + } + return state, nil +} + +func (f *Filesystem) nativeAppendState(nativePath string) *nativeAppendState { + f.mu.RLock() + state := f.nativeAppends[filepath.Clean(nativePath)] + f.mu.RUnlock() + return state +} + +func (f *Filesystem) nativePathBusyLocked(name string) bool { + nativePath := filepath.Clean(nativePathFromRoot(f.nativeRoot, name)) + if state := f.nativeAppends[nativePath]; state != nil && state.HasPending() { + return true + } + for _, handle := range f.handles { + if filepath.Clean(handle.nativePath) == nativePath { + return true + } + } + return false +} + func (f *Filesystem) sessionForPath(name string) (*vfs.Session, syscall.Errno) { cleaned := cleanPath(name) if f.canonical { @@ -800,6 +1241,15 @@ func canonicalSessionPath(name string) bool { return strings.HasPrefix(name, "/sessions/") || strings.HasPrefix(name, "/archived_sessions/") } +func fskitOpenUnlinkPath(name string) bool { + base := path.Base(name) + return canonicalNamespacePath(name) && strings.HasPrefix(base, ".nfs.") && len(base) > len(".nfs.") +} + +func nativeTransactionPath(name string) bool { + return canonicalSessionPath(name) && !strings.HasPrefix(path.Base(name), "._") +} + func canonicalNamespacePath(name string) bool { return name == "/" || name == "/sessions" || name == "/archived_sessions" || strings.HasPrefix(name, "/sessions/") || strings.HasPrefix(name, "/archived_sessions/") @@ -863,6 +1313,18 @@ func (f *Filesystem) handle(handleID uint64) (*fileHandle, syscall.Errno) { return handle, 0 } +func (f *Filesystem) HandlePath(handleID uint64) (string, syscall.Errno) { + f.mu.RLock() + handle := f.handles[handleID] + if handle == nil { + f.mu.RUnlock() + return "", syscall.EBADF + } + name := handle.path + f.mu.RUnlock() + return name, 0 +} + func (f *Filesystem) nativePath(name string) (string, bool) { f.mu.RLock() root := f.nativeRoot @@ -874,6 +1336,66 @@ func (f *Filesystem) nativePath(name string) (string, bool) { return nativePathFromRoot(root, name), true } +func (f *Filesystem) nativeMetadataPath(name string) (string, bool) { + f.mu.RLock() + root := f.nativeRoot + f.mu.RUnlock() + if !f.canonical || root == "" || !canonicalNamespacePath(name) { + return "", false + } + if name == "/" { + return root, true + } + return nativePathFromRoot(root, name), true +} + +func (f *Filesystem) metadataPath(name string) (string, bool, syscall.Errno) { + cleaned := cleanPath(name) + if session, errno := f.sessionForPath(cleaned); errno == 0 { + return session.MetadataPath(), true, 0 + } + if nativePath, ok := f.nativeMetadataPath(cleaned); ok { + if _, err := os.Lstat(nativePath); err != nil { + return "", false, errnoFor(err) + } + return nativePath, false, 0 + } + return "", false, syscall.ENOENT +} + +func (f *Filesystem) xattrPath(name string, create bool) (string, bool, syscall.Errno) { + cleaned := cleanPath(name) + if _, errno := f.sessionForPath(cleaned); errno == 0 { + f.mu.RLock() + root := f.nativeRoot + f.mu.RUnlock() + if root == "" { + return "", true, syscall.ENOTSUP + } + carrier := managedXattrCarrier(root, cleaned) + if create { + if err := os.MkdirAll(filepath.Dir(carrier), 0o700); err != nil { + return "", true, errnoFor(err) + } + file, err := os.OpenFile(carrier, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return "", true, errnoFor(err) + } + if err := file.Close(); err != nil { + return "", true, errnoFor(err) + } + } + return carrier, true, 0 + } + if nativePath, ok := f.nativeMetadataPath(cleaned); ok { + if _, err := os.Lstat(nativePath); err != nil { + return "", false, errnoFor(err) + } + return nativePath, false, 0 + } + return "", false, syscall.ENOENT +} + func (f *Filesystem) registerRetainedPathLocked(sessionID string, session *vfs.Session) { for retained, currentID := range f.retained { if currentID == sessionID { @@ -909,6 +1431,10 @@ func errnoFor(err error) syscall.Errno { if err == nil { return 0 } + var errno syscall.Errno + if errors.As(err, &errno) { + return errno + } switch { case errors.Is(err, vfs.ErrWriterBusy): return syscall.EBUSY diff --git a/internal/mountfs/filesystem_test.go b/internal/mountfs/filesystem_test.go index 66e161b..7893204 100644 --- a/internal/mountfs/filesystem_test.go +++ b/internal/mountfs/filesystem_test.go @@ -181,6 +181,209 @@ func TestFilesystemStaleTailOffsetWithArbitraryBytesUsesCopyOnWrite(t *testing.T } } +func TestNativePassthroughPreservesOutOfOrderAppendChunksByOffset(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := "/sessions/2026/07/16/rollout-repro.jsonl" + nativePath := nativePathFromRoot(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + large := append([]byte("{\"large\":\""), bytes.Repeat([]byte("x"), 40*1024)...) + large = append(large, []byte("\"}\n")...) + small := []byte("{\"record\":2}\n") + split := 32 * 1024 + + largeHandle, errno := filesystem.Open(route, os.O_WRONLY|os.O_APPEND) + if errno != 0 { + t.Fatalf("open large writer: %v", errno) + } + defer filesystem.Release(largeHandle) + smallHandle, errno := filesystem.Open(route, os.O_WRONLY|os.O_APPEND) + if errno != 0 { + t.Fatalf("open small writer: %v", errno) + } + defer filesystem.Release(smallHandle) + + baseOffset := int64(len(base)) + if n, errno := filesystem.Write(largeHandle, large[:split], baseOffset); errno != 0 || n != split { + t.Fatalf("write large prefix: n=%d errno=%v", n, errno) + } + if n, errno := filesystem.Write(smallHandle, small, baseOffset+int64(len(large))); errno != 0 || n != len(small) { + t.Fatalf("write later record: n=%d errno=%v", n, errno) + } + if n, errno := filesystem.Write(largeHandle, large[split:], baseOffset+int64(split)); errno != 0 || n != len(large)-split { + t.Fatalf("write large suffix: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(largeHandle); errno != 0 { + t.Fatalf("commit out-of-order chunks: %v", errno) + } + + got, err := os.ReadFile(nativePath) + if err != nil { + t.Fatal(err) + } + want := append(append(append([]byte(nil), base...), large...), small...) + if !bytes.Equal(got, want) { + t.Fatalf("native append chunks interleaved: got=%d bytes want=%d bytes", len(got), len(want)) + } +} + +func TestNativePassthroughRetryAtOverlappingOffsetIsIdempotent(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := "/sessions/2026/07/16/rollout-retry.jsonl" + nativePath := nativePathFromRoot(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + handle, errno := filesystem.Open(route, os.O_WRONLY|os.O_APPEND) + if errno != 0 { + t.Fatalf("open writer: %v", errno) + } + defer filesystem.Release(handle) + record := []byte("{\"payload\":{\"internal_chat_message_metadata_passthrough\":{\"turn_id\":\"turn\"}}}\n") + retry := record[len(record)-71:] + baseOffset := int64(len(base)) + if n, errno := filesystem.Write(handle, record, baseOffset); errno != 0 || n != len(record) { + t.Fatalf("write record: n=%d errno=%v", n, errno) + } + if n, errno := filesystem.Write(handle, retry, baseOffset+int64(len(record)-len(retry))); errno != 0 || n != len(retry) { + t.Fatalf("retry suffix: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("commit overlapping retry: %v", errno) + } + + got, err := os.ReadFile(nativePath) + if err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), base...), record...) + if !bytes.Equal(got, want) { + t.Fatalf("overlapping retry was appended: got=%q want=%q", got, want) + } +} + +func TestNativePassthroughStagesAppendUntilFsync(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := "/sessions/2026/07/16/rollout-staged.jsonl" + nativePath := nativePathFromRoot(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + writer, errno := filesystem.Open(route, os.O_WRONLY|os.O_APPEND) + if errno != 0 { + t.Fatalf("open writer: %v", errno) + } + defer filesystem.Release(writer) + record := []byte("{\"record\":1}\n") + if n, errno := filesystem.Write(writer, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("stage append: n=%d errno=%v", n, errno) + } + + backing, err := os.ReadFile(nativePath) + if err != nil || !bytes.Equal(backing, base) { + t.Fatalf("uncommitted bytes reached backing: got=%q err=%v", backing, err) + } + attribute, errno := filesystem.Getattr(route) + if errno != 0 || attribute.Size != int64(len(base)+len(record)) { + t.Fatalf("visible staged size=%d errno=%v", attribute.Size, errno) + } + reader, errno := filesystem.Open(route, os.O_RDONLY) + if errno != 0 { + t.Fatalf("open staged reader: %v", errno) + } + defer filesystem.Release(reader) + visible := make([]byte, len(base)+len(record)) + if n, errno := filesystem.Read(reader, visible, 0); errno != 0 || n != len(visible) { + t.Fatalf("read staged bytes: n=%d errno=%v", n, errno) + } + want := append(append([]byte(nil), base...), record...) + if !bytes.Equal(visible, want) { + t.Fatalf("staged visible bytes=%q want=%q", visible, want) + } + + if errno := filesystem.Fsync(writer); errno != 0 { + t.Fatalf("commit staged append: %v", errno) + } + backing, err = os.ReadFile(nativePath) + if err != nil || !bytes.Equal(backing, want) { + t.Fatalf("committed backing=%q err=%v", backing, err) + } +} + +func TestNativePassthroughInvalidAppendFailsClosed(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := "/sessions/2026/07/16/rollout-invalid.jsonl" + nativePath := nativePathFromRoot(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + writer, errno := filesystem.Open(route, os.O_WRONLY|os.O_APPEND) + if errno != 0 { + t.Fatalf("open writer: %v", errno) + } + defer filesystem.Release(writer) + if n, errno := filesystem.Write(writer, []byte("not-json\n"), int64(len(base))); errno != 0 || n != len("not-json\n") { + t.Fatalf("stage invalid append: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(writer); errno != syscall.EIO { + t.Fatalf("invalid append fsync errno=%v, want EIO", errno) + } + backing, err := os.ReadFile(nativePath) + if err != nil || !bytes.Equal(backing, base) { + t.Fatalf("invalid append changed backing: got=%q err=%v", backing, err) + } + attribute, errno := filesystem.Getattr(route) + if errno != 0 || attribute.Size != int64(len(base)) { + t.Fatalf("invalid append remained visible: size=%d errno=%v", attribute.Size, errno) + } + + record := []byte("{\"record\":1}\n") + if n, errno := filesystem.Write(writer, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("retry valid append: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(writer); errno != 0 { + t.Fatalf("commit valid retry: %v", errno) + } + want := append(append([]byte(nil), base...), record...) + backing, err = os.ReadFile(nativePath) + if err != nil || !bytes.Equal(backing, want) { + t.Fatalf("valid retry backing=%q err=%v", backing, err) + } +} + func TestFilesystemPathTruncateUsesTheActiveWriter(t *testing.T) { filesystem, source := mountFixture(t) handle, errno := filesystem.Open("/session.jsonl", os.O_RDWR) @@ -633,7 +836,7 @@ func TestCanonicalFilesystemPassesThroughNativeSessionFiles(t *testing.T) { if errno != 0 { t.Fatalf("native create Open errno=%v", errno) } - createdBytes := []byte("created-session\n") + createdBytes := []byte("{\"created\":\"session\"}\n") if n, errno := filesystem.Write(created, createdBytes, 0); errno != 0 || n != len(createdBytes) { t.Fatalf("native create Write = %d errno=%v", n, errno) } @@ -662,6 +865,65 @@ func TestCanonicalFilesystemPassesThroughNativeSessionFiles(t *testing.T) { } } +func TestCanonicalFilesystemSupportsFSKitOpenUnlinkStaging(t *testing.T) { + root := t.TempDir() + directory := filepath.Join(root, "sessions", "2026", "07", "12") + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + + original := "/sessions/2026/07/12/open.jsonl" + hidden := "/sessions/2026/07/12/.nfs.20051026.83fd" + base := []byte("{\"record\":0}\n") + if err := os.WriteFile(filepath.Join(directory, "open.jsonl"), base, 0o600); err != nil { + t.Fatal(err) + } + handle, errno := filesystem.Open(original, os.O_RDWR) + if errno != 0 { + t.Fatalf("open errno=%v", errno) + } + if errno := filesystem.Rename(original, hidden); errno != 0 { + t.Fatalf("open-unlink rename errno=%v", errno) + } + if got, errno := filesystem.HandlePath(handle); errno != 0 || got != hidden { + t.Fatalf("handle path = %q errno=%v, want %q", got, errno, hidden) + } + appended := []byte("{\"record\":1}\n") + if n, errno := filesystem.Write(handle, appended, int64(len(base))); errno != 0 || n != len(appended) { + t.Fatalf("write after hidden rename = %d errno=%v", n, errno) + } + if errno := filesystem.Release(handle); errno != 0 { + t.Fatalf("release errno=%v", errno) + } + hiddenNative := filepath.Join(directory, filepath.Base(hidden)) + want := append(append([]byte(nil), base...), appended...) + if got, err := os.ReadFile(hiddenNative); err != nil || !bytes.Equal(got, want) { + t.Fatalf("hidden bytes = %q err=%v, want %q", got, err, want) + } + if errno := filesystem.Unlink(hidden); errno != 0 { + t.Fatalf("unlink hidden errno=%v", errno) + } + if _, err := os.Stat(hiddenNative); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("hidden file remained: %v", err) + } + + second := "/sessions/2026/07/12/second.jsonl" + if err := os.WriteFile(filepath.Join(directory, "second.jsonl"), base, 0o600); err != nil { + t.Fatal(err) + } + if errno := filesystem.Rename(second, "/sessions/2026/07/12/.not-nfs"); errno != syscall.EPERM { + t.Fatalf("non-FSKit hidden rename errno=%v, want EPERM", errno) + } + if errno := filesystem.Rename(second, "/archived_sessions/.nfs.20051026.83fd"); errno != syscall.EPERM { + t.Fatalf("cross-directory open-unlink rename errno=%v, want EPERM", errno) + } +} + func TestFilesystemUpsertChangesNewOpensWithoutInvalidatingExistingHandles(t *testing.T) { first := mountSessionFixture(t, "first-session", []byte("first")) second := mountSessionFixture(t, "second-session", []byte("second")) diff --git a/internal/mountfs/fuse_integration_linux_test.go b/internal/mountfs/fuse_integration_linux_test.go new file mode 100644 index 0000000..a4a476f --- /dev/null +++ b/internal/mountfs/fuse_integration_linux_test.go @@ -0,0 +1,405 @@ +//go:build linux && fuse && fuse3 && cgo + +package mountfs + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "sync" + "testing" + "time" +) + +func TestRealFuse3ManagedReadWriteAndRestart(t *testing.T) { + requireRealFuse3(t) + root := t.TempDir() + source := []byte("{\"record\":0}\n") + managed := mountSessionFixture(t, "linux-managed", source) + filesystem := New() + if err := filesystem.AddSession("linux-managed", managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealFuse3Mount(t, mountPoint, filesystem) + target := filepath.Join(mountPoint, "linux-managed.jsonl") + if got, err := os.ReadFile(target); err != nil || !bytes.Equal(got, source) { + t.Fatalf("initial managed read = %q err=%v", got, err) + } + + file, err := os.OpenFile(target, os.O_RDWR|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + tail := []byte("{\"record\":1}\n") + if _, err := file.Write(tail); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), source...), tail...) + + file, err = os.OpenFile(target, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteAt([]byte("PATCH"), 2); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Truncate(int64(len(want) - 2)); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + copy(want[2:], []byte("PATCH")) + want = want[:len(want)-2] + if got, err := os.ReadFile(target); err != nil || !bytes.Equal(got, want) { + t.Fatalf("mutated managed read = %q err=%v", got, err) + } + if managed.State().BackingPath == "" { + t.Fatal("random write did not enter copy-on-write backing") + } + + stopMount() + waitForRealFuse3Unmount(t, mountPoint) + assertRealFuse3BackingSealed(t, mountPoint) + stopRemount := startRealFuse3Mount(t, mountPoint, filesystem) + if got, err := os.ReadFile(target); err != nil || !bytes.Equal(got, want) { + t.Fatalf("remounted managed read = %q err=%v", got, err) + } + stopRemount() + waitForRealFuse3Unmount(t, mountPoint) + assertRealFuse3BackingSealed(t, mountPoint) +} + +func TestRealFuse3CanonicalArchiveUnarchiveRename(t *testing.T) { + requireRealFuse3(t) + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + activeDirectory := filepath.Join(nativeRoot, "sessions", "2026", "07", "16") + archivedDirectory := filepath.Join(nativeRoot, "archived_sessions") + for _, directory := range []string{activeDirectory, archivedDirectory} { + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + } + source := []byte("{\"canonical\":true}\n") + managed := mountSessionFixture(t, "linux-canonical", source) + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + filename := "rollout-linux-canonical.jsonl" + if err := filesystem.AddSessionAt("linux-canonical", "/archived_sessions/"+filename, managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealFuse3Mount(t, mountPoint, filesystem) + archivedPath := filepath.Join(mountPoint, "archived_sessions", filename) + activePath := filepath.Join(mountPoint, "sessions", "2026", "07", "16", filename) + if err := os.Rename(archivedPath, activePath); err != nil { + t.Fatalf("unarchive rename: %v", err) + } + if got, err := os.ReadFile(activePath); err != nil || !bytes.Equal(got, source) { + t.Fatalf("active managed read = %q err=%v", got, err) + } + if _, err := os.Stat(archivedPath); !os.IsNotExist(err) { + t.Fatalf("archived route remained after unarchive: %v", err) + } + if err := os.Rename(activePath, archivedPath); err != nil { + t.Fatalf("archive rename: %v", err) + } + if got, err := os.ReadFile(archivedPath); err != nil || !bytes.Equal(got, source) { + t.Fatalf("restored archived read = %q err=%v", got, err) + } + nativePath := filepath.Join(nativeRoot, "archived_sessions", filename) + nativeBytes := []byte("{\"native_fallback\":true}\n") + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + if err := filesystem.PreferNativeSession("linux-canonical"); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(archivedPath); err != nil || !bytes.Equal(got, nativeBytes) { + t.Fatalf("preferred native read = %q err=%v", got, err) + } + if err := os.Remove(nativePath); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(archivedPath); err != nil || !bytes.Equal(got, source) { + t.Fatalf("managed fallback read = %q err=%v", got, err) + } + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + if err := filesystem.RemoveSession("linux-canonical"); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(archivedPath); err != nil || !bytes.Equal(got, nativeBytes) { + t.Fatalf("native read after managed removal = %q err=%v", got, err) + } + stopMount() + waitForRealFuse3Unmount(t, mountPoint) + assertRealFuse3BackingSealed(t, mountPoint) +} + +func TestRealFuse3HostCrashUnmountsAndRestarts(t *testing.T) { + requireRealFuse3(t) + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := filepath.Join("sessions", "2026", "07", "16", "rollout-crash.jsonl") + nativePath := filepath.Join(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + source := []byte("{\"crash_recovery\":true}\n") + if err := os.WriteFile(nativePath, source, 0o600); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + + for attempt := 1; attempt <= 2; attempt++ { + process, done, output := startRealFuse3Helper(t, mountPoint, nativeRoot) + waitForRealFuse3ProcessMount(t, mountPoint, done, output) + mountedPath := filepath.Join(mountPoint, route) + if got, err := os.ReadFile(mountedPath); err != nil || !bytes.Equal(got, source) { + t.Fatalf("attempt %d mounted read = %q err=%v", attempt, got, err) + } + if err := process.Kill(); err != nil { + t.Fatal(err) + } + if err := <-done; err == nil { + t.Fatalf("attempt %d killed helper exited successfully", attempt) + } + if attempt == 2 { + if err := recoverStaleMount(mountPoint); err != nil { + t.Fatal(err) + } + waitForRealFuse3Unmount(t, mountPoint) + assertRealFuse3BackingSealed(t, mountPoint) + } + } +} + +func TestRealFuse3ReadAndFsyncPerformance(t *testing.T) { + requireRealFuse3(t) + root := t.TempDir() + line := []byte("{\"payload\":\"0123456789abcdef0123456789abcdef0123456789abcdef\"}\n") + source := bytes.Repeat(line, (16<<20)/len(line)+1) + source = source[:16<<20] + managed := mountSessionFixture(t, "linux-performance", source) + filesystem := New() + if err := filesystem.AddSession("linux-performance", managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealFuse3Mount(t, mountPoint, filesystem) + target := filepath.Join(mountPoint, "linux-performance.jsonl") + + readStart := time.Now() + file, err := os.Open(target) + if err != nil { + t.Fatal(err) + } + readBytes, copyErr := io.Copy(io.Discard, file) + closeErr := file.Close() + if copyErr != nil || closeErr != nil || readBytes != int64(len(source)) { + t.Fatalf("mounted performance read bytes=%d copy=%v close=%v", readBytes, copyErr, closeErr) + } + readDuration := time.Since(readStart) + readMiBPerSecond := float64(readBytes) / (1024 * 1024) / readDuration.Seconds() + if readMiBPerSecond < 25 { + t.Fatalf("mounted read throughput %.2f MiB/s is below the 25 MiB/s safety floor", readMiBPerSecond) + } + + file, err = os.OpenFile(target, os.O_RDWR|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + durations := make([]time.Duration, 0, 50) + for index := range 50 { + started := time.Now() + if _, err := fmt.Fprintf(file, "{\"append\":%d}\n", index); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + t.Fatal(err) + } + durations = append(durations, time.Since(started)) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) + p95 := durations[(len(durations)*95+99)/100-1] + if p95 > 250*time.Millisecond { + t.Fatalf("mounted append+fsync p95 %s exceeds the 250ms safety ceiling", p95) + } + t.Logf("FUSE3 read=%.2f MiB/s append_fsync_p95=%s", readMiBPerSecond, p95) + stopMount() + waitForRealFuse3Unmount(t, mountPoint) + assertRealFuse3BackingSealed(t, mountPoint) +} + +func TestRealFuse3CrashHelper(t *testing.T) { + if os.Getenv("CODEXFOLD_FUSE3_CRASH_HELPER") != "1" { + t.Skip("helper process") + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(os.Getenv("CODEXFOLD_FUSE3_NATIVE_ROOT")) + if err := Mount(context.Background(), HostOptions{ + MountPoint: os.Getenv("CODEXFOLD_FUSE3_MOUNT_POINT"), Filesystem: filesystem, Foreground: true, + }); err != nil { + t.Fatal(err) + } +} + +func requireRealFuse3(t *testing.T) { + t.Helper() + if os.Getenv("CODEXFOLD_RUN_FUSE3_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE3_TEST=1 to run the real Linux FUSE3 adapter test") + } +} + +func startRealFuse3Mount(t *testing.T, mountPoint string, filesystem *Filesystem) func() { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + mountDone := make(chan error, 1) + go func() { + mountDone <- Mount(ctx, HostOptions{MountPoint: mountPoint, Filesystem: filesystem, Foreground: true}) + }() + var stopOnce sync.Once + stop := func() { + stopOnce.Do(func() { + cancel() + select { + case err := <-mountDone: + if err != nil && !errors.Is(err, context.Canceled) { + t.Errorf("FUSE3 mount shutdown: %v", err) + } + case <-time.After(10 * time.Second): + t.Error("FUSE3 mount did not stop after cancellation") + } + }) + } + t.Cleanup(stop) + deadline := time.Now().Add(20 * time.Second) + var lastProbeErr error + for time.Now().Before(deadline) { + if err := probeRealFuse3Mount(mountPoint); err == nil { + return stop + } else { + lastProbeErr = err + } + select { + case err := <-mountDone: + t.Fatalf("FUSE3 mount exited before health: %v", err) + case <-time.After(100 * time.Millisecond): + } + } + t.Fatalf("FUSE3 mount did not become healthy: %v", lastProbeErr) + return stop +} + +func waitForRealFuse3Unmount(t *testing.T, mountPoint string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if !linuxFuseMountVisible(mountPoint) { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("FUSE3 mount remained active after shutdown") +} + +func probeRealFuse3Mount(mountPoint string) error { + identity, err := os.ReadFile(filepath.Join(mountPoint, ".codexfold-health")) + if err != nil { + return err + } + if len(identity) < 16 { + return fmt.Errorf("mount identity is too short: %d", len(identity)) + } + return nil +} + +func assertRealFuse3BackingSealed(t *testing.T, mountPoint string) { + t.Helper() + info, err := os.Stat(mountPoint) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm()&0o200 != 0 { + t.Fatalf("unmounted FUSE3 backing remained writable: mode=%#o", info.Mode().Perm()) + } +} + +func startRealFuse3Helper(t *testing.T, mountPoint string, nativeRoot string) (*os.Process, <-chan error, *bytes.Buffer) { + t.Helper() + command := exec.Command(os.Args[0], "-test.run=^TestRealFuse3CrashHelper$", "-test.v") + command.Env = append(os.Environ(), + "CODEXFOLD_FUSE3_CRASH_HELPER=1", + "CODEXFOLD_FUSE3_MOUNT_POINT="+mountPoint, + "CODEXFOLD_FUSE3_NATIVE_ROOT="+nativeRoot, + ) + var output bytes.Buffer + command.Stdout = &output + command.Stderr = &output + if err := command.Start(); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- command.Wait() }() + return command.Process, done, &output +} + +func waitForRealFuse3ProcessMount(t *testing.T, mountPoint string, done <-chan error, output *bytes.Buffer) { + t.Helper() + deadline := time.Now().Add(20 * time.Second) + var lastProbeErr error + for time.Now().Before(deadline) { + if err := probeRealFuse3Mount(mountPoint); err == nil { + return + } else { + lastProbeErr = err + } + select { + case err := <-done: + t.Fatalf("FUSE3 helper exited before health: %v output=%s", err, output.String()) + case <-time.After(100 * time.Millisecond): + } + } + t.Fatalf("FUSE3 helper did not become healthy: %v output=%s", lastProbeErr, output.String()) +} diff --git a/internal/mountfs/fuse_integration_test.go b/internal/mountfs/fuse_integration_test.go index d01e3ba..d9db5ef 100644 --- a/internal/mountfs/fuse_integration_test.go +++ b/internal/mountfs/fuse_integration_test.go @@ -8,9 +8,11 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "fmt" "io" "os" "path/filepath" + "sort" "strings" "sync" "sync/atomic" @@ -32,11 +34,15 @@ func TestOperationTraceRecordsWriteShapeWithoutPath(t *testing.T) { privatePath := "/sessions/private-session.jsonl" filesystem.recordOpen("open", privatePath, 0x9, os.O_WRONLY|os.O_APPEND, 17, 0) filesystem.recordIO("write", privatePath, 17, 1234, 89, 89) + filesystem.recordIO("read", privatePath, 17, 1200, 64, 64) + filesystem.recordHandleResult("flush", privatePath, 17, 0) joined := strings.Join(recorded, "\n") for _, field := range []string{ "open kind=session flags=0x9 translated=0x9 handle=17 result=0", "write kind=session handle=17 offset=1234 bytes=89 result=89", + "read kind=session handle=17 offset=1200 bytes=64 result=64", + "flush kind=session handle=17 result=0", } { if !strings.Contains(joined, field) { t.Fatalf("operation trace missing %q: %s", field, joined) @@ -79,9 +85,20 @@ func TestOpenExUsesDirectIOOnlyForWritableSessions(t *testing.T) { } } +func TestFuseStatfsFallsBackToMountParentWithoutNativeRoot(t *testing.T) { + filesystem := &fuseFilesystem{core: New(), statRoot: t.TempDir()} + var stat fuse.Statfs_t + if result := filesystem.Statfs("/", &stat); result != 0 { + t.Fatalf("Statfs result=%d", result) + } + if stat.Bsize == 0 || stat.Blocks == 0 { + t.Fatalf("Statfs returned no backing capacity: %#v", stat) + } +} + func TestRealFuseMountNativeFileOperations(t *testing.T) { if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { - t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real macFUSE adapter test") + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") } root := t.TempDir() source := []byte("first\nsecond\nthird\n") @@ -155,6 +172,10 @@ func TestRealFuseMountNativeFileOperations(t *testing.T) { _ = appendFile.Close() t.Fatal(err) } + if _, err := unix.FcntlInt(appendFile.Fd(), unix.F_FULLFSYNC, 0); err != nil { + _ = appendFile.Close() + t.Fatalf("F_FULLFSYNC: %v", err) + } if err := appendFile.Close(); err != nil { t.Fatal(err) } @@ -208,6 +229,105 @@ func TestRealFuseMountNativeFileOperations(t *testing.T) { waitForRealUnmount(t, mountPoint) } +func TestRealFuseTReadAndFsyncPerformance(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + line := []byte("{\"payload\":\"0123456789abcdef0123456789abcdef0123456789abcdef\"}\n") + source := bytes.Repeat(line, (32<<20)/len(line)+1) + source = source[:32<<20] + nativeReadPath := filepath.Join(root, "native-read.jsonl") + if err := os.WriteFile(nativeReadPath, source, 0o600); err != nil { + t.Fatal(err) + } + + managed := mountSessionFixture(t, "darwin-performance", source) + filesystem := New() + if err := filesystem.AddSession("darwin-performance", managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealMount(t, mountPoint, filesystem) + target := filepath.Join(mountPoint, "darwin-performance.jsonl") + + nativeRead := bestSequentialRead(t, nativeReadPath, int64(len(source)), 3) + fuseRead := bestSequentialRead(t, target, int64(len(source)), 3) + ratio := fuseRead / nativeRead + if fuseRead < 1024 || ratio < 0.25 { + t.Fatalf("FUSE-T read %.2f MiB/s is %.1f%% of APFS %.2f MiB/s", fuseRead, ratio*100, nativeRead) + } + + nativeAppendPath := filepath.Join(root, "native-append.jsonl") + if err := os.WriteFile(nativeAppendPath, []byte("{\"record\":0}\n"), 0o600); err != nil { + t.Fatal(err) + } + nativeP95 := appendFsyncP95(t, nativeAppendPath, 100) + fuseP95 := appendFsyncP95(t, target, 100) + ceiling := 50 * time.Millisecond + if relative := nativeP95 * 5; relative > ceiling { + ceiling = relative + } + if fuseP95 > ceiling { + t.Fatalf("FUSE-T append+fsync p95 %s exceeds ceiling %s; APFS p95=%s", fuseP95, ceiling, nativeP95) + } + t.Logf("FUSE-T read=%.2f MiB/s APFS=%.2f MiB/s ratio=%.1f%% append_fsync_p95=%s APFS_p95=%s", fuseRead, nativeRead, ratio*100, fuseP95, nativeP95) + + stopMount() + waitForRealUnmount(t, mountPoint) +} + +func bestSequentialRead(t *testing.T, path string, size int64, rounds int) float64 { + t.Helper() + best := float64(0) + for range rounds { + started := time.Now() + file, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + readBytes, copyErr := io.Copy(io.Discard, file) + closeErr := file.Close() + if copyErr != nil || closeErr != nil || readBytes != size { + t.Fatalf("sequential read %q bytes=%d copy=%v close=%v", path, readBytes, copyErr, closeErr) + } + throughput := float64(readBytes) / (1024 * 1024) / time.Since(started).Seconds() + if throughput > best { + best = throughput + } + } + return best +} + +func appendFsyncP95(t *testing.T, path string, rounds int) time.Duration { + t.Helper() + file, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + durations := make([]time.Duration, 0, rounds) + for index := range rounds { + started := time.Now() + if _, err := fmt.Fprintf(file, "{\"append\":%d}\n", index); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + t.Fatal(err) + } + durations = append(durations, time.Since(started)) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) + return durations[(len(durations)*95+99)/100-1] +} + func TestRealFuseManagedStaleTailOffsetsPreserveJSONL(t *testing.T) { if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") @@ -506,6 +626,126 @@ func TestRealFuseCanonicalNativePreferenceNeverLosesPath(t *testing.T) { waitForRealUnmount(t, mountPoint) } +func TestRealFuseCanonicalNativeAppendTransactionSurvivesRestart(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := filepath.Join("sessions", "2026", "07", "16", "rollout-native-transaction.jsonl") + nativePath := filepath.Join(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(mountPoint, route) + var traceMu sync.Mutex + var trace []string + start := func() func() { + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + if err := filesystem.RecoverNativeAppendTransactions(); err != nil { + t.Fatal(err) + } + return startRealMountWithOptions(t, HostOptions{ + MountPoint: mountPoint, Filesystem: filesystem, Foreground: true, + OperationRecorder: func(operation string) { + traceMu.Lock() + trace = append(trace, operation) + traceMu.Unlock() + }, + }) + } + + stopMount := start() + large := append([]byte("{\"large\":\""), bytes.Repeat([]byte("x"), 1<<20)...) + large = append(large, []byte("\"}\n")...) + file, err := os.OpenFile(target, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + if n, err := file.Write(large); err != nil || n != len(large) { + _ = file.Close() + t.Fatalf("write large append: n=%d err=%v", n, err) + } + want := append(append([]byte(nil), base...), large...) + backingAfterWrite, err := os.ReadFile(nativePath) + if err != nil || (!bytes.Equal(backingAfterWrite, base) && !bytes.Equal(backingAfterWrite, want)) { + _ = file.Close() + traceMu.Lock() + currentTrace := strings.Join(trace, "\n") + traceMu.Unlock() + t.Fatalf("backing exposed a partial transaction: bytes=%d err=%v trace=%s", len(backingAfterWrite), err, currentTrace) + } + visible, err := os.ReadFile(target) + if err != nil || !bytes.Equal(visible, want) { + _ = file.Close() + t.Fatalf("pending mounted view: bytes=%d err=%v want=%d", len(visible), err, len(want)) + } + if err := file.Sync(); err != nil { + _ = file.Close() + t.Fatalf("fsync large append: %v", err) + } + if err := file.Close(); err != nil { + t.Fatalf("close large append: %v", err) + } + assertNativeBytes(t, nativePath, want) + + stopMount() + waitForRealUnmount(t, mountPoint) + stopMount = start() + waitForRealFile(t, target, want) + + afterRestart := []byte("{\"after_restart\":true}\n") + file, err = os.OpenFile(target, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + if n, err := file.Write(afterRestart); err != nil || n != len(afterRestart) { + _ = file.Close() + t.Fatalf("write after restart: n=%d err=%v", n, err) + } + if err := file.Close(); err != nil { + t.Fatalf("release commit after restart: %v", err) + } + want = append(want, afterRestart...) + assertNativeBytes(t, nativePath, want) + if !completeJSONL(want) { + t.Fatal("real FUSE append produced invalid JSONL") + } + + traceMu.Lock() + joined := strings.Join(trace, "\n") + traceMu.Unlock() + for _, marker := range []string{"open kind=session", "write kind=session", "fsync", "flush", "release"} { + if !strings.Contains(joined, marker) { + t.Fatalf("real append trace missing %q: %s", marker, joined) + } + } + if writes := strings.Count(joined, "write kind=session"); writes < 30 { + t.Fatalf("large append was not exercised as split FUSE writes: writes=%d", writes) + } + for _, entry := range strings.Split(joined, "\n") { + if strings.Contains(entry, "kind=session") && strings.Contains(entry, "result=-") { + t.Fatalf("session operation failed in real append trace: %s", entry) + } + } + for _, entry := range strings.Split(joined, "\n") { + if strings.Contains(entry, "kind=appledouble") && strings.Contains(entry, "result=-5") { + t.Fatalf("AppleDouble metadata was routed through JSONL validation: %s", entry) + } + } + stopMount() + waitForRealUnmount(t, mountPoint) +} + func TestRealFuseMountCanonicalManagedRename(t *testing.T) { if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") @@ -650,7 +890,8 @@ func waitForRealUnmount(t *testing.T, mountPoint string) { t.Helper() deadline := time.Now().Add(10 * time.Second) for time.Now().Before(deadline) { - if err := service.ProbeMount(mountPoint); err != nil { + var stat unix.Statfs_t + if err := unix.Statfs(mountPoint, &stat); err != nil || !sameRealMountPath(unix.ByteSliceToString(stat.Mntonname[:]), mountPoint) { return } time.Sleep(100 * time.Millisecond) @@ -658,6 +899,17 @@ func waitForRealUnmount(t *testing.T, mountPoint string) { t.Fatal("FUSE mount remained active after shutdown") } +func sameRealMountPath(left string, right string) bool { + canonical := func(path string) string { + resolved, err := filepath.EvalSymlinks(path) + if err == nil { + return filepath.Clean(resolved) + } + return filepath.Clean(path) + } + return canonical(left) == canonical(right) +} + func waitForRealFile(t *testing.T, path string, want []byte) { t.Helper() deadline := time.Now().Add(5 * time.Second) diff --git a/internal/mountfs/fuse_provider_darwin.go b/internal/mountfs/fuse_provider_darwin.go new file mode 100644 index 0000000..3963243 --- /dev/null +++ b/internal/mountfs/fuse_provider_darwin.go @@ -0,0 +1,45 @@ +//go:build darwin && fuse && cgo + +package mountfs + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +const darwinFUSEtLibraryPath = "/usr/local/lib/libfuse-t.dylib" + +var darwinHigherPriorityFUSELibraries = []string{ + "/usr/local/lib/libfuse.2.dylib", + "/usr/local/lib/libosxfuse.2.dylib", +} + +func validateFuseProvider() error { + return validateDarwinFUSEProviderPaths(darwinFUSEtLibraryPath, darwinHigherPriorityFUSELibraries) +} + +func validateDarwinFUSEProviderPaths(fuseTPath string, higherPriority []string) error { + for _, path := range higherPriority { + if _, err := os.Stat(path); err == nil { + return fmt.Errorf("unsupported macOS FUSE library %q would take precedence over FUSE-T", path) + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect competing macOS FUSE library %q: %w", path, err) + } + } + + resolved, err := filepath.EvalSymlinks(fuseTPath) + if err != nil { + return fmt.Errorf("FUSE-T library %q is unavailable: %w", fuseTPath, err) + } + info, err := os.Stat(resolved) + if err != nil { + return fmt.Errorf("inspect FUSE-T library %q: %w", resolved, err) + } + if !info.Mode().IsRegular() || !strings.Contains(filepath.Base(resolved), "libfuse-t") { + return fmt.Errorf("FUSE-T library resolves to an unexpected file %q", resolved) + } + return nil +} diff --git a/internal/mountfs/fuse_provider_darwin_test.go b/internal/mountfs/fuse_provider_darwin_test.go new file mode 100644 index 0000000..e36c713 --- /dev/null +++ b/internal/mountfs/fuse_provider_darwin_test.go @@ -0,0 +1,41 @@ +//go:build darwin && fuse && cgo + +package mountfs + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestValidateDarwinFUSEProviderRequiresFUSEt(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "libfuse-t-1.2.7.dylib") + if err := os.WriteFile(target, []byte("fixture"), 0o600); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "libfuse-t.dylib") + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + + if err := validateDarwinFUSEProviderPaths(link, nil); err != nil { + t.Fatalf("valid FUSE-T layout rejected: %v", err) + } + + competitor := filepath.Join(root, "libfuse.2.dylib") + if err := os.WriteFile(competitor, []byte("fixture"), 0o600); err != nil { + t.Fatal(err) + } + if err := validateDarwinFUSEProviderPaths(link, []string{competitor}); err == nil || !strings.Contains(err.Error(), "take precedence") { + t.Fatalf("competing FUSE library was not rejected: %v", err) + } +} + +func TestValidateDarwinFUSEProviderRejectsMissingFUSEt(t *testing.T) { + err := validateDarwinFUSEProviderPaths(filepath.Join(t.TempDir(), "libfuse-t.dylib"), nil) + if err == nil || !strings.Contains(err.Error(), "unavailable") { + t.Fatalf("missing FUSE-T library was not rejected: %v", err) + } +} diff --git a/internal/mountfs/fuse_provider_other.go b/internal/mountfs/fuse_provider_other.go new file mode 100644 index 0000000..5f06e01 --- /dev/null +++ b/internal/mountfs/fuse_provider_other.go @@ -0,0 +1,5 @@ +//go:build (linux && fuse && fuse3 && cgo) || (windows && winfsp) + +package mountfs + +func validateFuseProvider() error { return nil } diff --git a/internal/mountfs/host.go b/internal/mountfs/host.go index 1594874..ad54a5c 100644 --- a/internal/mountfs/host.go +++ b/internal/mountfs/host.go @@ -14,6 +14,7 @@ type HostOptions struct { Filesystem *Filesystem Foreground bool OperationRecorder func(string) + BuildSHA256 string } func Mount(ctx context.Context, options HostOptions) error { @@ -30,7 +31,16 @@ func Mount(ctx context.Context, options HostOptions) error { } func prepareMountPoint(path string) error { + if err := recoverStaleMount(path); err != nil { + return fmt.Errorf("recover stale mount: %w", err) + } info, err := os.Lstat(path) + if os.IsNotExist(err) { + if err := os.MkdirAll(path, 0o700); err != nil { + return fmt.Errorf("create mount backing directory: %w", err) + } + info, err = os.Lstat(path) + } if err != nil { return fmt.Errorf("inspect mount backing directory: %w", err) } diff --git a/internal/mountfs/host_cgofuse.go b/internal/mountfs/host_cgofuse.go index c8bfa07..b738c71 100644 --- a/internal/mountfs/host_cgofuse.go +++ b/internal/mountfs/host_cgofuse.go @@ -1,9 +1,8 @@ -//go:build fuse && cgo +//go:build (darwin && fuse && cgo) || (linux && fuse && fuse3 && cgo) || (windows && winfsp) package mountfs import ( - "bytes" "context" "errors" "fmt" @@ -13,10 +12,11 @@ import ( "strings" "sync/atomic" "syscall" + "time" + "github.com/jstar0/codexfold/internal/buildid" "github.com/jstar0/codexfold/internal/mountid" "github.com/winfsp/cgofuse/fuse" - "golang.org/x/sys/unix" ) type fuseFilesystem struct { @@ -24,6 +24,7 @@ type fuseFilesystem struct { core *Filesystem recorder func(string) mountIdentity []byte + statRoot string mountReady atomic.Bool } @@ -32,9 +33,9 @@ const healthHandle = ^uint64(0) - 1 func Available() bool { return true } func (f *fuseFilesystem) Getattr(name string, stat *fuse.Stat_t, _ uint64) int { - f.record("getattr") if cleanPath(name) == "/"+mountid.Path { if !f.mountReady.Load() { + f.recordResult("getattr", name, -int(syscall.ENOENT)) return -int(syscall.ENOENT) } stat.Mode = syscall.S_IFREG | 0o400 @@ -43,10 +44,12 @@ func (f *fuseFilesystem) Getattr(name string, stat *fuse.Stat_t, _ uint64) int { stat.Blksize = 4096 stat.Blocks = (stat.Size + 511) / 512 stat.Uid, stat.Gid, _ = fuse.Getcontext() + f.recordResult("getattr", name, 0) return 0 } attribute, errno := f.core.Getattr(name) if errno != 0 { + f.recordResult("getattr", name, -int(errno)) return -int(errno) } stat.Mode = attribute.Mode @@ -58,6 +61,7 @@ func (f *fuseFilesystem) Getattr(name string, stat *fuse.Stat_t, _ uint64) int { stat.Ctim = stat.Mtim stat.Atim = stat.Mtim stat.Uid, stat.Gid, _ = fuse.Getcontext() + f.recordResult("getattr", name, 0) return 0 } @@ -66,27 +70,9 @@ func (f *fuseFilesystem) Statfs(name string, stat *fuse.Statfs_t) int { root := f.core.nativeRoot f.core.mu.RUnlock() if root == "" { - result := -int(syscall.ENOENT) - f.recordResult("statfs", name, result) - return result - } - var source unix.Statfs_t - result := unixResult(unix.Statfs(root, &source)) - if result == 0 { - stat.Bsize = uint64(source.Bsize) - if source.Iosize > 0 { - stat.Frsize = uint64(source.Iosize) - } else { - stat.Frsize = uint64(source.Bsize) - } - stat.Blocks = source.Blocks - stat.Bfree = source.Bfree - stat.Bavail = source.Bavail - stat.Files = source.Files - stat.Ffree = source.Ffree - stat.Favail = source.Ffree - stat.Namemax = 255 + root = f.statRoot } + result := populateFilesystemStat(root, stat) f.recordResult("statfs", name, result) return result } @@ -133,6 +119,13 @@ func (f *fuseFilesystem) Open(name string, flags int) (int, uint64) { } translated := translateOpenFlags(flags) handle, errno := f.core.Open(name, translated) + if errno == syscall.EBUSY && writableSession(name, flags) { + deadline := time.Now().Add(250 * time.Millisecond) + for errno == syscall.EBUSY && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + handle, errno = f.core.Open(name, translated) + } + } if errno != 0 { result := -int(errno) f.recordOpen("open", name, flags, translated, handle, result) @@ -174,18 +167,23 @@ func (f *fuseFilesystem) CreateEx(name string, _ uint32, info *fuse.FileInfo_t) return result } -func (f *fuseFilesystem) Read(_ string, destination []byte, offset int64, handle uint64) int { - f.record("read") +func (f *fuseFilesystem) Read(name string, destination []byte, offset int64, handle uint64) int { if handle == healthHandle { if offset < 0 || offset >= int64(len(f.mountIdentity)) { + f.recordIO("read", name, handle, offset, len(destination), 0) return 0 } - return copy(destination, f.mountIdentity[offset:]) + n := copy(destination, f.mountIdentity[offset:]) + f.recordIO("read", name, handle, offset, len(destination), n) + return n } n, errno := f.core.Read(handle, destination, offset) if errno != 0 { - return -int(errno) + result := -int(errno) + f.recordIO("read", name, handle, offset, len(destination), result) + return result } + f.recordIO("read", name, handle, offset, len(destination), n) return n } @@ -212,28 +210,34 @@ func (f *fuseFilesystem) Truncate(name string, size int64, handle uint64) int { return result } -func (f *fuseFilesystem) Flush(_ string, handle uint64) int { - f.record("flush") +func (f *fuseFilesystem) Flush(name string, handle uint64) int { if handle == healthHandle { + f.recordHandleResult("flush", name, handle, 0) return 0 } - return -int(f.core.Flush(handle)) + result := -int(f.core.Flush(handle)) + f.recordHandleResult("flush", name, handle, result) + return result } -func (f *fuseFilesystem) Fsync(_ string, _ bool, handle uint64) int { - f.record("fsync") +func (f *fuseFilesystem) Fsync(name string, dataOnly bool, handle uint64) int { if handle == healthHandle { + f.record(fmt.Sprintf("fsync kind=%s handle=%d datasync=%t result=0", operationKind(name), handle, dataOnly)) return 0 } - return -int(f.core.Fsync(handle)) + result := -int(f.core.Fsync(handle)) + f.record(fmt.Sprintf("fsync kind=%s handle=%d datasync=%t result=%d", operationKind(name), handle, dataOnly, result)) + return result } -func (f *fuseFilesystem) Release(_ string, handle uint64) int { - f.record("release") +func (f *fuseFilesystem) Release(name string, handle uint64) int { if handle == healthHandle { + f.recordHandleResult("release", name, handle, 0) return 0 } - return -int(f.core.Release(handle)) + result := -int(f.core.Release(handle)) + f.recordHandleResult("release", name, handle, result) + return result } func (f *fuseFilesystem) Mkdir(name string, mode uint32) int { @@ -326,8 +330,7 @@ func (f *fuseFilesystem) Utimens(name string, times []fuse.Timespec) int { if len(times) != 2 { result = -int(syscall.EINVAL) } else { - unixTimes := []unix.Timespec{{Sec: times[0].Sec, Nsec: times[0].Nsec}, {Sec: times[1].Sec, Nsec: times[1].Nsec}} - result = unixResult(unix.UtimesNanoAt(unix.AT_FDCWD, path, unixTimes, 0)) + result = setFileTimes(path, times) } } f.recordResult("utimens", name, result) @@ -340,7 +343,7 @@ func (f *fuseFilesystem) Setxattr(name string, attribute string, value []byte, f if errc != 0 { return errc } - return unixResult(unix.Setxattr(path, attribute, value, flags)) + return setExtendedAttribute(path, attribute, value, flags) } func (f *fuseFilesystem) Getxattr(name string, attribute string) (int, []byte) { @@ -349,16 +352,7 @@ func (f *fuseFilesystem) Getxattr(name string, attribute string) (int, []byte) { if errc != 0 { return errc, nil } - size, err := unix.Getxattr(path, attribute, nil) - if err != nil { - return unixResult(err), nil - } - value := make([]byte, size) - n, err := unix.Getxattr(path, attribute, value) - if err != nil { - return unixResult(err), nil - } - return 0, value[:n] + return getExtendedAttribute(path, attribute) } func (f *fuseFilesystem) Listxattr(name string, fill func(string) bool) int { @@ -367,17 +361,12 @@ func (f *fuseFilesystem) Listxattr(name string, fill func(string) bool) int { if errc != 0 { return errc } - size, err := unix.Listxattr(path, nil) - if err != nil { - return unixResult(err) - } - buffer := make([]byte, size) - n, err := unix.Listxattr(path, buffer) - if err != nil { - return unixResult(err) + result, attributes := listExtendedAttributes(path) + if result != 0 { + return result } - for _, attribute := range bytes.Split(bytes.TrimRight(buffer[:n], "\x00"), []byte{0}) { - if len(attribute) != 0 && !fill(string(attribute)) { + for _, attribute := range attributes { + if attribute != "" && !fill(attribute) { break } } @@ -390,7 +379,7 @@ func (f *fuseFilesystem) Removexattr(name string, attribute string) int { if errc != 0 { return errc } - return unixResult(unix.Removexattr(path, attribute)) + return removeExtendedAttribute(path, attribute) } func (f *fuseFilesystem) xattrPath(name string, create bool) (string, int) { @@ -463,6 +452,10 @@ func (f *fuseFilesystem) recordIO(operation string, name string, handle uint64, f.record(fmt.Sprintf("%s kind=%s handle=%d offset=%d bytes=%d result=%d", operation, operationKind(name), handle, offset, bytes, result)) } +func (f *fuseFilesystem) recordHandleResult(operation string, name string, handle uint64, result int) { + f.record(fmt.Sprintf("%s kind=%s handle=%d result=%d", operation, operationKind(name), handle, result)) +} + func operationKind(name string) string { kind := "other" base := filepath.Base(name) @@ -507,18 +500,44 @@ func mountHost(ctx context.Context, options HostOptions) (result error) { result = fmt.Errorf("%w: %v", ErrPrerequisite, recovered) } }() - identity, err := mountid.New() + if err := validateFuseProvider(); err != nil { + return fmt.Errorf("validate selected FUSE provider: %w", err) + } + buildSHA256 := options.BuildSHA256 + var err error + if buildSHA256 == "" { + buildSHA256, err = buildid.CurrentSHA256() + if err != nil { + return fmt.Errorf("hash mounted executable: %w", err) + } + } + identity, err := mountid.New(buildSHA256) if err != nil { return fmt.Errorf("generate mount identity: %w", err) } - filesystem := &fuseFilesystem{core: options.Filesystem, recorder: options.OperationRecorder, mountIdentity: []byte(identity)} + filesystem := &fuseFilesystem{ + core: options.Filesystem, + recorder: options.OperationRecorder, + mountIdentity: []byte(identity), + statRoot: filepath.Dir(options.MountPoint), + } host := fuse.NewFileSystemHost(filesystem) + backing, err := prepareMountedBacking(options.MountPoint) + if err != nil { + return fmt.Errorf("prepare mount backing permissions: %w", err) + } + backingClosed := false + defer func() { + if !backingClosed { + _ = backing.Close() + } + }() arguments := []string{"-o", "fsname=codexfold", "-o", "default_permissions", "-o", "attr_timeout=0", "-o", "entry_timeout=0", "-o", "negative_timeout=0"} if options.Foreground { arguments = append(arguments, "-f") } if runtime.GOOS == "darwin" { - arguments = append(arguments, "-o", "volname=CodexFold") + arguments = append(arguments, "-o", "backend=nfs", "-o", "volname=CodexFold") } done := make(chan struct{}) go func() { @@ -532,6 +551,9 @@ func mountHost(ctx context.Context, options HostOptions) (result error) { policyDone := make(chan error, 1) go func() { err := configureMountedFilesystem(policyContext, options.MountPoint) + if err == nil { + err = backing.Seal() + } if err == nil { filesystem.mountReady.Store(true) } else { @@ -542,6 +564,8 @@ func mountHost(ctx context.Context, options HostOptions) (result error) { mounted := host.Mount(options.MountPoint, arguments) cancelPolicy() policyErr := <-policyDone + backingErr := backing.Close() + backingClosed = true close(done) if err := ctx.Err(); err != nil { return err @@ -552,5 +576,8 @@ func mountHost(ctx context.Context, options HostOptions) (result error) { if policyErr != nil { return fmt.Errorf("configure mounted filesystem: %w", policyErr) } + if backingErr != nil { + return fmt.Errorf("seal unmounted backing directory: %w", backingErr) + } return ctx.Err() } diff --git a/internal/mountfs/host_platform_posix.go b/internal/mountfs/host_platform_posix.go new file mode 100644 index 0000000..c28e612 --- /dev/null +++ b/internal/mountfs/host_platform_posix.go @@ -0,0 +1,56 @@ +//go:build (darwin && fuse && cgo) || (linux && fuse && fuse3 && cgo) + +package mountfs + +import ( + "bytes" + + "github.com/winfsp/cgofuse/fuse" + "golang.org/x/sys/unix" +) + +func setFileTimes(path string, times []fuse.Timespec) int { + values := []unix.Timespec{{Sec: times[0].Sec, Nsec: times[0].Nsec}, {Sec: times[1].Sec, Nsec: times[1].Nsec}} + return unixResult(unix.UtimesNanoAt(unix.AT_FDCWD, path, values, 0)) +} + +func setExtendedAttribute(path string, attribute string, value []byte, flags int) int { + return unixResult(unix.Setxattr(path, attribute, value, flags)) +} + +func getExtendedAttribute(path string, attribute string) (int, []byte) { + size, err := unix.Getxattr(path, attribute, nil) + if err != nil { + return unixResult(err), nil + } + value := make([]byte, size) + n, err := unix.Getxattr(path, attribute, value) + if err != nil { + return unixResult(err), nil + } + return 0, value[:n] +} + +func listExtendedAttributes(path string) (int, []string) { + size, err := unix.Listxattr(path, nil) + if err != nil { + return unixResult(err), nil + } + buffer := make([]byte, size) + n, err := unix.Listxattr(path, buffer) + if err != nil { + return unixResult(err), nil + } + parts := bytes.Split(bytes.TrimRight(buffer[:n], "\x00"), []byte{0}) + attributes := make([]string, 0, len(parts)) + for _, part := range parts { + if len(part) != 0 { + attributes = append(attributes, string(part)) + } + } + return 0, attributes +} + +func removeExtendedAttribute(path string, attribute string) int { + return unixResult(unix.Removexattr(path, attribute)) +} diff --git a/internal/mountfs/host_platform_windows.go b/internal/mountfs/host_platform_windows.go new file mode 100644 index 0000000..330a020 --- /dev/null +++ b/internal/mountfs/host_platform_windows.go @@ -0,0 +1,53 @@ +//go:build windows && winfsp + +package mountfs + +import ( + "os" + "syscall" + "time" + + "github.com/winfsp/cgofuse/fuse" + "golang.org/x/sys/windows" +) + +func populateFilesystemStat(path string, stat *fuse.Statfs_t) int { + pointer, err := windows.UTF16PtrFromString(path) + if err != nil { + return unixResult(err) + } + var available, total, free uint64 + if err := windows.GetDiskFreeSpaceEx(pointer, &available, &total, &free); err != nil { + return unixResult(err) + } + const blockSize = uint64(4096) + stat.Bsize = blockSize + stat.Frsize = blockSize + stat.Blocks = total / blockSize + stat.Bfree = free / blockSize + stat.Bavail = available / blockSize + stat.Namemax = 255 + return 0 +} + +func setFileTimes(path string, times []fuse.Timespec) int { + atime := time.Unix(times[0].Sec, times[0].Nsec) + mtime := time.Unix(times[1].Sec, times[1].Nsec) + return unixResult(os.Chtimes(path, atime, mtime)) +} + +func setExtendedAttribute(string, string, []byte, int) int { + return -int(syscall.ENOSYS) +} + +func getExtendedAttribute(string, string) (int, []byte) { + return -int(syscall.ENOSYS), nil +} + +func listExtendedAttributes(string) (int, []string) { + return -int(syscall.ENOSYS), nil +} + +func removeExtendedAttribute(string, string) int { + return -int(syscall.ENOSYS) +} diff --git a/internal/mountfs/host_safety_test.go b/internal/mountfs/host_safety_test.go index feae965..497df42 100644 --- a/internal/mountfs/host_safety_test.go +++ b/internal/mountfs/host_safety_test.go @@ -20,6 +20,20 @@ func TestPrepareMountPointRejectsOrdinaryFiles(t *testing.T) { } } +func TestPrepareMountPointCreatesAndSealsMissingBackingDirectory(t *testing.T) { + mountPoint := filepath.Join(t.TempDir(), "missing", "mount") + if err := prepareMountPoint(mountPoint); err != nil { + t.Fatal(err) + } + info, err := os.Stat(mountPoint) + if err != nil { + t.Fatal(err) + } + if !info.IsDir() || info.Mode().Perm() != 0o500 { + t.Fatalf("created mount backing mode=%#o directory=%t", info.Mode().Perm(), info.IsDir()) + } +} + func TestPrepareMountPointSealsEmptyBackingDirectory(t *testing.T) { mountPoint := filepath.Join(t.TempDir(), "mount") if err := os.MkdirAll(mountPoint, 0o700); err != nil { diff --git a/internal/mountfs/host_statfs_darwin.go b/internal/mountfs/host_statfs_darwin.go new file mode 100644 index 0000000..730d9a0 --- /dev/null +++ b/internal/mountfs/host_statfs_darwin.go @@ -0,0 +1,30 @@ +//go:build darwin && fuse && cgo + +package mountfs + +import ( + "github.com/winfsp/cgofuse/fuse" + "golang.org/x/sys/unix" +) + +func populateFilesystemStat(path string, stat *fuse.Statfs_t) int { + var source unix.Statfs_t + result := unixResult(unix.Statfs(path, &source)) + if result != 0 { + return result + } + stat.Bsize = uint64(source.Bsize) + if source.Iosize > 0 { + stat.Frsize = uint64(source.Iosize) + } else { + stat.Frsize = uint64(source.Bsize) + } + stat.Blocks = source.Blocks + stat.Bfree = source.Bfree + stat.Bavail = source.Bavail + stat.Files = source.Files + stat.Ffree = source.Ffree + stat.Favail = source.Ffree + stat.Namemax = 255 + return 0 +} diff --git a/internal/mountfs/host_statfs_linux.go b/internal/mountfs/host_statfs_linux.go new file mode 100644 index 0000000..f98dde8 --- /dev/null +++ b/internal/mountfs/host_statfs_linux.go @@ -0,0 +1,26 @@ +//go:build linux && fuse && fuse3 && cgo + +package mountfs + +import ( + "github.com/winfsp/cgofuse/fuse" + "golang.org/x/sys/unix" +) + +func populateFilesystemStat(path string, stat *fuse.Statfs_t) int { + var source unix.Statfs_t + result := unixResult(unix.Statfs(path, &source)) + if result != 0 { + return result + } + stat.Bsize = uint64(source.Bsize) + stat.Frsize = uint64(source.Bsize) + stat.Blocks = source.Blocks + stat.Bfree = source.Bfree + stat.Bavail = source.Bavail + stat.Files = source.Files + stat.Ffree = source.Ffree + stat.Favail = source.Ffree + stat.Namemax = 255 + return 0 +} diff --git a/internal/mountfs/host_stub.go b/internal/mountfs/host_stub.go index 2a366c8..7fc0b95 100644 --- a/internal/mountfs/host_stub.go +++ b/internal/mountfs/host_stub.go @@ -1,4 +1,4 @@ -//go:build !fuse || !cgo +//go:build (!darwin && !linux && !windows) || (darwin && (!fuse || !cgo)) || (linux && (!fuse || !fuse3 || !cgo)) || (windows && !winfsp) package mountfs diff --git a/internal/mountfs/mount_backing_linux.go b/internal/mountfs/mount_backing_linux.go new file mode 100644 index 0000000..dfdf145 --- /dev/null +++ b/internal/mountfs/mount_backing_linux.go @@ -0,0 +1,60 @@ +//go:build linux && fuse && fuse3 && cgo + +package mountfs + +import ( + "errors" + "os" + "sync" +) + +type linuxMountBacking struct { + mu sync.Mutex + directory *os.File + sealed bool + closed bool +} + +func prepareMountedBacking(path string) (*linuxMountBacking, error) { + directory, err := os.Open(path) + if err != nil { + return nil, err + } + if err := directory.Chmod(0o700); err != nil { + _ = directory.Close() + return nil, err + } + return &linuxMountBacking{directory: directory}, nil +} + +func (backing *linuxMountBacking) Seal() error { + backing.mu.Lock() + defer backing.mu.Unlock() + if backing.closed { + return errors.New("mount backing guard is already closed") + } + if backing.sealed { + return nil + } + if err := backing.directory.Chmod(0o500); err != nil { + return err + } + backing.sealed = true + return nil +} + +func (backing *linuxMountBacking) Close() error { + backing.mu.Lock() + defer backing.mu.Unlock() + if backing.closed { + return nil + } + var sealErr error + if !backing.sealed { + sealErr = backing.directory.Chmod(0o500) + backing.sealed = sealErr == nil + } + closeErr := backing.directory.Close() + backing.closed = true + return errors.Join(sealErr, closeErr) +} diff --git a/internal/mountfs/mount_backing_other.go b/internal/mountfs/mount_backing_other.go new file mode 100644 index 0000000..6a7df4a --- /dev/null +++ b/internal/mountfs/mount_backing_other.go @@ -0,0 +1,12 @@ +//go:build (darwin && fuse && cgo) || (windows && winfsp) + +package mountfs + +type noOpMountBacking struct{} + +func prepareMountedBacking(string) (*noOpMountBacking, error) { + return &noOpMountBacking{}, nil +} + +func (*noOpMountBacking) Seal() error { return nil } +func (*noOpMountBacking) Close() error { return nil } diff --git a/internal/mountfs/mount_linux.go b/internal/mountfs/mount_linux.go new file mode 100644 index 0000000..a4470ae --- /dev/null +++ b/internal/mountfs/mount_linux.go @@ -0,0 +1,89 @@ +//go:build linux + +package mountfs + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" +) + +type linuxMountRecord struct { + Filesystem string + Source string +} + +func recoverStaleMount(mountPoint string) error { + record, mounted := findLinuxMount(mountPoint) + if !mounted { + return nil + } + if !strings.HasPrefix(record.Filesystem, "fuse") || !strings.Contains(strings.ToLower(record.Source), "codexfold") { + return fmt.Errorf("mount point is already used by %s source %s", record.Filesystem, record.Source) + } + _, healthErr := os.ReadFile(filepath.Join(mountPoint, ".codexfold-health")) + if healthErr == nil { + return errors.New("a healthy CodexFold mount is already active") + } + if !errors.Is(healthErr, syscall.ENOTCONN) && !errors.Is(healthErr, syscall.EIO) { + return fmt.Errorf("CodexFold mount is not proven stale: %w", healthErr) + } + fusermount, err := exec.LookPath("fusermount3") + if err != nil { + return fmt.Errorf("locate fusermount3 for stale mount recovery: %w", err) + } + if output, err := exec.Command(fusermount, "-uz", mountPoint).CombinedOutput(); err != nil { + return fmt.Errorf("unmount stale CodexFold FUSE3 mount: %w: %s", err, strings.TrimSpace(string(output))) + } + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if _, exists := findLinuxMount(mountPoint); !exists { + return os.Chmod(mountPoint, 0o500) + } + time.Sleep(25 * time.Millisecond) + } + return errors.New("stale CodexFold FUSE3 mount remained after fusermount3") +} + +func linuxFuseMountVisible(mountPoint string) bool { + record, mounted := findLinuxMount(mountPoint) + return mounted && strings.HasPrefix(record.Filesystem, "fuse") +} + +func findLinuxMount(mountPoint string) (linuxMountRecord, bool) { + data, err := os.ReadFile("/proc/self/mountinfo") + if err != nil { + return linuxMountRecord{}, false + } + want := filepath.Clean(mountPoint) + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) < 7 { + continue + } + separator := -1 + for index := 6; index < len(fields); index++ { + if fields[index] == "-" { + separator = index + break + } + } + if separator < 0 || separator+2 >= len(fields) { + continue + } + mountedAt := unescapeLinuxMountField(fields[4]) + if filepath.Clean(mountedAt) == want { + return linuxMountRecord{Filesystem: fields[separator+1], Source: unescapeLinuxMountField(fields[separator+2])}, true + } + } + return linuxMountRecord{}, false +} + +func unescapeLinuxMountField(value string) string { + return strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`).Replace(value) +} diff --git a/internal/mountfs/mount_policy_linux.go b/internal/mountfs/mount_policy_linux.go new file mode 100644 index 0000000..4684716 --- /dev/null +++ b/internal/mountfs/mount_policy_linux.go @@ -0,0 +1,24 @@ +//go:build linux && fuse && fuse3 && cgo + +package mountfs + +import ( + "context" + "errors" + "time" +) + +func configureMountedFilesystem(ctx context.Context, mountPoint string) error { + ticker := time.NewTicker(25 * time.Millisecond) + defer ticker.Stop() + for { + if linuxFuseMountVisible(mountPoint) { + return nil + } + select { + case <-ctx.Done(): + return errors.New("FUSE3 mount did not become visible before cancellation") + case <-ticker.C: + } + } +} diff --git a/internal/mountfs/mount_policy_other.go b/internal/mountfs/mount_policy_windows.go similarity index 77% rename from internal/mountfs/mount_policy_other.go rename to internal/mountfs/mount_policy_windows.go index 6f68c6a..6c6d9d0 100644 --- a/internal/mountfs/mount_policy_other.go +++ b/internal/mountfs/mount_policy_windows.go @@ -1,4 +1,4 @@ -//go:build !darwin && fuse && cgo +//go:build windows && winfsp package mountfs diff --git a/internal/mountfs/mount_stale_other.go b/internal/mountfs/mount_stale_other.go new file mode 100644 index 0000000..b49d91d --- /dev/null +++ b/internal/mountfs/mount_stale_other.go @@ -0,0 +1,5 @@ +//go:build !linux + +package mountfs + +func recoverStaleMount(string) error { return nil } diff --git a/internal/mountfs/native_append.go b/internal/mountfs/native_append.go new file mode 100644 index 0000000..75c4370 --- /dev/null +++ b/internal/mountfs/native_append.go @@ -0,0 +1,598 @@ +package mountfs + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "unicode/utf8" +) + +const ( + nativeAppendJournalVersion = 1 + maxNativeAppendPendingBytes = 256 << 20 +) + +var errNativeAppendGap = errors.New("native append transaction contains an unfilled offset gap") + +// nativeAppendJournalCheckpoint is nil in production. Integration tests use it +// in a subprocess to terminate exactly after the recovery journal is durable. +var nativeAppendJournalCheckpoint func(nativeAppendJournal, []byte) + +type nativeAppendSegment struct { + offset int64 + data []byte +} + +type nativeAppendState struct { + mu sync.Mutex + path string + journalRoot string + baseSize int64 + visibleEnd int64 + segments []nativeAppendSegment +} + +type nativeAppendJournal struct { + Version int `json:"version"` + TargetPath string `json:"target_path"` + BaseSize int64 `json:"base_size"` + FinalSize int64 `json:"final_size"` + TailSHA256 string `json:"tail_sha256"` +} + +func newNativeAppendState(path string, journalRoot string) (*nativeAppendState, error) { + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, errors.New("native append target is not a regular file") + } + return &nativeAppendState{ + path: filepath.Clean(path), journalRoot: filepath.Clean(journalRoot), + baseSize: info.Size(), visibleEnd: info.Size(), + }, nil +} + +func (s *nativeAppendState) Stage(data []byte, offset int64) (int, error) { + if offset < 0 { + return 0, errors.New("negative native append offset") + } + if len(data) == 0 { + return 0, nil + } + originalBytes := len(data) + s.mu.Lock() + defer s.mu.Unlock() + + end, overflow := addInt64(offset, int64(len(data))) + if overflow { + return 0, errors.New("native append offset overflow") + } + if offset < s.baseSize { + committedEnd := min(end, s.baseSize) + committed := make([]byte, committedEnd-offset) + file, err := os.Open(s.path) + if err != nil { + return 0, err + } + _, readErr := file.ReadAt(committed, offset) + closeErr := file.Close() + if readErr != nil && !errors.Is(readErr, io.EOF) { + return 0, readErr + } + if closeErr != nil { + return 0, closeErr + } + overlap := int(committedEnd - offset) + if !bytes.Equal(committed, data[:overlap]) { + s.clearPendingLocked() + return 0, errors.New("native append conflicts with committed bytes") + } + data = data[overlap:] + offset = committedEnd + } + if len(data) == 0 { + return originalBytes, nil + } + end, overflow = addInt64(offset, int64(len(data))) + if overflow || end-s.baseSize > maxNativeAppendPendingBytes { + s.clearPendingLocked() + return 0, errors.New("native append transaction exceeds the pending byte limit") + } + if err := s.stageSegmentLocked(offset, data); err != nil { + s.clearPendingLocked() + return 0, err + } + if end > s.visibleEnd { + s.visibleEnd = end + } + return originalBytes, nil +} + +func (s *nativeAppendState) stageSegmentLocked(offset int64, data []byte) error { + segments := append(append([]nativeAppendSegment(nil), s.segments...), nativeAppendSegment{ + offset: offset, + data: append([]byte(nil), data...), + }) + sort.SliceStable(segments, func(i, j int) bool { return segments[i].offset < segments[j].offset }) + normalized := make([]nativeAppendSegment, 0, len(segments)) + for _, segment := range segments { + if len(normalized) == 0 { + normalized = append(normalized, segment) + continue + } + last := &normalized[len(normalized)-1] + lastEnd := last.offset + int64(len(last.data)) + segmentEnd := segment.offset + int64(len(segment.data)) + if segment.offset > lastEnd { + normalized = append(normalized, segment) + continue + } + overlapEnd := min(lastEnd, segmentEnd) + if overlapEnd > segment.offset { + lastStart := segment.offset - last.offset + overlap := overlapEnd - segment.offset + if !bytes.Equal(last.data[int(lastStart):int(lastStart+overlap)], segment.data[:int(overlap)]) { + return errors.New("native append segments contain conflicting overlap") + } + } + if segmentEnd > lastEnd { + last.data = append(last.data, segment.data[int(lastEnd-segment.offset):]...) + } + } + s.segments = normalized + return nil +} + +func (s *nativeAppendState) ReadAt(file *os.File, destination []byte, offset int64) (int, error) { + if offset < 0 { + return 0, errors.New("negative native read offset") + } + s.mu.Lock() + defer s.mu.Unlock() + visibleEnd := s.contiguousEndLocked() + if offset >= visibleEnd { + return 0, io.EOF + } + limit := len(destination) + if remaining := visibleEnd - offset; int64(limit) > remaining { + limit = int(remaining) + } + visible := destination[:limit] + clear(visible) + if offset < s.baseSize { + committedBytes := limit + if remaining := s.baseSize - offset; int64(committedBytes) > remaining { + committedBytes = int(remaining) + } + n, err := file.ReadAt(visible[:committedBytes], offset) + if err != nil && !errors.Is(err, io.EOF) { + return n, err + } + } + for _, segment := range s.segments { + segmentEnd := segment.offset + int64(len(segment.data)) + readEnd := offset + int64(limit) + start := max(offset, segment.offset) + end := min(readEnd, segmentEnd) + if start >= end { + continue + } + copy(visible[start-offset:end-offset], segment.data[start-segment.offset:end-segment.offset]) + } + if limit < len(destination) { + return limit, io.EOF + } + return limit, nil +} + +func (s *nativeAppendState) VisibleSize() int64 { + s.mu.Lock() + defer s.mu.Unlock() + return s.contiguousEndLocked() +} + +func (s *nativeAppendState) HasPending() bool { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.segments) != 0 +} + +func (s *nativeAppendState) Commit() error { + return s.commit(true) +} + +func (s *nativeAppendState) CommitAvailable() error { + return s.commit(false) +} + +func (s *nativeAppendState) commit(strict bool) error { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.segments) == 0 { + file, err := os.OpenFile(s.path, os.O_RDWR, 0) + if err != nil { + return err + } + syncErr := file.Sync() + closeErr := file.Close() + return errors.Join(syncErr, closeErr) + } + + tail, err := s.assembleLocked() + if err != nil { + if !strict && errors.Is(err, errNativeAppendGap) { + return nil + } + s.clearPendingLocked() + return err + } + if tail[len(tail)-1] != '\n' { + if !strict { + return nil + } + s.clearPendingLocked() + return errors.New("native append transaction ends with an incomplete JSONL record") + } + if !utf8.Valid(tail) || !completeJSONL(tail) { + s.clearPendingLocked() + return errors.New("native append transaction is not complete valid JSONL") + } + if err := commitNativeAppend(s.path, s.journalRoot, s.baseSize, tail); err != nil { + s.clearPendingLocked() + return err + } + s.baseSize += int64(len(tail)) + s.clearPendingLocked() + return nil +} + +func (s *nativeAppendState) Truncate(size int64) error { + if size < 0 { + return errors.New("negative native truncate size") + } + s.mu.Lock() + defer s.mu.Unlock() + if len(s.segments) != 0 { + return errors.New("cannot truncate a native append transaction with pending writes") + } + if err := os.Truncate(s.path, size); err != nil { + return err + } + file, err := os.OpenFile(s.path, os.O_RDWR, 0) + if err != nil { + return err + } + syncErr := file.Sync() + closeErr := file.Close() + if syncErr != nil || closeErr != nil { + return errors.Join(syncErr, closeErr) + } + s.baseSize = size + s.visibleEnd = size + return nil +} + +func (s *nativeAppendState) Refresh() error { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.segments) != 0 { + return errors.New("cannot refresh native append state with pending writes") + } + info, err := os.Stat(s.path) + if err != nil { + return err + } + s.baseSize = info.Size() + s.visibleEnd = info.Size() + return nil +} + +func (s *nativeAppendState) RefreshIfIdle() error { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.segments) != 0 { + return nil + } + info, err := os.Stat(s.path) + if err != nil { + return err + } + s.baseSize = info.Size() + s.visibleEnd = info.Size() + return nil +} + +func (s *nativeAppendState) Relocate(newPath string) error { + if !filepath.IsAbs(newPath) { + return errors.New("absolute native append relocation path is required") + } + s.mu.Lock() + defer s.mu.Unlock() + newPath = filepath.Clean(newPath) + if err := os.Rename(s.path, newPath); err != nil { + return err + } + s.path = newPath + return nil +} + +func (s *nativeAppendState) assembleLocked() ([]byte, error) { + if s.visibleEnd < s.baseSize || s.visibleEnd-s.baseSize > maxNativeAppendPendingBytes { + return nil, errors.New("native append transaction has an invalid visible range") + } + tail := make([]byte, s.visibleEnd-s.baseSize) + covered := make([]byte, len(tail)) + segments := append([]nativeAppendSegment(nil), s.segments...) + sort.SliceStable(segments, func(i, j int) bool { return segments[i].offset < segments[j].offset }) + for _, segment := range segments { + start := segment.offset - s.baseSize + if start < 0 || start+int64(len(segment.data)) > int64(len(tail)) { + return nil, errors.New("native append segment lies outside the transaction range") + } + for index, value := range segment.data { + position := int(start) + index + if covered[position] != 0 && tail[position] != value { + return nil, errors.New("native append segments contain conflicting overlap") + } + tail[position] = value + covered[position] = 1 + } + } + if bytes.IndexByte(covered, 0) >= 0 { + return nil, errNativeAppendGap + } + return tail, nil +} + +func (s *nativeAppendState) contiguousEndLocked() int64 { + end := s.baseSize + for _, segment := range s.segments { + if segment.offset > end { + break + } + segmentEnd := segment.offset + int64(len(segment.data)) + if segmentEnd > end { + end = segmentEnd + } + } + return end +} + +func (s *nativeAppendState) clearPendingLocked() { + s.segments = nil + s.visibleEnd = s.baseSize +} + +func commitNativeAppend(path string, journalRoot string, baseSize int64, tail []byte) error { + info, err := os.Stat(path) + if err != nil { + return err + } + if info.Size() != baseSize { + return fmt.Errorf("native append backing changed: size=%d expected=%d", info.Size(), baseSize) + } + digest := sha256.Sum256(tail) + record := nativeAppendJournal{ + Version: nativeAppendJournalVersion, TargetPath: filepath.Clean(path), BaseSize: baseSize, + FinalSize: baseSize + int64(len(tail)), TailSHA256: hex.EncodeToString(digest[:]), + } + journalPath, err := writeNativeAppendJournal(journalRoot, record) + if err != nil { + return err + } + if nativeAppendJournalCheckpoint != nil { + nativeAppendJournalCheckpoint(record, tail) + } + rollback := func(commitErr error) error { + truncateErr := os.Truncate(path, baseSize) + syncErr := syncNativePath(path) + if truncateErr != nil || syncErr != nil { + // Leave the journal in place so startup recovery can finish rollback. + return errors.Join(commitErr, truncateErr, syncErr) + } + return errors.Join(commitErr, removeNativeAppendJournal(journalPath)) + } + + file, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + return rollback(err) + } + n, writeErr := file.WriteAt(tail, baseSize) + if writeErr == nil && n != len(tail) { + writeErr = io.ErrShortWrite + } + syncErr := file.Sync() + closeErr := file.Close() + if writeErr != nil || syncErr != nil || closeErr != nil { + return rollback(errors.Join(writeErr, syncErr, closeErr)) + } + verified, err := hashNativeAppendTail(path, baseSize, int64(len(tail))) + if err != nil || verified != record.TailSHA256 { + return rollback(fmt.Errorf("verify native append transaction: digest=%s expected=%s err=%w", verified, record.TailSHA256, err)) + } + return removeNativeAppendJournal(journalPath) +} + +func writeNativeAppendJournal(root string, record nativeAppendJournal) (string, error) { + if root == "" || !filepath.IsAbs(root) { + return "", errors.New("absolute native append journal root is required") + } + if err := os.MkdirAll(root, 0o700); err != nil { + return "", err + } + digest := sha256.Sum256([]byte(record.TargetPath)) + finalPath := filepath.Join(root, hex.EncodeToString(digest[:])+".json") + data, err := json.Marshal(record) + if err != nil { + return "", err + } + temporary, err := os.CreateTemp(root, ".native-append-*.tmp") + if err != nil { + return "", err + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return "", err + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return "", err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return "", err + } + if err := temporary.Close(); err != nil { + return "", err + } + if err := os.Rename(temporaryPath, finalPath); err != nil { + return "", err + } + if err := syncDirectory(root); err != nil { + return "", err + } + return finalPath, nil +} + +func removeNativeAppendJournal(path string) error { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return syncDirectory(filepath.Dir(path)) +} + +func recoverNativeAppendTransactions(nativeRoot string, journalRoot string) error { + entries, err := os.ReadDir(journalRoot) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + for _, entry := range entries { + if entry.IsDir() { + return fmt.Errorf("native append journal contains an unexpected directory %q", entry.Name()) + } + path := filepath.Join(journalRoot, entry.Name()) + if strings.HasPrefix(entry.Name(), ".native-append-") && strings.HasSuffix(entry.Name(), ".tmp") { + if err := os.Remove(path); err != nil { + return err + } + continue + } + if !strings.HasSuffix(entry.Name(), ".json") { + return fmt.Errorf("native append journal contains an unexpected file %q", entry.Name()) + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + var record nativeAppendJournal + if err := json.Unmarshal(data, &record); err != nil { + return fmt.Errorf("decode native append journal %s: %w", entry.Name(), err) + } + if err := validateNativeAppendJournal(nativeRoot, record); err != nil { + return fmt.Errorf("validate native append journal %s: %w", entry.Name(), err) + } + info, err := os.Stat(record.TargetPath) + if err != nil { + return err + } + committed := false + if info.Size() == record.FinalSize { + digest, hashErr := hashNativeAppendTail(record.TargetPath, record.BaseSize, record.FinalSize-record.BaseSize) + committed = hashErr == nil && digest == record.TailSHA256 + } + if !committed { + if info.Size() < record.BaseSize { + return fmt.Errorf("native append target is shorter than its rollback size: %s", record.TargetPath) + } + if err := os.Truncate(record.TargetPath, record.BaseSize); err != nil { + return err + } + if err := syncNativePath(record.TargetPath); err != nil { + return err + } + } + if err := os.Remove(path); err != nil { + return err + } + } + return syncDirectory(journalRoot) +} + +func validateNativeAppendJournal(nativeRoot string, record nativeAppendJournal) error { + if record.Version != nativeAppendJournalVersion || record.BaseSize < 0 || record.FinalSize < record.BaseSize || + record.FinalSize-record.BaseSize > maxNativeAppendPendingBytes { + return errors.New("native append journal metadata is invalid") + } + target := filepath.Clean(record.TargetPath) + relative, err := filepath.Rel(filepath.Clean(nativeRoot), target) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return errors.New("native append journal target is outside the native root") + } + if !nativeTransactionPath("/" + filepath.ToSlash(relative)) { + return errors.New("native append journal target is not a canonical session path") + } + digest, err := hex.DecodeString(record.TailSHA256) + if err != nil || len(digest) != sha256.Size { + return errors.New("native append journal digest is invalid") + } + return nil +} + +func hashNativeAppendTail(path string, offset int64, size int64) (string, error) { + if offset < 0 || size < 0 { + return "", errors.New("invalid native append hash range") + } + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + hasher := sha256.New() + if _, err := io.CopyN(hasher, io.NewSectionReader(file, offset, size), size); err != nil { + return "", err + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func syncNativePath(path string) error { + file, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return err + } + syncErr := file.Sync() + closeErr := file.Close() + return errors.Join(syncErr, closeErr) +} + +func syncDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + syncErr := directory.Sync() + closeErr := directory.Close() + return errors.Join(syncErr, closeErr) +} + +func addInt64(left int64, right int64) (int64, bool) { + if right > 0 && left > int64(^uint64(0)>>1)-right { + return 0, true + } + return left + right, false +} diff --git a/internal/mountfs/native_append_real_integration_test.go b/internal/mountfs/native_append_real_integration_test.go new file mode 100644 index 0000000..0e8e1b1 --- /dev/null +++ b/internal/mountfs/native_append_real_integration_test.go @@ -0,0 +1,368 @@ +//go:build darwin && fuse && cgo + +package mountfs + +import ( + "bufio" + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "testing" +) + +const ( + nativeAppendCrashHelperEnv = "CODEXFOLD_NATIVE_APPEND_CRASH_HELPER" + nativeAppendCrashRootEnv = "CODEXFOLD_NATIVE_APPEND_CRASH_ROOT" + realCodexTraceBaseBytes = 51836 + realCodexTraceFinalBytes = 57255 +) + +type codexWriteReplayOperation struct { + kind string + offset int64 + size int + flags int +} + +func TestRealFuseReplaysSanitizedCodexWriteTraceAgainstAPFS(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + route := filepath.Join("sessions", "2026", "07", "16", "rollout-trace-replay.jsonl") + nativeRoot := filepath.Join(root, "native") + nativePath := filepath.Join(nativeRoot, route) + referencePath := filepath.Join(root, "apfs-reference.jsonl") + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + + base := exactJSONLRecord(t, realCodexTraceBaseBytes, 'b') + tail := exactJSONLRecord(t, realCodexTraceFinalBytes-realCodexTraceBaseBytes, 't') + want := append(append([]byte(nil), base...), tail...) + for _, target := range []string{referencePath, nativePath} { + if err := os.WriteFile(target, base, 0o600); err != nil { + t.Fatal(err) + } + } + operations := loadCodexWriteReplay(t, filepath.Join("testdata", "codex-real-resume-write.trace")) + + var recordedMu sync.Mutex + var recorded []string + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + stopMount := startRealMountWithOptions(t, HostOptions{ + MountPoint: mountPoint, + Filesystem: filesystem, + Foreground: true, + OperationRecorder: func(operation string) { + recordedMu.Lock() + defer recordedMu.Unlock() + recorded = append(recorded, operation) + }, + }) + target := filepath.Join(mountPoint, route) + waitForRealFile(t, target, base) + + replayCodexWriteOperations(t, referencePath, want, operations) + replayCodexWriteOperations(t, target, want, operations) + assertNativeBytes(t, nativePath, want) + assertNativeBytes(t, referencePath, want) + visible, err := os.ReadFile(target) + if err != nil || !bytes.Equal(visible, want) { + t.Fatalf("mounted replay bytes=%d err=%v want=%d", len(visible), err, len(want)) + } + if !completeJSONL(want) { + t.Fatal("trace replay fixture did not produce valid JSONL") + } + + recordedMu.Lock() + joined := strings.Join(recorded, "\n") + recordedMu.Unlock() + for _, operation := range operations { + if operation.kind != "write" { + continue + } + marker := fmt.Sprintf("offset=%d bytes=%d result=%d", operation.offset, operation.size, operation.size) + if !strings.Contains(joined, marker) { + t.Fatalf("real FUSE trace did not replay %q: %s", marker, joined) + } + } + for _, marker := range []string{"open kind=session flags=0x2", "fsync kind=session", "flush kind=session", "release kind=session"} { + if !strings.Contains(joined, marker) { + t.Fatalf("real FUSE replay trace missing %q: %s", marker, joined) + } + } + for _, entry := range strings.Split(joined, "\n") { + if strings.Contains(entry, "kind=session") && strings.Contains(entry, "result=-") { + t.Fatalf("real FUSE replay operation failed: %s", entry) + } + } + stopMount() + waitForRealUnmount(t, mountPoint) +} + +func TestRealFuseNativeAppendSIGKILLRecovery(t *testing.T) { + if os.Getenv(nativeAppendCrashHelperEnv) == "1" { + runNativeAppendCrashHelper(t) + return + } + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := filepath.Join("sessions", "2026", "07", "16", "rollout-sigkill-recovery.jsonl") + nativePath := filepath.Join(nativeRoot, route) + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":\"before-crash\"}\n") + crashTail := exactJSONLRecord(t, 128<<10, 'c') + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + + command := exec.Command(os.Args[0], "-test.run=^TestRealFuseNativeAppendSIGKILLRecovery$", "-test.v") + command.Env = append(os.Environ(), nativeAppendCrashHelperEnv+"=1", nativeAppendCrashRootEnv+"="+root) + helperLogPath := filepath.Join(root, "crash-helper.log") + helperLog, err := os.Create(helperLogPath) + if err != nil { + t.Fatal(err) + } + command.Stdout, command.Stderr = helperLog, helperLog + runErr := command.Run() + closeErr := helperLog.Close() + output, readErr := os.ReadFile(helperLogPath) + if closeErr != nil || readErr != nil { + t.Fatalf("read crash helper log: close=%v read=%v", closeErr, readErr) + } + if runErr == nil { + t.Fatalf("crash helper exited normally: %s", output) + } + exitError, ok := runErr.(*exec.ExitError) + if !ok { + t.Fatalf("crash helper did not return an exit status: %v: %s", runErr, output) + } + waitStatus, ok := exitError.Sys().(syscall.WaitStatus) + if !ok || !waitStatus.Signaled() || waitStatus.Signal() != syscall.SIGKILL { + t.Fatalf("crash helper was not SIGKILLed: status=%v err=%v: %s", exitError.Sys(), runErr, output) + } + + info, err := os.Stat(nativePath) + if err != nil { + t.Fatal(err) + } + if info.Size() <= int64(len(base)) || info.Size() >= int64(len(base)+len(crashTail)) { + t.Fatalf("crash did not leave a partial backing write: size=%d base=%d final=%d", info.Size(), len(base), len(base)+len(crashTail)) + } + journalRoot := filepath.Join(nativeRoot, ".codexfold-native-journal") + entries, err := os.ReadDir(journalRoot) + if err != nil || len(entries) != 1 { + t.Fatalf("durable recovery journal missing after SIGKILL: entries=%d err=%v", len(entries), err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + if err := filesystem.RecoverNativeAppendTransactions(); err != nil { + t.Fatalf("recover SIGKILL transaction: %v", err) + } + assertNativeBytes(t, nativePath, base) + assertEmptyJournal(t, journalRoot) + + stopMount := startRealMount(t, mountPoint, filesystem) + target := filepath.Join(mountPoint, route) + waitForRealFile(t, target, base) + afterRecovery := []byte("{\"record\":\"after-recovery\"}\n") + file, err := os.OpenFile(target, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + if n, writeErr := file.Write(afterRecovery); writeErr != nil || n != len(afterRecovery) { + _ = file.Close() + t.Fatalf("append after SIGKILL recovery: n=%d err=%v", n, writeErr) + } + if err := file.Close(); err != nil { + t.Fatalf("close after SIGKILL recovery: %v", err) + } + want := append(append([]byte(nil), base...), afterRecovery...) + assertNativeBytes(t, nativePath, want) + if !completeJSONL(want) { + t.Fatal("post-recovery append produced invalid JSONL") + } + stopMount() + waitForRealUnmount(t, mountPoint) +} + +func runNativeAppendCrashHelper(t *testing.T) { + root := os.Getenv(nativeAppendCrashRootEnv) + if root == "" { + t.Fatal("crash helper root is missing") + } + nativeRoot := filepath.Join(root, "native") + nativePath := filepath.Join(nativeRoot, "sessions", "2026", "07", "16", "rollout-sigkill-recovery.jsonl") + journalRoot := filepath.Join(nativeRoot, ".codexfold-native-journal") + tail := exactJSONLRecord(t, 128<<10, 'c') + nativeAppendJournalCheckpoint = func(record nativeAppendJournal, committedTail []byte) { + if !bytes.Equal(committedTail, tail) { + panic("checkpoint received an unexpected append tail") + } + file, err := os.OpenFile(record.TargetPath, os.O_WRONLY, 0) + if err != nil { + panic(err) + } + partial := committedTail[:len(committedTail)/2] + if n, err := file.WriteAt(partial, record.BaseSize); err != nil || n != len(partial) { + panic(fmt.Sprintf("partial crash write n=%d err=%v", n, err)) + } + if err := file.Sync(); err != nil { + panic(err) + } + if err := file.Close(); err != nil { + panic(err) + } + if err := syscall.Kill(os.Getpid(), syscall.SIGKILL); err != nil { + panic(err) + } + select {} + } + info, err := os.Stat(nativePath) + if err != nil { + t.Fatal(err) + } + if err := commitNativeAppend(nativePath, journalRoot, info.Size(), tail); err != nil { + t.Fatalf("crash checkpoint was not reached: %v", err) + } + t.Fatal("crash checkpoint was not reached") +} + +func loadCodexWriteReplay(t *testing.T, path string) []codexWriteReplayOperation { + t.Helper() + file, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + var operations []codexWriteReplayOperation + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + fields := strings.Fields(line) + operation := codexWriteReplayOperation{kind: fields[0]} + switch operation.kind { + case "open": + if len(fields) != 2 { + t.Fatalf("invalid open replay operation: %q", line) + } + flags, err := strconv.ParseInt(fields[1], 0, 32) + if err != nil { + t.Fatal(err) + } + operation.flags = int(flags) + case "write": + if len(fields) != 3 { + t.Fatalf("invalid write replay operation: %q", line) + } + offset, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil { + t.Fatal(err) + } + size, err := strconv.Atoi(fields[2]) + if err != nil { + t.Fatal(err) + } + operation.offset, operation.size = offset, size + case "fsync", "flush", "release": + if len(fields) != 1 { + t.Fatalf("invalid %s replay operation: %q", operation.kind, line) + } + default: + t.Fatalf("unknown replay operation: %q", line) + } + operations = append(operations, operation) + } + if err := scanner.Err(); err != nil { + t.Fatal(err) + } + return operations +} + +func replayCodexWriteOperations(t *testing.T, path string, final []byte, operations []codexWriteReplayOperation) { + t.Helper() + var file *os.File + for _, operation := range operations { + switch operation.kind { + case "open": + if file != nil { + t.Fatal("replay opened an already-open file") + } + var err error + file, err = os.OpenFile(path, operation.flags, 0) + if err != nil { + t.Fatal(err) + } + case "write": + if file == nil || operation.offset < 0 || operation.size < 0 || operation.offset+int64(operation.size) > int64(len(final)) { + t.Fatalf("invalid replay write: offset=%d size=%d final=%d", operation.offset, operation.size, len(final)) + } + chunk := final[operation.offset : operation.offset+int64(operation.size)] + if n, err := file.WriteAt(chunk, operation.offset); err != nil || n != len(chunk) { + t.Fatalf("replay write offset=%d size=%d: n=%d err=%v", operation.offset, operation.size, n, err) + } + case "fsync": + if file == nil { + t.Fatal("replay fsync without an open file") + } + if err := file.Sync(); err != nil { + t.Fatal(err) + } + case "flush": + // FUSE emits flush during close; there is no separate portable os.File call. + case "release": + if file == nil { + t.Fatal("replay release without an open file") + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + file = nil + } + } + if file != nil { + _ = file.Close() + t.Fatal("replay trace ended with an open file") + } +} + +func exactJSONLRecord(t *testing.T, size int, fill byte) []byte { + t.Helper() + prefix := []byte("{\"payload\":\"") + suffix := []byte("\"}\n") + if size < len(prefix)+len(suffix) { + t.Fatalf("JSONL record size %d is too small", size) + } + record := append([]byte(nil), prefix...) + record = append(record, bytes.Repeat([]byte{fill}, size-len(prefix)-len(suffix))...) + record = append(record, suffix...) + if len(record) != size || !completeJSONL(record) { + t.Fatalf("invalid exact JSONL record: size=%d want=%d", len(record), size) + } + return record +} diff --git a/internal/mountfs/native_append_test.go b/internal/mountfs/native_append_test.go new file mode 100644 index 0000000..4903f0f --- /dev/null +++ b/internal/mountfs/native_append_test.go @@ -0,0 +1,409 @@ +package mountfs + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "sync" + "syscall" + "testing" +) + +func TestNativeAppendGapFailsClosedAndCanRetry(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "gap") + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + + record := []byte("{\"record\":1}\n") + if n, errno := filesystem.Write(handle, record, int64(len(base)+5)); errno != 0 || n != len(record) { + t.Fatalf("stage gapped append: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("intermediate gapped fsync errno=%v", errno) + } + if errno := filesystem.Flush(handle); errno != syscall.EIO { + t.Fatalf("final gapped flush errno=%v, want EIO", errno) + } + assertNativeBytes(t, nativePath, base) + + if n, errno := filesystem.Write(handle, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("stage retry: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("valid retry fsync: %v", errno) + } + assertNativeBytes(t, nativePath, append(append([]byte(nil), base...), record...)) +} + +func TestNativeAppendIntermediateFsyncKeepsPartialRecord(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "partial-fsync") + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + record := append([]byte("{\"payload\":\""), bytes.Repeat([]byte("x"), 96*1024)...) + record = append(record, []byte("\"}\n")...) + split := 32 * 1024 + if n, errno := filesystem.Write(handle, record[:split], int64(len(base))); errno != 0 || n != split { + t.Fatalf("stage partial record: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("intermediate fsync: %v", errno) + } + assertNativeBytes(t, nativePath, base) + if n, errno := filesystem.Write(handle, record[split:], int64(len(base)+split)); errno != 0 || n != len(record)-split { + t.Fatalf("finish record: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("complete fsync: %v", errno) + } + assertNativeBytes(t, nativePath, append(append([]byte(nil), base...), record...)) +} + +func TestNativeAppendEarlyHandleReleaseDoesNotDiscardOtherWriter(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "multi-handle-release") + largeHandle := openNativeAppendTestHandle(t, filesystem, route) + smallHandle := openNativeAppendTestHandle(t, filesystem, route) + large := append([]byte("{\"large\":\""), bytes.Repeat([]byte("x"), 96*1024)...) + large = append(large, []byte("\"}\n")...) + small := []byte("{\"small\":true}\n") + split := 32 * 1024 + if n, errno := filesystem.Write(largeHandle, large[:split], int64(len(base))); errno != 0 || n != split { + t.Fatalf("stage large prefix: n=%d errno=%v", n, errno) + } + if n, errno := filesystem.Write(smallHandle, small, int64(len(base)+len(large))); errno != 0 || n != len(small) { + t.Fatalf("stage later record: n=%d errno=%v", n, errno) + } + if errno := filesystem.Release(smallHandle); errno != 0 { + t.Fatalf("release later writer: %v", errno) + } + assertNativeBytes(t, nativePath, base) + if n, errno := filesystem.Write(largeHandle, large[split:], int64(len(base)+split)); errno != 0 || n != len(large)-split { + t.Fatalf("finish large record: n=%d errno=%v", n, errno) + } + if errno := filesystem.Release(largeHandle); errno != 0 { + t.Fatalf("release final writer: %v", errno) + } + want := append(append(append([]byte(nil), base...), large...), small...) + assertNativeBytes(t, nativePath, want) +} + +func TestNativeAppendConflictingOverlapFailsImmediately(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "conflict") + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + + record := []byte("{\"record\":1}\n") + if n, errno := filesystem.Write(handle, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("stage record: n=%d errno=%v", n, errno) + } + if n, errno := filesystem.Write(handle, []byte("X"), int64(len(base)+2)); errno != syscall.EIO || n != 0 { + t.Fatalf("conflicting overlap: n=%d errno=%v, want n=0 EIO", n, errno) + } + attribute, errno := filesystem.Getattr(route) + if errno != 0 || attribute.Size != int64(len(base)) { + t.Fatalf("conflict remained visible: size=%d errno=%v", attribute.Size, errno) + } + assertNativeBytes(t, nativePath, base) + + if n, errno := filesystem.Write(handle, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("stage valid retry: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("valid retry fsync: %v", errno) + } +} + +func TestNativeAppendIdenticalRetriesStayDeduplicated(t *testing.T) { + filesystem, route, _, base := nativeAppendTestFilesystem(t, "deduplicated") + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + record := []byte("{\"record\":1}\n") + for index := 0; index < 10_000; index++ { + if n, errno := filesystem.Write(handle, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("retry %d: n=%d errno=%v", index, n, errno) + } + } + state := filesystem.handles[handle].nativeAppend + state.mu.Lock() + defer state.mu.Unlock() + if len(state.segments) != 1 || len(state.segments[0].data) != len(record) { + t.Fatalf("duplicate retries retained: segments=%d bytes=%d", len(state.segments), len(state.segments[0].data)) + } +} + +func TestNativeAppendFlushCommits(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "flush") + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + record := []byte("{\"flush\":true}\n") + if n, errno := filesystem.Write(handle, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("stage append: n=%d errno=%v", n, errno) + } + if errno := filesystem.Flush(handle); errno != 0 { + t.Fatalf("flush: %v", errno) + } + assertNativeBytes(t, nativePath, append(append([]byte(nil), base...), record...)) +} + +func TestNativeAppendReleaseCommits(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "release") + handle := openNativeAppendTestHandle(t, filesystem, route) + record := []byte("{\"release\":true}\n") + if n, errno := filesystem.Write(handle, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("stage append: n=%d errno=%v", n, errno) + } + if errno := filesystem.Release(handle); errno != 0 { + t.Fatalf("release: %v", errno) + } + assertNativeBytes(t, nativePath, append(append([]byte(nil), base...), record...)) +} + +func TestNativeAppendTruncateFailsWhilePending(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "truncate") + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + record := []byte("{\"pending\":true}\n") + if n, errno := filesystem.Write(handle, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("stage append: n=%d errno=%v", n, errno) + } + if errno := filesystem.Truncate(handle, 0); errno != syscall.EIO { + t.Fatalf("handle truncate errno=%v, want EIO", errno) + } + if errno := filesystem.TruncatePath(route, 0); errno != syscall.EIO { + t.Fatalf("path truncate errno=%v, want EIO", errno) + } + assertNativeBytes(t, nativePath, base) + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("commit after rejected truncate: %v", errno) + } +} + +func TestNativeAppendRejectsInvalidUTF8(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "utf8") + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + invalid := []byte{'{', '"', 'x', '"', ':', '"', 0xff, '"', '}', '\n'} + if n, errno := filesystem.Write(handle, invalid, int64(len(base))); errno != 0 || n != len(invalid) { + t.Fatalf("stage invalid UTF-8: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(handle); errno != syscall.EIO { + t.Fatalf("invalid UTF-8 fsync errno=%v, want EIO", errno) + } + assertNativeBytes(t, nativePath, base) +} + +func TestNativeAppendHandlesLargeOutOfOrderRecords(t *testing.T) { + for _, payloadBytes := range []int{32*1024 + 1, 64*1024 + 1, 1 << 20} { + t.Run(fmt.Sprintf("payload-%d", payloadBytes), func(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, fmt.Sprintf("large-%d", payloadBytes)) + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + record := append([]byte("{\"payload\":\""), bytes.Repeat([]byte("x"), payloadBytes)...) + record = append(record, []byte("\"}\n")...) + const chunkBytes = 31 * 1024 + for end := len(record); end > 0; { + start := max(0, end-chunkBytes) + offset := int64(len(base) + start) + if n, errno := filesystem.Write(handle, record[start:end], offset); errno != 0 || n != end-start { + t.Fatalf("stage chunk [%d:%d]: n=%d errno=%v", start, end, n, errno) + } + end = start + } + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("commit large record: %v", errno) + } + assertNativeBytes(t, nativePath, append(append([]byte(nil), base...), record...)) + }) + } +} + +func TestNativeAppendConcurrentSessionsRemainIndependent(t *testing.T) { + const sessionCount = 8 + type fixture struct { + filesystem *Filesystem + route string + path string + base []byte + handle uint64 + tail []byte + } + fixtures := make([]fixture, 0, sessionCount) + for session := 0; session < sessionCount; session++ { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, fmt.Sprintf("parallel-%d", session)) + handle := openNativeAppendTestHandle(t, filesystem, route) + t.Cleanup(func() { _ = filesystem.Release(handle) }) + var tail []byte + for record := 0; record < 200; record++ { + tail = append(tail, fmt.Appendf(nil, "{\"session\":%d,\"record\":%d}\n", session, record)...) + } + fixtures = append(fixtures, fixture{filesystem: filesystem, route: route, path: nativePath, base: base, handle: handle, tail: tail}) + } + + var group sync.WaitGroup + errors := make(chan error, sessionCount) + for _, item := range fixtures { + item := item + group.Add(1) + go func() { + defer group.Done() + if n, errno := item.filesystem.Write(item.handle, item.tail, int64(len(item.base))); errno != 0 || n != len(item.tail) { + errors <- fmt.Errorf("write %s: n=%d errno=%v", item.route, n, errno) + return + } + if errno := item.filesystem.Fsync(item.handle); errno != 0 { + errors <- fmt.Errorf("fsync %s: %v", item.route, errno) + } + }() + } + group.Wait() + close(errors) + for err := range errors { + t.Error(err) + } + for _, item := range fixtures { + assertNativeBytes(t, item.path, append(append([]byte(nil), item.base...), item.tail...)) + } +} + +func TestRecoverNativeAppendTransactionRollsBackPartialCommit(t *testing.T) { + nativeRoot, journalRoot, nativePath, base, tail, record := nativeAppendRecoveryFixture(t, "partial") + if _, err := writeNativeAppendJournal(journalRoot, record); err != nil { + t.Fatal(err) + } + file, err := os.OpenFile(nativePath, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteAt(tail[:len(tail)/2], int64(len(base))); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + if err := recoverNativeAppendTransactions(nativeRoot, journalRoot); err != nil { + t.Fatal(err) + } + assertNativeBytes(t, nativePath, base) + assertEmptyJournal(t, journalRoot) +} + +func TestRecoverNativeAppendTransactionKeepsVerifiedCommit(t *testing.T) { + nativeRoot, journalRoot, nativePath, base, tail, record := nativeAppendRecoveryFixture(t, "committed") + if _, err := writeNativeAppendJournal(journalRoot, record); err != nil { + t.Fatal(err) + } + file, err := os.OpenFile(nativePath, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteAt(tail, int64(len(base))); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + if err := recoverNativeAppendTransactions(nativeRoot, journalRoot); err != nil { + t.Fatal(err) + } + assertNativeBytes(t, nativePath, append(append([]byte(nil), base...), tail...)) + assertEmptyJournal(t, journalRoot) +} + +func TestRecoverNativeAppendTransactionRejectsOutsideTarget(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + journalRoot := filepath.Join(nativeRoot, ".codexfold-native-journal") + outside := filepath.Join(root, "outside.jsonl") + if err := os.MkdirAll(nativeRoot, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(outside, []byte("{\"outside\":true}\n"), 0o600); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256([]byte("{\"tail\":true}\n")) + record := nativeAppendJournal{ + Version: nativeAppendJournalVersion, TargetPath: outside, + BaseSize: int64(len("{\"outside\":true}\n")), FinalSize: int64(len("{\"outside\":true}\n{\"tail\":true}\n")), + TailSHA256: hex.EncodeToString(digest[:]), + } + if _, err := writeNativeAppendJournal(journalRoot, record); err != nil { + t.Fatal(err) + } + if err := recoverNativeAppendTransactions(nativeRoot, journalRoot); err == nil { + t.Fatal("outside journal target was accepted") + } + assertNativeBytes(t, outside, []byte("{\"outside\":true}\n")) +} + +func nativeAppendTestFilesystem(t *testing.T, name string) (*Filesystem, string, string, []byte) { + t.Helper() + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := "/sessions/2026/07/16/rollout-" + name + ".jsonl" + nativePath := nativePathFromRoot(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + return filesystem, route, nativePath, base +} + +func openNativeAppendTestHandle(t *testing.T, filesystem *Filesystem, route string) uint64 { + t.Helper() + handle, errno := filesystem.Open(route, os.O_WRONLY|os.O_APPEND) + if errno != 0 { + t.Fatalf("open native append handle: %v", errno) + } + return handle +} + +func nativeAppendRecoveryFixture(t *testing.T, name string) (string, string, string, []byte, []byte, nativeAppendJournal) { + t.Helper() + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + journalRoot := filepath.Join(nativeRoot, ".codexfold-native-journal") + nativePath := filepath.Join(nativeRoot, "sessions", "2026", "07", "16", "rollout-"+name+".jsonl") + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + tail := []byte("{\"record\":1}\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(tail) + record := nativeAppendJournal{ + Version: nativeAppendJournalVersion, TargetPath: nativePath, + BaseSize: int64(len(base)), FinalSize: int64(len(base) + len(tail)), + TailSHA256: hex.EncodeToString(digest[:]), + } + return nativeRoot, journalRoot, nativePath, base, tail, record +} + +func assertNativeBytes(t *testing.T, path string, want []byte) { + t.Helper() + got, err := os.ReadFile(path) + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("native bytes at %s = %q err=%v, want %q", path, got, err, want) + } +} + +func assertEmptyJournal(t *testing.T, root string) { + t.Helper() + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("journal entries remained: %v", entries) + } +} diff --git a/internal/mountfs/native_fskit_metadata_darwin_test.go b/internal/mountfs/native_fskit_metadata_darwin_test.go new file mode 100644 index 0000000..f8cac79 --- /dev/null +++ b/internal/mountfs/native_fskit_metadata_darwin_test.go @@ -0,0 +1,139 @@ +//go:build darwin + +package mountfs + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "slices" + "syscall" + "testing" + "time" + + "github.com/jstar0/codexfold/internal/fskitproto" +) + +func TestNativeFSKitServerPersistsMetadataAndExtendedAttributes(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + directory := filepath.Join(nativeRoot, "sessions", "2026", "07", "17") + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(nativeRoot, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(directory, "session.jsonl") + if err := os.WriteFile(target, []byte("{}\n"), 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + client, stop := startNativeFSKitTestServer(t, filesystem, root) + defer stop() + defer client.Close() + + path := "/sessions/2026/07/17/session.jsonl" + atime := time.Unix(1_720_000_000, 123_000_000) + mtime := time.Unix(1_720_000_100, 456_000_000) + setattr := fskitproto.NewEncoder(160) + setattr.String(path) + setattr.Uint32(fskitproto.SetAttrMode | fskitproto.SetAttrUID | fskitproto.SetAttrGID | fskitproto.SetAttrAccessTime | fskitproto.SetAttrModifyTime) + setattr.Uint32(0o640) + setattr.Uint32(uint32(os.Getuid())) + setattr.Uint32(uint32(os.Getgid())) + setattr.Time(atime) + setattr.Time(mtime) + if _, err := client.Call(fskitproto.OpSetattr, setattr.Data()); err != nil { + t.Fatalf("setattr: %v", err) + } + + getattr := fskitproto.NewEncoder(128) + getattr.String(path) + response, err := client.Call(fskitproto.OpGetattr, getattr.Data()) + if err != nil { + t.Fatal(err) + } + decoder := fskitproto.NewDecoder(response) + entry, err := decoder.Entry() + if err != nil || decoder.Done() != nil { + t.Fatalf("decode getattr: %v", err) + } + if entry.Mode != 0o640 || entry.UID != uint32(os.Getuid()) || entry.GID != uint32(os.Getgid()) { + t.Fatalf("metadata = mode %#o uid %d gid %d", entry.Mode, entry.UID, entry.GID) + } + if !entry.AccessTime.Equal(atime) || !entry.ModTime.Equal(mtime) { + t.Fatalf("times = atime %s mtime %s, want %s and %s", entry.AccessTime, entry.ModTime, atime, mtime) + } + + attribute := "vip.jstar.codexfold.test" + value := []byte("first") + setXattr := fskitproto.NewEncoder(160) + setXattr.String(path) + setXattr.String(attribute) + setXattr.Uint32(uint32(fskitproto.XattrAlwaysSet)) + setXattr.Bytes(value) + if _, err := client.Call(fskitproto.OpSetXattr, setXattr.Data()); err != nil { + t.Fatalf("set xattr: %v", err) + } + + getXattr := fskitproto.NewEncoder(160) + getXattr.String(path) + getXattr.String(attribute) + response, err = client.Call(fskitproto.OpGetXattr, getXattr.Data()) + if err != nil { + t.Fatalf("get xattr: %v", err) + } + decoder = fskitproto.NewDecoder(response) + got, err := decoder.Bytes(1024) + if err != nil || decoder.Done() != nil || !bytes.Equal(got, value) { + t.Fatalf("xattr value = %q err=%v", got, err) + } + + createAgain := fskitproto.NewEncoder(160) + createAgain.String(path) + createAgain.String(attribute) + createAgain.Uint32(uint32(fskitproto.XattrMustCreate)) + createAgain.Bytes([]byte("duplicate")) + if _, err := client.Call(fskitproto.OpSetXattr, createAgain.Data()); fskitproto.ErrorNumber(err) != syscall.EEXIST { + t.Fatalf("must-create error = %v, want EEXIST", err) + } + + list := fskitproto.NewEncoder(128) + list.String(path) + response, err = client.Call(fskitproto.OpListXattrs, list.Data()) + if err != nil { + t.Fatalf("list xattrs: %v", err) + } + decoder = fskitproto.NewDecoder(response) + count, err := decoder.Uint32() + if err != nil { + t.Fatal(err) + } + names := make([]string, 0, count) + for range count { + name, decodeErr := decoder.String(4096) + if decodeErr != nil { + t.Fatal(decodeErr) + } + names = append(names, name) + } + if err := decoder.Done(); err != nil || !slices.Contains(names, attribute) { + t.Fatalf("xattr names = %v err=%v", names, err) + } + + remove := fskitproto.NewEncoder(160) + remove.String(path) + remove.String(attribute) + remove.Uint32(uint32(fskitproto.XattrDelete)) + remove.Bytes(nil) + if _, err := client.Call(fskitproto.OpSetXattr, remove.Data()); err != nil { + t.Fatalf("remove xattr: %v", err) + } + if _, err := client.Call(fskitproto.OpGetXattr, getXattr.Data()); !errors.Is(err, fskitproto.StatusError{Operation: fskitproto.OpGetXattr, Errno: syscall.ENOATTR}) && fskitproto.ErrorNumber(err) != syscall.ENOATTR { + t.Fatalf("removed xattr error = %v, want ENOATTR", err) + } +} diff --git a/internal/mountfs/native_fskit_mount_darwin_test.go b/internal/mountfs/native_fskit_mount_darwin_test.go new file mode 100644 index 0000000..6626335 --- /dev/null +++ b/internal/mountfs/native_fskit_mount_darwin_test.go @@ -0,0 +1,796 @@ +//go:build darwin + +package mountfs + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "sort" + "syscall" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +const ( + nativeFSKitMountEnv = "CODEXFOLD_NATIVE_FSKIT_MOUNT" + nativeFSKitNativeRootEnv = "CODEXFOLD_NATIVE_FSKIT_NATIVE_ROOT" +) + +func TestNativeFSKitMountedMetadataAndXattrs(t *testing.T) { + root := nativeFSKitMountedTestRoot(t) + target := filepath.Join(root, "metadata.bin") + writeMountedTestFile(t, target, []byte("{}\n")) + + if err := os.Chmod(target, 0o640); err != nil { + t.Fatalf("chmod: %v", err) + } + if err := os.Chown(target, os.Getuid(), os.Getgid()); err != nil { + t.Fatalf("chown: %v", err) + } + atime := time.Unix(1_720_000_000, 123_000_000) + mtime := time.Unix(1_720_000_100, 456_000_000) + if err := os.Chtimes(target, atime, mtime); err != nil { + t.Fatalf("chtimes: %v", err) + } + info, err := os.Stat(target) + if err != nil { + t.Fatal(err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + t.Fatalf("stat payload = %T", info.Sys()) + } + if info.Mode().Perm() != 0o640 || stat.Uid != uint32(os.Getuid()) || stat.Gid != uint32(os.Getgid()) { + t.Fatalf("metadata = mode %#o uid %d gid %d", info.Mode().Perm(), stat.Uid, stat.Gid) + } + if !info.ModTime().Equal(mtime) { + t.Fatalf("mtime = %s, want %s", info.ModTime(), mtime) + } + gotAtime := time.Unix(stat.Atimespec.Sec, stat.Atimespec.Nsec) + if !gotAtime.Equal(atime) { + t.Fatalf("atime = %s, want %s", gotAtime, atime) + } + + attribute := "vip.jstar.codexfold.integration" + first := []byte("first") + if err := unix.Setxattr(target, attribute, first, unix.XATTR_CREATE); err != nil { + t.Fatalf("create xattr: %v", err) + } + if err := unix.Setxattr(target, attribute, []byte("duplicate"), unix.XATTR_CREATE); !errors.Is(err, syscall.EEXIST) { + t.Fatalf("duplicate create xattr error = %v, want EEXIST", err) + } + second := []byte("second") + if err := unix.Setxattr(target, attribute, second, unix.XATTR_REPLACE); err != nil { + t.Fatalf("replace xattr: %v", err) + } + if got := mountedTestXattr(t, target, attribute); !bytes.Equal(got, second) { + t.Fatalf("xattr value = %q, want %q", got, second) + } + if names := mountedTestXattrNames(t, target); !slices.Contains(names, attribute) { + t.Fatalf("xattr names = %v, missing %s", names, attribute) + } + finderInfo := make([]byte, 32) + copy(finderInfo, []byte("CodexFold-FSKit")) + if err := unix.Setxattr(target, "com.apple.FinderInfo", finderInfo, 0); err != nil { + t.Fatalf("set FinderInfo xattr: %v", err) + } + if got := mountedTestXattr(t, target, "com.apple.FinderInfo"); !bytes.Equal(got, finderInfo) { + t.Fatalf("FinderInfo xattr = %x, want %x", got, finderInfo) + } + if _, err := os.Stat(filepath.Join(filepath.Dir(target), "._"+filepath.Base(target))); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unexpected AppleDouble sidecar: %v", err) + } + if err := unix.Removexattr(target, attribute); err != nil { + t.Fatalf("remove xattr: %v", err) + } + if _, err := readMountedTestXattr(target, attribute); !errors.Is(err, syscall.ENOATTR) { + t.Fatalf("removed xattr error = %v, want ENOATTR", err) + } + if err := unix.Setxattr(target, attribute, []byte("missing"), unix.XATTR_REPLACE); !errors.Is(err, syscall.ENOATTR) { + t.Fatalf("replace missing xattr error = %v, want ENOATTR", err) + } +} + +func TestNativeFSKitMountedWritesAndFullSync(t *testing.T) { + root := nativeFSKitMountedTestRoot(t) + target := filepath.Join(root, "writes.bin") + writeMountedTestFile(t, target, []byte("0123456789\n")) + + file, err := os.OpenFile(target, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteAt([]byte("AB"), 2); err != nil { + file.Close() + t.Fatalf("random overwrite: %v", err) + } + if err := file.Sync(); err != nil { + file.Close() + t.Fatalf("fsync: %v", err) + } + if _, err := unix.FcntlInt(file.Fd(), unix.F_FULLFSYNC, 0); err != nil { + file.Close() + t.Fatalf("F_FULLFSYNC: %v", err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + assertMountedTestContent(t, target, []byte("01AB456789\n")) + + if err := os.Truncate(target, 6); err != nil { + t.Fatalf("truncate: %v", err) + } + appendFile, err := os.OpenFile(target, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + if _, err := appendFile.Write([]byte("TAIL")); err != nil { + appendFile.Close() + t.Fatalf("append: %v", err) + } + if err := appendFile.Sync(); err != nil { + appendFile.Close() + t.Fatalf("append fsync: %v", err) + } + if err := appendFile.Close(); err != nil { + t.Fatal(err) + } + assertMountedTestContent(t, target, []byte("01AB45TAIL")) + + oldEOF := filepath.Join(root, "old-eof.jsonl") + base := []byte("{\"record\":0}\n") + first := []byte("{\"record\":1}\n") + second := []byte("{\"record\":2}\n") + writeMountedTestFile(t, oldEOF, base) + oldEOFFile, err := os.OpenFile(oldEOF, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := oldEOFFile.WriteAt(first, int64(len(base))); err != nil { + oldEOFFile.Close() + t.Fatalf("first old-EOF write: %v", err) + } + if _, err := oldEOFFile.WriteAt(second, int64(len(base))); err != nil { + oldEOFFile.Close() + t.Fatalf("second old-EOF write: %v", err) + } + if err := oldEOFFile.Close(); err != nil { + t.Fatal(err) + } + want := append(append(append([]byte(nil), base...), first...), second...) + assertMountedTestContent(t, oldEOF, want) +} + +func TestNativeFSKitMountedNamespaceOperations(t *testing.T) { + root := nativeFSKitMountedTestRoot(t) + source := filepath.Join(root, "source.bin") + destination := filepath.Join(root, "destination.bin") + writeMountedTestFile(t, source, []byte("source\n")) + writeMountedTestFile(t, destination, []byte("destination\n")) + if err := os.Rename(source, destination); err != nil { + t.Fatalf("overwrite rename: %v", err) + } + if _, err := os.Stat(source); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("renamed source still exists: %v", err) + } + assertMountedTestContent(t, destination, []byte("source\n")) + + nested := filepath.Join(root, "nested", "child") + if err := os.MkdirAll(nested, 0o750); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.Remove(nested); err != nil { + t.Fatalf("rmdir child: %v", err) + } + if err := os.Remove(filepath.Dir(nested)); err != nil { + t.Fatalf("rmdir parent: %v", err) + } + + if err := os.Symlink(destination, filepath.Join(root, "symbolic")); !isNotSupported(err) { + t.Fatalf("symlink error = %v, want ENOTSUP", err) + } + if err := os.Link(destination, filepath.Join(root, "hard")); !isNotSupported(err) { + t.Fatalf("hardlink error = %v, want ENOTSUP", err) + } +} + +func TestNativeFSKitMountedArchiveRoundTripAndMmap(t *testing.T) { + mountPoint := nativeFSKitMountPoint(t) + relativeDirectory := filepath.Join("2099", "12", "31") + activeDirectory := filepath.Join(mountPoint, "sessions", relativeDirectory) + archiveDirectory := filepath.Join(mountPoint, "archived_sessions", relativeDirectory) + if err := os.MkdirAll(activeDirectory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(archiveDirectory, 0o700); err != nil { + t.Fatal(err) + } + name := fmt.Sprintf("archive-roundtrip-%d.jsonl", time.Now().UnixNano()) + active := filepath.Join(activeDirectory, name) + archived := filepath.Join(archiveDirectory, name) + t.Cleanup(func() { + _ = os.Remove(active) + _ = os.Remove(archived) + }) + content := []byte("{\"archive\":true}\n") + writeMountedTestFile(t, active, content) + + file, err := os.Open(active) + if err != nil { + t.Fatal(err) + } + mapped, err := unix.Mmap(int(file.Fd()), 0, len(content), unix.PROT_READ, unix.MAP_PRIVATE) + if err != nil { + file.Close() + t.Fatalf("mmap: %v", err) + } + if !bytes.Equal(mapped, content) { + unix.Munmap(mapped) + file.Close() + t.Fatalf("mmap bytes = %q, want %q", mapped, content) + } + if err := unix.Munmap(mapped); err != nil { + file.Close() + t.Fatalf("munmap: %v", err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + + if err := os.Rename(active, archived); err != nil { + t.Fatalf("archive rename: %v", err) + } + assertMountedTestContent(t, archived, content) + if err := os.Rename(archived, active); err != nil { + t.Fatalf("unarchive rename: %v", err) + } + assertMountedTestContent(t, active, content) +} + +func TestNativeFSKitMountedOpenUnlink(t *testing.T) { + root := nativeFSKitMountedTestRoot(t) + target := filepath.Join(root, "open-unlink.bin") + writeMountedTestFile(t, target, []byte("before")) + + file, err := os.OpenFile(target, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(target); err != nil { + file.Close() + t.Fatalf("unlink open file: %v", err) + } + if _, err := file.WriteAt([]byte("after"), 0); err != nil { + file.Close() + t.Fatalf("write unlinked file: %v", err) + } + buffer := make([]byte, 6) + if _, err := file.ReadAt(buffer, 0); err != nil { + file.Close() + t.Fatalf("read unlinked file: %v", err) + } + if !bytes.Equal(buffer, []byte("aftere")) { + file.Close() + t.Fatalf("unlinked content = %q", buffer) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unlinked path exists after close: %v", err) + } +} + +func TestNativeFSKitMountedExternalNamespaceRefresh(t *testing.T) { + mountPoint := nativeFSKitMountPoint(t) + nativeRoot := os.Getenv(nativeFSKitNativeRootEnv) + if nativeRoot == "" { + t.Skipf("set %s to run external namespace refresh", nativeFSKitNativeRootEnv) + } + if !filepath.IsAbs(nativeRoot) { + t.Fatalf("%s must be absolute", nativeFSKitNativeRootEnv) + } + + relative := filepath.Join("sessions", "2099", "12", "31", fmt.Sprintf("external-%d.bin", time.Now().UnixNano())) + nativePath := filepath.Join(nativeRoot, relative) + mountedPath := filepath.Join(mountPoint, relative) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Remove(nativePath) }) + if err := os.WriteFile(nativePath, []byte("external-one\n"), 0o600); err != nil { + t.Fatal(err) + } + waitForMountedContent(t, mountedPath, []byte("external-one\n"), 3*time.Second) + + if err := os.Remove(nativePath); err != nil { + t.Fatal(err) + } + waitForMountedAbsence(t, mountedPath, 3*time.Second) + if err := os.WriteFile(nativePath, []byte("external-two\n"), 0o600); err != nil { + t.Fatal(err) + } + waitForMountedContent(t, mountedPath, []byte("external-two\n"), 3*time.Second) +} + +func TestNativeFSKitMountedPerformance(t *testing.T) { + mountPoint := nativeFSKitMountPoint(t) + nativeRoot := os.Getenv(nativeFSKitNativeRootEnv) + if nativeRoot == "" { + t.Skipf("set %s to run native FSKit performance", nativeFSKitNativeRootEnv) + } + root := nativeFSKitMountedTestRoot(t) + relativeRoot, err := filepath.Rel(mountPoint, root) + if err != nil { + t.Fatal(err) + } + mountedPath := filepath.Join(root, "performance.bin") + nativePath := filepath.Join(nativeRoot, relativeRoot, "performance.bin") + const sourceBytes = int64(256 << 20) + if err := writePerformanceFixture(nativePath, sourceBytes); err != nil { + t.Fatal(err) + } + waitForMountedSize(t, mountedPath, sourceBytes, 5*time.Second) + + nativeCold, nativeBypass, err := sequentialReadMetric(nativePath, true) + if err != nil { + t.Fatal(err) + } + mountedCold, mountedBypass, err := sequentialReadMetric(mountedPath, true) + if err != nil { + t.Fatal(err) + } + nativeWarm, err := medianSequentialThroughput(nativePath, 3) + if err != nil { + t.Fatal(err) + } + mountedWarm, err := medianSequentialThroughput(mountedPath, 3) + if err != nil { + t.Fatal(err) + } + nativeHash, err := streamingSHA256(nativePath) + if err != nil { + t.Fatal(err) + } + mountedHash, err := streamingSHA256(mountedPath) + if err != nil { + t.Fatal(err) + } + if nativeHash != mountedHash { + t.Fatal("mounted performance file differs from the native source") + } + const minimumThroughput = float64(500 << 20) + if mountedCold < minimumThroughput || mountedWarm < minimumThroughput { + t.Fatalf("mounted throughput below 500 MiB/s: cold=%.2f MiB/s warm=%.2f MiB/s", mountedCold/(1<<20), mountedWarm/(1<<20)) + } + if mountedCold/nativeCold < 0.10 || mountedWarm/nativeWarm < 0.05 { + t.Fatalf("mounted/native throughput ratio too low: cold=%.3f warm=%.3f", mountedCold/nativeCold, mountedWarm/nativeWarm) + } + + latencyPath := filepath.Join(root, "append-fsync.jsonl") + writeMountedTestFile(t, latencyPath, []byte("{}\n")) + file, err := os.OpenFile(latencyPath, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + latencies := make([]time.Duration, 0, 100) + for index := 0; index < 100; index++ { + started := time.Now() + if _, err := fmt.Fprintf(file, "{\"append_fsync\":%d}\n", index); err != nil { + file.Close() + t.Fatal(err) + } + if err := file.Sync(); err != nil { + file.Close() + t.Fatal(err) + } + latencies = append(latencies, time.Since(started)) + } + if _, err := unix.FcntlInt(file.Fd(), unix.F_FULLFSYNC, 0); err != nil { + file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + p50 := durationPercentile(latencies, 0.50) + p95 := durationPercentile(latencies, 0.95) + p99 := durationPercentile(latencies, 0.99) + if p95 > 100*time.Millisecond { + t.Fatalf("append plus fsync p95 exceeded 100 ms: %s", p95) + } + t.Logf( + "native-fskit performance bytes=%d cold_native=%.2fMiB/s cold_mounted=%.2fMiB/s cold_ratio=%.3f warm_native=%.2fMiB/s warm_mounted=%.2fMiB/s warm_ratio=%.3f native_nocache=%t mounted_nocache=%t append_fsync_p50=%s append_fsync_p95=%s append_fsync_p99=%s", + sourceBytes, + nativeCold/(1<<20), mountedCold/(1<<20), mountedCold/nativeCold, + nativeWarm/(1<<20), mountedWarm/(1<<20), mountedWarm/nativeWarm, + nativeBypass, mountedBypass, p50, p95, p99, + ) +} + +func TestNativeFSKitMountedReadAheadCoherency(t *testing.T) { + mountPoint := nativeFSKitMountPoint(t) + nativeRoot := os.Getenv(nativeFSKitNativeRootEnv) + if nativeRoot == "" { + t.Skipf("set %s to run native FSKit read-ahead coherency", nativeFSKitNativeRootEnv) + } + root := nativeFSKitMountedTestRoot(t) + relativeRoot, err := filepath.Rel(mountPoint, root) + if err != nil { + t.Fatal(err) + } + mountedPath := filepath.Join(root, "read-ahead.bin") + nativePath := filepath.Join(nativeRoot, relativeRoot, "read-ahead.bin") + const blockSize = 1 << 20 + content := make([]byte, 2*blockSize+257) + for index := range content { + content[index] = byte((index*31 + index/251) % 251) + } + writeMountedTestFile(t, mountedPath, content) + + reader, err := os.Open(mountedPath) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + for _, testCase := range []struct { + offset int64 + length int + }{ + {offset: 0, length: 8192}, + {offset: blockSize - 4096, length: 8192}, + {offset: blockSize + 2048, length: 16384}, + {offset: 2*blockSize - 7777, length: 12000}, + {offset: 333333, length: 7777}, + } { + assertOpenFileRange(t, reader, content, testCase.offset, testCase.length) + } + + eofOffset := int64(len(content) - 100) + eofBuffer := make([]byte, 4096) + n, err := reader.ReadAt(eofBuffer, eofOffset) + if !errors.Is(err, io.EOF) || n != 100 || !bytes.Equal(eofBuffer[:n], content[eofOffset:]) { + t.Fatalf("EOF read n=%d err=%v", n, err) + } + + writeOffset := int64(128 << 10) + replacement := bytes.Repeat([]byte{0xa5}, 8192) + writer, err := os.OpenFile(mountedPath, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := writer.WriteAt(replacement, writeOffset); err != nil { + writer.Close() + t.Fatalf("mounted overwrite: %v", err) + } + if err := writer.Sync(); err != nil { + writer.Close() + t.Fatalf("mounted overwrite sync: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + copy(content[writeOffset:], replacement) + assertOpenFileRange(t, reader, content, writeOffset, len(replacement)) + + truncateSize := int64(blockSize + 123) + if err := os.Truncate(mountedPath, truncateSize); err != nil { + t.Fatalf("mounted truncate: %v", err) + } + truncated := content[:truncateSize] + truncateBuffer := make([]byte, 4096) + n, err = reader.ReadAt(truncateBuffer, truncateSize-100) + if !errors.Is(err, io.EOF) || n != 100 || !bytes.Equal(truncateBuffer[:n], truncated[truncateSize-100:]) { + t.Fatalf("post-truncate read n=%d err=%v", n, err) + } + + externalOffset := int64(64 << 10) + externalReplacement := bytes.Repeat([]byte{0x3c}, 4096) + native, err := os.OpenFile(nativePath, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := native.WriteAt(externalReplacement, externalOffset); err != nil { + native.Close() + t.Fatalf("native overwrite: %v", err) + } + if err := native.Sync(); err != nil { + native.Close() + t.Fatalf("native overwrite sync: %v", err) + } + if err := native.Close(); err != nil { + t.Fatal(err) + } + copy(truncated[externalOffset:], externalReplacement) + waitForOpenFileRange(t, reader, truncated, externalOffset, len(externalReplacement), 3*time.Second) + + renamedPath := filepath.Join(root, "read-ahead-renamed.bin") + if err := os.Rename(mountedPath, renamedPath); err != nil { + t.Fatalf("rename cached file: %v", err) + } + assertOpenFileRange(t, reader, truncated, 0, 8192) + assertMountedTestContent(t, renamedPath, truncated) +} + +func writePerformanceFixture(path string, size int64) error { + file, err := os.Create(path) + if err != nil { + return err + } + buffer := bytes.Repeat([]byte("{\"codexfold_performance\":true}\n"), 1<<15) + var written int64 + for written < size { + chunk := buffer + if remaining := size - written; int64(len(chunk)) > remaining { + chunk = chunk[:remaining] + } + n, writeErr := file.Write(chunk) + written += int64(n) + if writeErr != nil { + _ = file.Close() + return writeErr + } + } + return errors.Join(file.Sync(), file.Close()) +} + +func sequentialReadMetric(path string, bypassCache bool) (float64, bool, error) { + file, err := os.Open(path) + if err != nil { + return 0, false, err + } + defer file.Close() + bypassApplied := false + if bypassCache { + if _, err := unix.FcntlInt(file.Fd(), unix.F_NOCACHE, 1); err == nil { + bypassApplied = true + } + } + started := time.Now() + read, err := io.CopyBuffer(io.Discard, file, make([]byte, 4<<20)) + if err != nil { + return 0, bypassApplied, err + } + duration := time.Since(started) + if duration <= 0 { + return 0, bypassApplied, errors.New("sequential read duration is unavailable") + } + return float64(read) / duration.Seconds(), bypassApplied, nil +} + +func medianSequentialThroughput(path string, runs int) (float64, error) { + values := make([]float64, 0, runs) + for index := 0; index < runs; index++ { + value, _, err := sequentialReadMetric(path, false) + if err != nil { + return 0, err + } + values = append(values, value) + } + sort.Float64s(values) + return values[len(values)/2], nil +} + +func streamingSHA256(path string) ([sha256.Size]byte, error) { + file, err := os.Open(path) + if err != nil { + return [sha256.Size]byte{}, err + } + defer file.Close() + digest := sha256.New() + if _, err := io.CopyBuffer(digest, file, make([]byte, 4<<20)); err != nil { + return [sha256.Size]byte{}, err + } + var result [sha256.Size]byte + copy(result[:], digest.Sum(nil)) + return result, nil +} + +func durationPercentile(values []time.Duration, percentile float64) time.Duration { + ordered := append([]time.Duration(nil), values...) + slices.Sort(ordered) + index := int(float64(len(ordered)-1) * percentile) + return ordered[index] +} + +func waitForMountedSize(t *testing.T, path string, want int64, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var size int64 + var lastErr error + for time.Now().Before(deadline) { + info, err := os.Stat(path) + if err == nil { + size = info.Size() + if size == want { + return + } + } + lastErr = err + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("mounted size %s = %d err=%v, want %d", path, size, lastErr, want) +} + +func nativeFSKitMountedTestRoot(t *testing.T) string { + t.Helper() + mountPoint := nativeFSKitMountPoint(t) + base := filepath.Join(mountPoint, "sessions", "2099", "12", "31") + if err := os.MkdirAll(base, 0o700); err != nil { + t.Fatalf("create mounted test base: %v", err) + } + root, err := os.MkdirTemp(base, "native-fskit-integration-") + if err != nil { + t.Fatalf("create mounted test root: %v", err) + } + relativeRoot, err := filepath.Rel(mountPoint, root) + if err != nil { + t.Fatalf("resolve mounted test root: %v", err) + } + nativeRoot := os.Getenv(nativeFSKitNativeRootEnv) + nativePath := "" + if nativeRoot != "" { + if !filepath.IsAbs(nativeRoot) { + t.Fatalf("%s must be absolute", nativeFSKitNativeRootEnv) + } + nativePath = filepath.Join(nativeRoot, relativeRoot) + } + t.Cleanup(func() { + var cleanupErr error + if err := os.RemoveAll(root); err != nil && !errors.Is(err, os.ErrNotExist) { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("mounted path: %w", err)) + } + if nativePath != "" { + if err := os.RemoveAll(nativePath); err != nil && !errors.Is(err, os.ErrNotExist) { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("native path: %w", err)) + } + } + if cleanupErr != nil { + t.Errorf("cleanup mounted test root: %v", cleanupErr) + } + }) + return root +} + +func assertOpenFileRange(t *testing.T, file *os.File, want []byte, offset int64, length int) { + t.Helper() + buffer := make([]byte, length) + n, err := file.ReadAt(buffer, offset) + if err != nil { + t.Fatalf("read offset=%d length=%d: %v", offset, length, err) + } + if n != length || !bytes.Equal(buffer, want[offset:offset+int64(length)]) { + t.Fatalf("range offset=%d length=%d differs", offset, length) + } +} + +func waitForOpenFileRange(t *testing.T, file *os.File, want []byte, offset int64, length int, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var last []byte + var lastErr error + for time.Now().Before(deadline) { + last = make([]byte, length) + var n int + n, lastErr = file.ReadAt(last, offset) + if lastErr == nil && n == length && bytes.Equal(last, want[offset:offset+int64(length)]) { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("open file range offset=%d length=%d remained stale: err=%v bytes=%x", offset, length, lastErr, last) +} + +func nativeFSKitMountPoint(t *testing.T) string { + t.Helper() + mountPoint := os.Getenv(nativeFSKitMountEnv) + if mountPoint == "" { + t.Skipf("set %s to run native FSKit mount tests", nativeFSKitMountEnv) + } + if !filepath.IsAbs(mountPoint) { + t.Fatalf("%s must be absolute", nativeFSKitMountEnv) + } + return mountPoint +} + +func writeMountedTestFile(t *testing.T, path string, content []byte) { + t.Helper() + if err := os.WriteFile(path, content, 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func assertMountedTestContent(t *testing.T, path string, want []byte) { + t.Helper() + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + if !bytes.Equal(got, want) { + t.Fatalf("content %s = %q, want %q", path, got, want) + } +} + +func mountedTestXattr(t *testing.T, path string, attribute string) []byte { + t.Helper() + value, err := readMountedTestXattr(path, attribute) + if err != nil { + t.Fatalf("get xattr: %v", err) + } + return value +} + +func readMountedTestXattr(path string, attribute string) ([]byte, error) { + size, err := unix.Getxattr(path, attribute, nil) + if err != nil { + return nil, err + } + value := make([]byte, size) + n, err := unix.Getxattr(path, attribute, value) + return value[:n], err +} + +func mountedTestXattrNames(t *testing.T, path string) []string { + t.Helper() + size, err := unix.Listxattr(path, nil) + if err != nil { + t.Fatalf("size xattrs: %v", err) + } + buffer := make([]byte, size) + n, err := unix.Listxattr(path, buffer) + if err != nil { + t.Fatalf("list xattrs: %v", err) + } + buffer = buffer[:n] + var names []string + for len(buffer) > 0 { + index := bytes.IndexByte(buffer, 0) + if index < 0 { + t.Fatalf("malformed xattr name list %q", buffer) + } + names = append(names, string(buffer[:index])) + buffer = buffer[index+1:] + } + return names +} + +func isNotSupported(err error) bool { + return errors.Is(err, syscall.ENOTSUP) || errors.Is(err, syscall.EOPNOTSUPP) +} + +func waitForMountedContent(t *testing.T, path string, want []byte, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var last []byte + var lastErr error + for time.Now().Before(deadline) { + last, lastErr = os.ReadFile(path) + if lastErr == nil && bytes.Equal(last, want) { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("mounted content %s = %q err=%v, want %q", path, last, lastErr, want) +} + +func waitForMountedAbsence(t *testing.T, path string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + _, lastErr = os.Stat(path) + if errors.Is(lastErr, os.ErrNotExist) { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("mounted path %s remained visible: %v", path, lastErr) +} diff --git a/internal/mountfs/native_fskit_server.go b/internal/mountfs/native_fskit_server.go new file mode 100644 index 0000000..a4fad05 --- /dev/null +++ b/internal/mountfs/native_fskit_server.go @@ -0,0 +1,948 @@ +package mountfs + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/subtle" + "errors" + "fmt" + "io" + "net" + "os" + "path" + "path/filepath" + "sort" + "strings" + "sync" + "syscall" + "time" + + "github.com/jstar0/codexfold/internal/buildid" + "github.com/jstar0/codexfold/internal/fskitproto" + "github.com/jstar0/codexfold/internal/mountid" +) + +type NativeFSKitServerOptions struct { + SocketPath string + ResourcePath string + Token []byte + Generation uint64 + MaxPayload uint32 + BuildSHA256 string + Recorder func(string) +} + +type nativeFSKitServer struct { + filesystem *Filesystem + token []byte + generation uint64 + maxPayload uint32 + recorder func(string) + health []byte + startedAt time.Time + nodes nativeFSKitNodes +} + +type nativeFSKitNodes struct { + mu sync.Mutex + version uint64 + next uint64 + byPath map[string]uint64 +} + +type nativeFSKitConnection struct { + server *nativeFSKitServer + conn net.Conn + handles map[uint64]*nativeFSKitHandle + nextHandle uint64 +} + +type nativeFSKitHandle struct { + coreHandle uint64 + path string + flags int + snapshotWrite bool + snapshotFloor int64 + health bool +} + +func ServeNativeFSKit(ctx context.Context, filesystem *Filesystem, options NativeFSKitServerOptions) error { + if filesystem == nil { + return errors.New("FSKit server filesystem is required") + } + if options.SocketPath == "" || !filepath.IsAbs(options.SocketPath) { + return errors.New("FSKit server socket path must be absolute") + } + if len(options.SocketPath) >= 104 { + return errors.New("FSKit server socket path exceeds the macOS Unix socket limit") + } + if options.ResourcePath == "" || !filepath.IsAbs(options.ResourcePath) { + return errors.New("FSKit resource path must be absolute") + } + if fskitproto.UsesDirectoryResource(options.ResourcePath) { + relativeSocket, err := filepath.Rel(filepath.Clean(options.ResourcePath), filepath.Clean(options.SocketPath)) + if err != nil || relativeSocket == "." || relativeSocket == ".." || strings.HasPrefix(relativeSocket, ".."+string(filepath.Separator)) { + return errors.New("directory FSKit resource requires its Unix socket inside the resource directory") + } + } + if options.MaxPayload == 0 { + options.MaxPayload = fskitproto.DefaultMaxPayload + } + if len(options.Token) == 0 { + options.Token = make([]byte, 32) + if _, err := rand.Read(options.Token); err != nil { + return fmt.Errorf("generate FSKit authentication token: %w", err) + } + } + if len(options.Token) < 16 || len(options.Token) > 256 { + return errors.New("FSKit authentication token must contain 16 to 256 bytes") + } + if options.Generation == 0 { + var generation [8]byte + if _, err := rand.Read(generation[:]); err != nil { + return fmt.Errorf("generate FSKit mount generation: %w", err) + } + for _, value := range generation { + options.Generation = options.Generation<<8 | uint64(value) + } + if options.Generation == 0 { + options.Generation = 1 + } + } + if options.BuildSHA256 == "" { + var err error + options.BuildSHA256, err = buildid.CurrentSHA256() + if err != nil { + return fmt.Errorf("hash native FSKit daemon executable: %w", err) + } + } + health, err := mountid.New(options.BuildSHA256) + if err != nil { + return fmt.Errorf("generate native FSKit mount identity: %w", err) + } + if err := os.MkdirAll(filepath.Dir(options.SocketPath), 0o700); err != nil { + return fmt.Errorf("create FSKit socket directory: %w", err) + } + if err := removeStaleUnixSocket(options.SocketPath); err != nil { + return err + } + listener, err := net.Listen("unix", options.SocketPath) + if err != nil { + return fmt.Errorf("listen on FSKit socket: %w", err) + } + defer listener.Close() + defer os.Remove(options.SocketPath) + if err := os.Chmod(options.SocketPath, 0o600); err != nil { + return fmt.Errorf("restrict FSKit socket: %w", err) + } + descriptor, err := fskitproto.EncodeDescriptor(fskitproto.Descriptor{ + Generation: options.Generation, + SocketPath: options.SocketPath, + Token: options.Token, + }) + if err != nil { + return err + } + if err := writeNativeFSKitResource(options.ResourcePath, descriptor); err != nil { + return err + } + server := &nativeFSKitServer{ + filesystem: filesystem, + token: append([]byte(nil), options.Token...), + generation: options.Generation, + maxPayload: options.MaxPayload, + recorder: options.Recorder, + health: []byte(health), + startedAt: time.Now(), + nodes: nativeFSKitNodes{ + version: filesystem.NamespaceVersion(), + next: 4, + byPath: map[string]uint64{"/": 2}, + }, + } + go func() { + <-ctx.Done() + _ = listener.Close() + }() + for { + connection, err := listener.Accept() + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return ctx.Err() + } + return fmt.Errorf("accept FSKit connection: %w", err) + } + go (&nativeFSKitConnection{server: server, conn: connection, handles: make(map[uint64]*nativeFSKitHandle), nextHandle: 1}).serve() + } +} + +func removeStaleUnixSocket(socketPath string) error { + info, err := os.Lstat(socketPath) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("inspect FSKit socket path: %w", err) + } + if info.Mode()&os.ModeSocket == 0 { + return errors.New("FSKit socket path exists and is not a Unix socket") + } + connection, dialErr := net.DialTimeout("unix", socketPath, 100*time.Millisecond) + if dialErr == nil { + _ = connection.Close() + return errors.New("FSKit socket is already active") + } + if err := os.Remove(socketPath); err != nil { + return fmt.Errorf("remove stale FSKit socket: %w", err) + } + return nil +} + +func writeNativeFSKitResource(resourcePath string, data []byte) error { + descriptorPath, err := fskitproto.ResourceDescriptorPath(resourcePath) + if err != nil { + return err + } + if fskitproto.UsesDirectoryResource(resourcePath) { + if err := os.MkdirAll(resourcePath, 0o700); err != nil { + return fmt.Errorf("create FSKit resource directory: %w", err) + } + if err := os.Chmod(resourcePath, 0o700); err != nil { + return fmt.Errorf("restrict FSKit resource directory: %w", err) + } + } + if err := os.MkdirAll(filepath.Dir(descriptorPath), 0o700); err != nil { + return fmt.Errorf("create FSKit resource directory: %w", err) + } + temporary, err := os.CreateTemp(filepath.Dir(descriptorPath), ".codexfold-fskit-resource-*") + if err != nil { + return fmt.Errorf("create FSKit resource: %w", err) + } + temporaryPath := temporary.Name() + committed := false + defer func() { + _ = temporary.Close() + if !committed { + _ = os.Remove(temporaryPath) + } + }() + if err := temporary.Chmod(0o600); err != nil { + return fmt.Errorf("restrict FSKit resource: %w", err) + } + if _, err := temporary.Write(data); err != nil { + return fmt.Errorf("write FSKit resource: %w", err) + } + if err := temporary.Sync(); err != nil { + return fmt.Errorf("sync FSKit resource: %w", err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close FSKit resource: %w", err) + } + if err := os.Rename(temporaryPath, descriptorPath); err != nil { + return fmt.Errorf("publish FSKit resource: %w", err) + } + committed = true + return nil +} + +func (c *nativeFSKitConnection) serve() { + defer c.conn.Close() + defer c.releaseHandles() + authenticated := false + for { + request, err := fskitproto.ReadFrame(c.conn, c.server.maxPayload) + if err != nil { + if !errors.Is(err, io.EOF) && !errors.Is(err, net.ErrClosed) { + c.server.record(fmt.Sprintf("connection_error error=%q", err.Error())) + } + return + } + response := fskitproto.Frame{ + Kind: fskitproto.KindResponse, + Op: request.Op, + RequestID: request.RequestID, + Generation: c.server.generation, + } + if request.Kind != fskitproto.KindRequest { + response.Status = int32(syscall.EPROTO) + } else if !authenticated { + if request.Op != fskitproto.OpHello { + response.Status = int32(syscall.EACCES) + } else { + response.Payload, response.Status = c.hello(request.Payload) + authenticated = response.Status == 0 + } + } else if request.Generation != c.server.generation { + response.Status = int32(syscall.ESTALE) + } else { + response.Payload, response.Status = c.dispatch(request.Op, request.Payload) + } + c.server.record(fmt.Sprintf("operation=%s request=%d status=%d payload=%d", nativeFSKitOperationName(request.Op), request.RequestID, response.Status, len(request.Payload))) + if err := fskitproto.WriteFrame(c.conn, response, c.server.maxPayload); err != nil { + return + } + if !authenticated || response.Status == int32(syscall.ESTALE) { + return + } + } +} + +func (c *nativeFSKitConnection) hello(payload []byte) ([]byte, int32) { + decoder := fskitproto.NewDecoder(payload) + token, err := decoder.Bytes(256) + if err != nil || decoder.Done() != nil || len(token) != len(c.server.token) || subtle.ConstantTimeCompare(token, c.server.token) != 1 { + return nil, int32(syscall.EACCES) + } + encoder := fskitproto.NewEncoder(16) + encoder.Uint32(c.server.maxPayload) + encoder.Uint64(c.server.filesystem.NamespaceVersion()) + return encoder.Data(), 0 +} + +func (c *nativeFSKitConnection) dispatch(operation fskitproto.Op, payload []byte) ([]byte, int32) { + c.server.nodes.syncVersion(c.server.filesystem.NamespaceVersion()) + decoder := fskitproto.NewDecoder(payload) + switch operation { + case fskitproto.OpPing: + if err := decoder.Done(); err != nil { + return nil, int32(syscall.EINVAL) + } + return nil, 0 + case fskitproto.OpGetattr: + name, errno := decodePath(decoder) + if errno != 0 { + return nil, int32(errno) + } + entry, errno := c.server.entry(name) + if errno != 0 { + return nil, int32(errno) + } + encoder := fskitproto.NewEncoder(160) + encoder.Entry(entry) + return encoder.Data(), 0 + case fskitproto.OpReadDir: + name, errno := decodePath(decoder) + if errno != 0 { + return nil, int32(errno) + } + entries, errno := c.server.readDir(name) + if errno != 0 { + return nil, int32(errno) + } + encoder := fskitproto.NewEncoder(16 + len(entries)*160) + encoder.Uint32(uint32(len(entries))) + for _, entry := range entries { + encoder.Entry(entry) + } + return encoder.Data(), 0 + case fskitproto.OpOpen: + name, flags, errno := decodeOpen(decoder) + if errno != 0 { + return nil, int32(errno) + } + openFlags := int(flags &^ fskitproto.OpenFlagSnapshot) + if cleanPath(name) == "/"+mountid.Path { + if openFlags&(os.O_WRONLY|os.O_RDWR) != 0 { + return nil, int32(syscall.EPERM) + } + handle := c.addHealthHandle(name, openFlags) + encoder := fskitproto.NewEncoder(8) + encoder.Uint64(handle) + return encoder.Data(), 0 + } + coreHandle, errno := c.server.filesystem.Open(name, openFlags) + if errno != 0 { + return nil, int32(errno) + } + handle := c.addHandle(coreHandle, name, openFlags, flags&fskitproto.OpenFlagSnapshot != 0) + encoder := fskitproto.NewEncoder(8) + encoder.Uint64(handle) + return encoder.Data(), 0 + case fskitproto.OpCreate: + name, flags, errno := decodeOpen(decoder) + if errno != 0 { + return nil, int32(errno) + } + if _, existing := c.server.filesystem.Getattr(name); existing == 0 { + return nil, int32(syscall.EEXIST) + } + openFlags := int(flags&^fskitproto.OpenFlagSnapshot) | os.O_CREATE | os.O_EXCL + coreHandle, errno := c.server.filesystem.Open(name, openFlags) + if errno != 0 { + return nil, int32(errno) + } + handle := c.addHandle(coreHandle, name, openFlags, flags&fskitproto.OpenFlagSnapshot != 0) + c.server.filesystem.bumpNamespaceVersion() + c.server.nodes.acceptVersion(c.server.filesystem.NamespaceVersion()) + entry, errno := c.server.entry(name) + if errno != 0 { + _ = c.server.filesystem.Release(coreHandle) + delete(c.handles, handle) + return nil, int32(errno) + } + encoder := fskitproto.NewEncoder(176) + encoder.Uint64(handle) + encoder.Entry(entry) + return encoder.Data(), 0 + case fskitproto.OpRead: + handle, offset, length, errno := decodeRead(decoder, c.server.maxPayload) + c.server.record(fmt.Sprintf("io=read handle=%d offset=%d bytes=%d", handle, offset, length)) + handleState, exists := c.handles[handle] + if errno != 0 || !exists { + if errno == 0 { + errno = syscall.EBADF + } + return nil, int32(errno) + } + if handleState.health { + if offset >= int64(len(c.server.health)) { + encoder := fskitproto.NewEncoder(4) + encoder.Bytes(nil) + return encoder.Data(), 0 + } + end := min(int64(len(c.server.health)), offset+int64(length)) + encoder := fskitproto.NewEncoder(4 + int(end-offset)) + encoder.Bytes(c.server.health[offset:end]) + return encoder.Data(), 0 + } + buffer := make([]byte, length) + n, errno := c.server.filesystem.Read(handleState.coreHandle, buffer, offset) + if errno != 0 { + return nil, int32(errno) + } + encoder := fskitproto.NewEncoder(4 + n) + encoder.Bytes(buffer[:n]) + return encoder.Data(), 0 + case fskitproto.OpWrite: + handle, offset, data, errno := decodeWrite(decoder, c.server.maxPayload) + c.server.record(fmt.Sprintf("io=write handle=%d offset=%d bytes=%d", handle, offset, len(data))) + handleState, exists := c.handles[handle] + if errno != 0 || !exists { + if errno == 0 { + errno = syscall.EBADF + } + return nil, int32(errno) + } + if handleState.health { + return nil, int32(syscall.EROFS) + } + n, normalized, errno := c.writeHandle(handleState, data, offset) + if errno != 0 { + return nil, int32(errno) + } + if attribute, attrErrno := c.server.filesystem.Getattr(handleState.path); attrErrno == 0 { + c.server.record(fmt.Sprintf("write_result handle=%d reported=%d normalized=%t visible=%d", handle, n, normalized, attribute.Size)) + } + encoder := fskitproto.NewEncoder(4) + encoder.Uint32(uint32(n)) + return encoder.Data(), 0 + case fskitproto.OpFsync, fskitproto.OpFlush, fskitproto.OpRelease: + handle, err := decoder.Uint64() + handleState, exists := c.handles[handle] + if err != nil || decoder.Done() != nil || !exists { + return nil, int32(syscall.EBADF) + } + if handleState.health { + if operation == fskitproto.OpRelease { + delete(c.handles, handle) + } + return nil, 0 + } + var errno syscall.Errno + switch operation { + case fskitproto.OpFsync: + errno = c.server.filesystem.Fsync(handleState.coreHandle) + case fskitproto.OpFlush: + errno = c.server.filesystem.Flush(handleState.coreHandle) + case fskitproto.OpRelease: + errno = c.server.filesystem.Release(handleState.coreHandle) + delete(c.handles, handle) + } + return nil, int32(errno) + case fskitproto.OpTruncate: + name, errno := decodePathPrefix(decoder) + if errno != 0 { + return nil, int32(errno) + } + size, err := decoder.Int64() + if err != nil || size < 0 || decoder.Done() != nil { + return nil, int32(syscall.EINVAL) + } + return nil, int32(c.server.filesystem.TruncatePath(name, size)) + case fskitproto.OpMkdir: + name, errno := decodePathPrefix(decoder) + if errno != 0 { + return nil, int32(errno) + } + mode, err := decoder.Uint32() + if err != nil || decoder.Done() != nil { + return nil, int32(syscall.EINVAL) + } + errno = c.server.filesystem.Mkdir(name, mode) + if errno == 0 { + c.server.nodes.acceptVersion(c.server.filesystem.NamespaceVersion()) + } + return nil, int32(errno) + case fskitproto.OpRename: + oldName, errno := decodePathPrefix(decoder) + if errno != 0 { + return nil, int32(errno) + } + newName, errno := decodePath(decoder) + if errno != 0 { + return nil, int32(errno) + } + c.server.record(fmt.Sprintf("path_rename old=%q new=%q", oldName, newName)) + errno = c.server.filesystem.Rename(oldName, newName) + if errno == 0 { + c.server.nodes.rename(oldName, newName, c.server.filesystem.NamespaceVersion()) + } + return nil, int32(errno) + case fskitproto.OpUnlink, fskitproto.OpRmdir: + name, errno := decodePath(decoder) + if errno != 0 { + return nil, int32(errno) + } + if operation == fskitproto.OpUnlink { + errno = c.server.filesystem.Unlink(name) + } else { + errno = c.server.filesystem.Rmdir(name) + } + if errno == 0 { + c.server.nodes.remove(name, c.server.filesystem.NamespaceVersion()) + } + return nil, int32(errno) + case fskitproto.OpStatfs: + if err := decoder.Done(); err != nil { + return nil, int32(syscall.EINVAL) + } + stat, err := nativeFSKitStat(c.server.filesystem.nativeRoot) + if err != nil { + return nil, int32(errnoFor(err)) + } + encoder := fskitproto.NewEncoder(64) + encoder.StatFS(stat) + return encoder.Data(), 0 + case fskitproto.OpSync: + if err := decoder.Done(); err != nil { + return nil, int32(syscall.EINVAL) + } + return nil, int32(c.server.filesystem.SyncAll()) + case fskitproto.OpNamespaceVersion: + if err := decoder.Done(); err != nil { + return nil, int32(syscall.EINVAL) + } + encoder := fskitproto.NewEncoder(8) + encoder.Uint64(c.server.filesystem.NamespaceVersion()) + return encoder.Data(), 0 + case fskitproto.OpSetattr: + name, errno := decodePathPrefix(decoder) + if errno != 0 { + return nil, int32(errno) + } + valid, err := decoder.Uint32() + if err != nil || valid&^(fskitproto.SetAttrMode|fskitproto.SetAttrUID|fskitproto.SetAttrGID|fskitproto.SetAttrAccessTime|fskitproto.SetAttrModifyTime) != 0 { + return nil, int32(syscall.EINVAL) + } + mode, modeErr := decoder.Uint32() + uid, uidErr := decoder.Uint32() + gid, gidErr := decoder.Uint32() + accessTime, accessErr := decoder.Time() + modifyTime, modifyErr := decoder.Time() + if errors.Join(modeErr, uidErr, gidErr, accessErr, modifyErr, decoder.Done()) != nil { + return nil, int32(syscall.EINVAL) + } + request := SetAttrRequest{ + Valid: valid, Mode: mode, UID: uid, GID: gid, + AccessTime: accessTime, ModTime: modifyTime, + } + return nil, int32(c.server.filesystem.SetAttributes(name, request)) + case fskitproto.OpGetXattr: + name, errno := decodePathPrefix(decoder) + if errno != 0 { + return nil, int32(errno) + } + attribute, err := decoder.String(4096) + if err != nil || decoder.Done() != nil { + return nil, int32(syscall.EINVAL) + } + value, errno := c.server.filesystem.GetXattr(name, attribute) + if errno != 0 { + return nil, int32(errno) + } + encoder := fskitproto.NewEncoder(4 + len(value)) + encoder.Bytes(value) + return encoder.Data(), 0 + case fskitproto.OpSetXattr: + name, errno := decodePathPrefix(decoder) + if errno != 0 { + return nil, int32(errno) + } + attribute, err := decoder.String(4096) + policy, policyErr := decoder.Uint32() + value, valueErr := decoder.Bytes(int(c.server.maxPayload) - 8192) + if errors.Join(err, policyErr, valueErr, decoder.Done()) != nil || policy > uint32(fskitproto.XattrDelete) { + return nil, int32(syscall.EINVAL) + } + return nil, int32(c.server.filesystem.SetXattr(name, attribute, value, fskitproto.XattrPolicy(policy))) + case fskitproto.OpListXattrs: + name, errno := decodePath(decoder) + if errno != 0 { + return nil, int32(errno) + } + attributes, errno := c.server.filesystem.ListXattrs(name) + if errno != 0 { + return nil, int32(errno) + } + encoder := fskitproto.NewEncoder(4 + len(attributes)*64) + encoder.Uint32(uint32(len(attributes))) + for _, attribute := range attributes { + encoder.String(attribute) + } + return encoder.Data(), 0 + default: + return nil, int32(syscall.ENOSYS) + } +} + +func (c *nativeFSKitConnection) releaseHandles() { + for _, handle := range c.handles { + if !handle.health { + _ = c.server.filesystem.Release(handle.coreHandle) + } + } +} + +func (c *nativeFSKitConnection) addHandle(coreHandle uint64, name string, flags int, snapshotWrite bool) uint64 { + handle := c.nextHandle + c.nextHandle++ + c.handles[handle] = &nativeFSKitHandle{ + coreHandle: coreHandle, path: name, flags: flags, + snapshotWrite: snapshotWrite, snapshotFloor: -1, + } + return handle +} + +func (c *nativeFSKitConnection) addHealthHandle(name string, flags int) uint64 { + handle := c.nextHandle + c.nextHandle++ + c.handles[handle] = &nativeFSKitHandle{path: name, flags: flags, health: true, snapshotFloor: -1} + return handle +} + +func (c *nativeFSKitConnection) writeHandle(handle *nativeFSKitHandle, data []byte, offset int64) (int, bool, syscall.Errno) { + if !handle.snapshotWrite || len(data) == 0 { + n, errno := c.server.filesystem.Write(handle.coreHandle, data, offset) + return n, false, errno + } + currentPath, errno := c.server.filesystem.HandlePath(handle.coreHandle) + if errno != 0 { + return 0, false, errno + } + handle.path = currentPath + attribute, errno := c.server.filesystem.Getattr(currentPath) + if errno != 0 { + return 0, false, errno + } + currentSize := attribute.Size + if offset < 0 || offset > currentSize { + return c.fallbackSnapshotWrite(handle, data, offset) + } + overlap := min(int64(len(data)), currentSize-offset) + current := make([]byte, overlap) + if overlap > 0 { + n, readErrno := c.server.filesystem.Read(handle.coreHandle, current, offset) + if readErrno != 0 { + return 0, false, readErrno + } + current = current[:n] + } + common := commonPrefixBytes(current, data) + if common == len(data) { + return len(data), true, 0 + } + if common == len(current) && offset+int64(common) == currentSize && completeJSONL(data[common:]) { + floor := currentSize + n, writeErrno := c.server.filesystem.Write(handle.coreHandle, data[common:], currentSize) + if writeErrno == 0 { + handle.snapshotFloor = floor + return len(data), true, 0 + } + return n, true, writeErrno + } + if handle.snapshotFloor >= offset && handle.snapshotFloor <= offset+int64(len(data)) { + floorIndex := int(handle.snapshotFloor - offset) + if floorIndex <= len(current) && bytes.Equal(data[:floorIndex], current[:floorIndex]) && completeJSONL(data[floorIndex:]) { + n, writeErrno := c.server.filesystem.Write(handle.coreHandle, data[floorIndex:], currentSize) + if writeErrno == 0 { + return len(data), true, 0 + } + return n, true, writeErrno + } + } + return c.fallbackSnapshotWrite(handle, data, offset) +} + +func (c *nativeFSKitConnection) fallbackSnapshotWrite(handle *nativeFSKitHandle, data []byte, offset int64) (int, bool, syscall.Errno) { + if errno := c.server.filesystem.UseRandomWrites(handle.coreHandle); errno != 0 { + return 0, false, errno + } + handle.flags &^= os.O_APPEND + handle.snapshotWrite = false + handle.snapshotFloor = -1 + n, writeErrno := c.server.filesystem.Write(handle.coreHandle, data, offset) + return n, false, writeErrno +} + +func commonPrefixBytes(left []byte, right []byte) int { + limit := min(len(left), len(right)) + for index := 0; index < limit; index++ { + if left[index] != right[index] { + return index + } + } + return limit +} + +func (s *nativeFSKitServer) readDir(name string) ([]fskitproto.Entry, syscall.Errno) { + names, errno := s.filesystem.ReadDir(name) + if errno != 0 { + return nil, errno + } + entries := make([]fskitproto.Entry, 0, len(names)) + for _, child := range names { + entry, errno := s.entry(path.Join(name, child)) + if errno == syscall.ENOENT { + continue + } + if errno != 0 { + return nil, errno + } + entries = append(entries, entry) + } + if cleanPath(name) == "/" { + entry, errno := s.entry("/" + mountid.Path) + if errno != 0 { + return nil, errno + } + entries = append(entries, entry) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name }) + return entries, 0 +} + +func (s *nativeFSKitServer) entry(name string) (fskitproto.Entry, syscall.Errno) { + cleaned := cleanPath(name) + if cleaned == "/"+mountid.Path { + return fskitproto.Entry{ + Path: cleaned, Name: mountid.Path, NodeID: 3, ParentID: 2, + Type: fskitproto.EntryFile, Mode: 0o400, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), + Size: uint64(len(s.health)), AllocSize: uint64((len(s.health) + 4095) &^ 4095), + ModTime: s.startedAt, ChangeTime: s.startedAt, AccessTime: s.startedAt, + NamespaceID: s.filesystem.NamespaceVersion(), + }, 0 + } + attribute, errno := s.filesystem.Getattr(cleaned) + if errno != 0 { + return fskitproto.Entry{}, errno + } + entryType := fskitproto.EntryUnknown + switch attribute.Mode & syscall.S_IFMT { + case syscall.S_IFREG: + entryType = fskitproto.EntryFile + case syscall.S_IFDIR: + entryType = fskitproto.EntryDirectory + case syscall.S_IFLNK: + entryType = fskitproto.EntrySymlink + } + nodeID := s.nodes.node(cleaned) + parentID := uint64(1) + if cleaned == "/" { + parentID = 1 + } else { + parentID = s.nodes.node(path.Dir(cleaned)) + } + allocated := uint64(0) + if attribute.Size > 0 { + allocated = uint64((attribute.Size + 4095) &^ 4095) + } + return fskitproto.Entry{ + Path: cleaned, Name: path.Base(cleaned), NodeID: nodeID, ParentID: parentID, + Type: entryType, Mode: attribute.Mode & 0o7777, UID: attribute.UID, GID: attribute.GID, + Size: uint64(max(attribute.Size, 0)), AllocSize: allocated, + ModTime: attribute.ModTime, ChangeTime: attribute.ChangeTime, AccessTime: attribute.AccessTime, + NamespaceID: s.filesystem.NamespaceVersion(), + }, 0 +} + +func (s *nativeFSKitServer) record(message string) { + if s.recorder != nil { + s.recorder(message) + } +} + +func (n *nativeFSKitNodes) syncVersion(version uint64) { + n.mu.Lock() + defer n.mu.Unlock() + if n.version == version { + return + } + n.version = version + // FSKit object IDs must not be reused while the volume remains mounted. + // Namespace refreshes invalidate path mappings, but the kernel can still + // hold references to the old items until reclaim callbacks arrive. + n.byPath = map[string]uint64{"/": 2} +} + +func (n *nativeFSKitNodes) acceptVersion(version uint64) { + n.mu.Lock() + n.version = version + n.mu.Unlock() +} + +func (n *nativeFSKitNodes) node(name string) uint64 { + n.mu.Lock() + defer n.mu.Unlock() + if nodeID, exists := n.byPath[name]; exists { + return nodeID + } + nodeID := n.next + n.next++ + n.byPath[name] = nodeID + return nodeID +} + +func (n *nativeFSKitNodes) rename(oldName string, newName string, version uint64) { + n.mu.Lock() + defer n.mu.Unlock() + nodeID, exists := n.byPath[oldName] + if exists { + delete(n.byPath, oldName) + n.byPath[newName] = nodeID + } + n.version = version +} + +func (n *nativeFSKitNodes) remove(name string, version uint64) { + n.mu.Lock() + defer n.mu.Unlock() + delete(n.byPath, name) + n.version = version +} + +func decodePath(decoder *fskitproto.Decoder) (string, syscall.Errno) { + name, errno := decodePathPrefix(decoder) + if errno != 0 { + return "", errno + } + if decoder.Done() != nil { + return "", syscall.EINVAL + } + return name, 0 +} + +func decodePathPrefix(decoder *fskitproto.Decoder) (string, syscall.Errno) { + name, err := decoder.String(1 << 20) + if err != nil || name == "" || cleanPath(name) != name { + return "", syscall.EINVAL + } + return name, 0 +} + +func decodeOpen(decoder *fskitproto.Decoder) (string, uint32, syscall.Errno) { + name, errno := decodePathPrefix(decoder) + if errno != 0 { + return "", 0, errno + } + flags, err := decoder.Uint32() + if err != nil || decoder.Done() != nil { + return "", 0, syscall.EINVAL + } + return name, flags, 0 +} + +func decodeRead(decoder *fskitproto.Decoder, maxPayload uint32) (uint64, int64, int, syscall.Errno) { + handle, err := decoder.Uint64() + if err != nil { + return 0, 0, 0, syscall.EINVAL + } + offset, err := decoder.Int64() + if err != nil || offset < 0 { + return 0, 0, 0, syscall.EINVAL + } + length, err := decoder.Uint32() + if err != nil || length > maxPayload-4 || decoder.Done() != nil { + return 0, 0, 0, syscall.EINVAL + } + return handle, offset, int(length), 0 +} + +func decodeWrite(decoder *fskitproto.Decoder, maxPayload uint32) (uint64, int64, []byte, syscall.Errno) { + handle, err := decoder.Uint64() + if err != nil { + return 0, 0, nil, syscall.EINVAL + } + offset, err := decoder.Int64() + if err != nil || offset < 0 { + return 0, 0, nil, syscall.EINVAL + } + data, err := decoder.Bytes(int(maxPayload) - 20) + if err != nil || decoder.Done() != nil { + return 0, 0, nil, syscall.EINVAL + } + return handle, offset, data, 0 +} + +func nativeFSKitOperationName(operation fskitproto.Op) string { + switch operation { + case fskitproto.OpHello: + return "hello" + case fskitproto.OpPing: + return "ping" + case fskitproto.OpGetattr: + return "getattr" + case fskitproto.OpReadDir: + return "readdir" + case fskitproto.OpOpen: + return "open" + case fskitproto.OpCreate: + return "create" + case fskitproto.OpRead: + return "read" + case fskitproto.OpWrite: + return "write" + case fskitproto.OpFsync: + return "fsync" + case fskitproto.OpFlush: + return "flush" + case fskitproto.OpRelease: + return "release" + case fskitproto.OpTruncate: + return "truncate" + case fskitproto.OpMkdir: + return "mkdir" + case fskitproto.OpRename: + return "rename" + case fskitproto.OpUnlink: + return "unlink" + case fskitproto.OpRmdir: + return "rmdir" + case fskitproto.OpStatfs: + return "statfs" + case fskitproto.OpSync: + return "sync" + case fskitproto.OpNamespaceVersion: + return "namespace_version" + case fskitproto.OpSetattr: + return "setattr" + case fskitproto.OpGetXattr: + return "getxattr" + case fskitproto.OpSetXattr: + return "setxattr" + case fskitproto.OpListXattrs: + return "listxattrs" + default: + return "unknown" + } +} diff --git a/internal/mountfs/native_fskit_server_test.go b/internal/mountfs/native_fskit_server_test.go new file mode 100644 index 0000000..2630ad2 --- /dev/null +++ b/internal/mountfs/native_fskit_server_test.go @@ -0,0 +1,419 @@ +package mountfs + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/jstar0/codexfold/internal/fskitproto" + "github.com/jstar0/codexfold/internal/mountid" +) + +func TestNativeFSKitServerPreservesJSONLWritesAndNamespaceMutations(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + for _, directory := range []string{"sessions", "archived_sessions"} { + if err := os.MkdirAll(filepath.Join(nativeRoot, directory), 0o700); err != nil { + t.Fatal(err) + } + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + client, stop := startNativeFSKitTestServer(t, filesystem, root) + defer stop() + defer client.Close() + + for _, directory := range []string{"/sessions/2026", "/sessions/2026/07", "/sessions/2026/07/17"} { + encoder := fskitproto.NewEncoder(128) + encoder.String(directory) + encoder.Uint32(0o700) + if _, err := client.Call(fskitproto.OpMkdir, encoder.Data()); err != nil { + t.Fatalf("mkdir %s: %v", directory, err) + } + } + + filePath := "/sessions/2026/07/17/session.jsonl" + create := fskitproto.NewEncoder(128) + create.String(filePath) + create.Uint32(uint32(os.O_RDWR | os.O_APPEND)) + created, err := client.Call(fskitproto.OpCreate, create.Data()) + if err != nil { + t.Fatal(err) + } + createdDecoder := fskitproto.NewDecoder(created) + handle, err := createdDecoder.Uint64() + if err != nil { + t.Fatal(err) + } + if _, err := createdDecoder.Entry(); err != nil { + t.Fatal(err) + } + if err := createdDecoder.Done(); err != nil { + t.Fatal(err) + } + + first := []byte("{\"record\":1}\n") + second := []byte("{\"record\":2}\n") + writeNativeFSKitTestPayload(t, client, handle, 0, first) + writeNativeFSKitTestPayload(t, client, handle, int64(len(first)), second) + callNativeFSKitHandle(t, client, fskitproto.OpFsync, handle) + + read := fskitproto.NewEncoder(24) + read.Uint64(handle) + read.Int64(0) + read.Uint32(4096) + readPayload, err := client.Call(fskitproto.OpRead, read.Data()) + if err != nil { + t.Fatal(err) + } + readDecoder := fskitproto.NewDecoder(readPayload) + got, err := readDecoder.Bytes(4096) + if err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), first...), second...) + if !bytes.Equal(got, want) { + t.Fatalf("visible bytes = %q, want %q", got, want) + } + callNativeFSKitHandle(t, client, fskitproto.OpRelease, handle) + + for _, directory := range []string{"/archived_sessions/2026", "/archived_sessions/2026/07", "/archived_sessions/2026/07/17"} { + encoder := fskitproto.NewEncoder(128) + encoder.String(directory) + encoder.Uint32(0o700) + if _, err := client.Call(fskitproto.OpMkdir, encoder.Data()); err != nil { + t.Fatalf("mkdir %s: %v", directory, err) + } + } + archivedPath := "/archived_sessions/2026/07/17/session.jsonl" + rename := fskitproto.NewEncoder(256) + rename.String(filePath) + rename.String(archivedPath) + if _, err := client.Call(fskitproto.OpRename, rename.Data()); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(nativeRoot, "archived_sessions/2026/07/17/session.jsonl")); err != nil { + t.Fatal(err) + } + + unlink := fskitproto.NewEncoder(128) + unlink.String(archivedPath) + if _, err := client.Call(fskitproto.OpUnlink, unlink.Data()); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(nativeRoot, "archived_sessions/2026/07/17/session.jsonl")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("archived path still exists: %v", err) + } +} + +func TestNativeFSKitServerRejectsWrongToken(t *testing.T) { + root := t.TempDir() + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + client, stop := startNativeFSKitTestServer(t, filesystem, root) + if err := client.Close(); err != nil { + t.Fatal(err) + } + defer stop() + + resource, err := os.ReadFile(filepath.Join(root, "resource.bin")) + if err != nil { + t.Fatal(err) + } + descriptor, err := fskitproto.DecodeDescriptor(resource) + if err != nil { + t.Fatal(err) + } + descriptor.Token[0] ^= 0xff + if _, err := fskitproto.Dial(descriptor, time.Second); fskitproto.ErrorNumber(err) != syscall.EACCES { + t.Fatalf("wrong token error = %v, want EACCES", err) + } +} + +func TestNativeFSKitServerExposesReadOnlyMountIdentity(t *testing.T) { + root := t.TempDir() + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + client, stop := startNativeFSKitTestServer(t, filesystem, root) + defer stop() + defer client.Close() + + readdir := fskitproto.NewEncoder(16) + readdir.String("/") + response, err := client.Call(fskitproto.OpReadDir, readdir.Data()) + if err != nil { + t.Fatal(err) + } + decoder := fskitproto.NewDecoder(response) + count, err := decoder.Uint32() + if err != nil { + t.Fatal(err) + } + found := false + for range count { + entry, err := decoder.Entry() + if err != nil { + t.Fatal(err) + } + if entry.Name == mountid.Path { + found = true + } + } + if err := decoder.Done(); err != nil || !found { + t.Fatalf("mount identity listed=%t err=%v", found, err) + } + + open := fskitproto.NewEncoder(128) + open.String("/" + mountid.Path) + open.Uint32(uint32(os.O_RDONLY)) + response, err = client.Call(fskitproto.OpOpen, open.Data()) + if err != nil { + t.Fatal(err) + } + decoder = fskitproto.NewDecoder(response) + handle, err := decoder.Uint64() + if err != nil || decoder.Done() != nil { + t.Fatalf("identity open handle=%d err=%v", handle, err) + } + read := fskitproto.NewEncoder(24) + read.Uint64(handle) + read.Int64(0) + read.Uint32(4096) + response, err = client.Call(fskitproto.OpRead, read.Data()) + if err != nil { + t.Fatal(err) + } + decoder = fskitproto.NewDecoder(response) + identityBytes, err := decoder.Bytes(4096) + if err != nil || decoder.Done() != nil { + t.Fatalf("decode identity: %v", err) + } + identity, err := mountid.Parse(identityBytes) + if err != nil || identity.BuildSHA256 != strings.Repeat("a", 64) { + t.Fatalf("mount identity = %#v err=%v", identity, err) + } + callNativeFSKitHandle(t, client, fskitproto.OpRelease, handle) + + writeOpen := fskitproto.NewEncoder(128) + writeOpen.String("/" + mountid.Path) + writeOpen.Uint32(uint32(os.O_WRONLY)) + if _, err := client.Call(fskitproto.OpOpen, writeOpen.Data()); fskitproto.ErrorNumber(err) != syscall.EPERM { + t.Fatalf("write identity open error = %v, want EPERM", err) + } +} + +func TestNativeFSKitServerPublishesDirectoryResourceWithScopedSocket(t *testing.T) { + root, err := os.MkdirTemp("/private/tmp", "cfs-r-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(root) }) + resource := filepath.Join(root, "native-fskit") + filesystem := NewCanonical() + filesystem.SetNativeRoot(filepath.Join(root, "native")) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + options := NativeFSKitServerOptions{ + SocketPath: filepath.Join(resource, "daemon.sock"), ResourcePath: resource, + Token: bytes.Repeat([]byte{0x24}, 32), Generation: 91, BuildSHA256: strings.Repeat("b", 64), + } + go func() { done <- ServeNativeFSKit(ctx, filesystem, options) }() + defer func() { + cancel() + select { + case err := <-done: + if err != nil && !errors.Is(err, context.Canceled) { + t.Errorf("directory resource server shutdown: %v", err) + } + case <-time.After(5 * time.Second): + t.Error("directory resource server did not stop") + } + }() + deadline := time.Now().Add(5 * time.Second) + var client *fskitproto.Client + err = nil + for time.Now().Before(deadline) { + client, err = fskitproto.DialResource(resource, 100*time.Millisecond) + if err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + if err != nil { + t.Fatalf("dial directory resource: %v", err) + } + defer client.Close() + if _, err := os.Stat(filepath.Join(resource, fskitproto.DescriptorFilename)); err != nil { + t.Fatalf("descriptor file: %v", err) + } + if _, err := os.Lstat(options.SocketPath); err != nil { + t.Fatalf("scoped socket: %v", err) + } +} + +func TestNativeFSKitNodesNeverReuseObjectIDsAcrossNamespaceRefresh(t *testing.T) { + nodes := nativeFSKitNodes{next: 3, byPath: map[string]uint64{"/": 2}} + first := nodes.node("/sessions/first") + nodes.syncVersion(2) + second := nodes.node("/sessions/second") + if second == first { + t.Fatalf("object ID %d was reused after namespace refresh", second) + } + if second <= first { + t.Fatalf("object IDs did not remain monotonic: first=%d second=%d", first, second) + } +} + +func TestNativeFSKitServerNormalizesFSKitWholeFileSnapshotsIntoJSONLAppends(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + targetDirectory := filepath.Join(nativeRoot, "sessions", "2026", "07", "17") + if err := os.MkdirAll(targetDirectory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(nativeRoot, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + target := filepath.Join(targetDirectory, "session.jsonl") + if err := os.WriteFile(target, base, 0o600); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + client, stop := startNativeFSKitTestServer(t, filesystem, root) + defer stop() + defer client.Close() + + open := fskitproto.NewEncoder(128) + open.String("/sessions/2026/07/17/session.jsonl") + open.Uint32(uint32(os.O_RDWR|os.O_APPEND) | fskitproto.OpenFlagSnapshot) + response, err := client.Call(fskitproto.OpOpen, open.Data()) + if err != nil { + t.Fatal(err) + } + decoder := fskitproto.NewDecoder(response) + handle, err := decoder.Uint64() + if err != nil || decoder.Done() != nil { + t.Fatalf("open handle=%d err=%v", handle, err) + } + first := append(append([]byte(nil), base...), []byte("{\"record\":1}\n")...) + second := append(append([]byte(nil), base...), []byte("{\"record\":2}\n")...) + writeNativeFSKitTestPayload(t, client, handle, 0, first) + writeNativeFSKitTestPayload(t, client, handle, 0, second) + callNativeFSKitHandle(t, client, fskitproto.OpFsync, handle) + callNativeFSKitHandle(t, client, fskitproto.OpRelease, handle) + + got, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + want := append(append(append([]byte(nil), base...), []byte("{\"record\":1}\n")...), []byte("{\"record\":2}\n")...) + if !bytes.Equal(got, want) { + t.Fatalf("normalized bytes = %q, want %q", got, want) + } + + reopen := fskitproto.NewEncoder(128) + reopen.String("/sessions/2026/07/17/session.jsonl") + reopen.Uint32(uint32(os.O_RDWR|os.O_APPEND) | fskitproto.OpenFlagSnapshot) + response, err = client.Call(fskitproto.OpOpen, reopen.Data()) + if err != nil { + t.Fatal(err) + } + decoder = fskitproto.NewDecoder(response) + handle, err = decoder.Uint64() + if err != nil || decoder.Done() != nil { + t.Fatalf("reopen handle=%d err=%v", handle, err) + } + replacement := bytes.Replace(want, []byte("{\"record\":0}"), []byte("{\"record\":9}"), 1) + writeNativeFSKitTestPayload(t, client, handle, 0, replacement) + callNativeFSKitHandle(t, client, fskitproto.OpFsync, handle) + callNativeFSKitHandle(t, client, fskitproto.OpRelease, handle) + got, err = os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, replacement) { + t.Fatalf("random snapshot bytes = %q, want %q", got, replacement) + } +} + +func startNativeFSKitTestServer(t *testing.T, filesystem *Filesystem, root string) (*fskitproto.Client, func()) { + t.Helper() + socketRoot, err := os.MkdirTemp("/private/tmp", "cfs-") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + options := NativeFSKitServerOptions{ + SocketPath: filepath.Join(socketRoot, "daemon.sock"), ResourcePath: filepath.Join(root, "resource.bin"), + Token: bytes.Repeat([]byte{0x42}, 32), Generation: 77, BuildSHA256: strings.Repeat("a", 64), + } + go func() { done <- ServeNativeFSKit(ctx, filesystem, options) }() + deadline := time.Now().Add(5 * time.Second) + var client *fskitproto.Client + var dialErr error + for time.Now().Before(deadline) { + select { + case serveErr := <-done: + cancel() + t.Fatalf("FSKit test server exited during startup: %v", serveErr) + default: + } + client, dialErr = fskitproto.DialResource(options.ResourcePath, 100*time.Millisecond) + if dialErr == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + if dialErr != nil { + cancel() + t.Fatalf("start FSKit test server: %v", dialErr) + } + stop := func() { + cancel() + defer os.RemoveAll(socketRoot) + select { + case err := <-done: + if err != nil && !errors.Is(err, context.Canceled) { + t.Errorf("FSKit server shutdown: %v", err) + } + case <-time.After(5 * time.Second): + t.Error("FSKit server did not stop") + } + } + return client, stop +} + +func writeNativeFSKitTestPayload(t *testing.T, client *fskitproto.Client, handle uint64, offset int64, data []byte) { + t.Helper() + encoder := fskitproto.NewEncoder(20 + len(data)) + encoder.Uint64(handle) + encoder.Int64(offset) + encoder.Bytes(data) + response, err := client.Call(fskitproto.OpWrite, encoder.Data()) + if err != nil { + t.Fatal(err) + } + decoder := fskitproto.NewDecoder(response) + written, err := decoder.Uint32() + if err != nil || decoder.Done() != nil || int(written) != len(data) { + t.Fatalf("write response bytes=%d err=%v", written, err) + } +} + +func callNativeFSKitHandle(t *testing.T, client *fskitproto.Client, operation fskitproto.Op, handle uint64) { + t.Helper() + encoder := fskitproto.NewEncoder(8) + encoder.Uint64(handle) + if _, err := client.Call(operation, encoder.Data()); err != nil { + t.Fatal(err) + } +} diff --git a/internal/mountfs/native_fskit_stat_darwin.go b/internal/mountfs/native_fskit_stat_darwin.go new file mode 100644 index 0000000..4464550 --- /dev/null +++ b/internal/mountfs/native_fskit_stat_darwin.go @@ -0,0 +1,34 @@ +//go:build darwin + +package mountfs + +import ( + "os" + "path/filepath" + + "github.com/jstar0/codexfold/internal/fskitproto" + "golang.org/x/sys/unix" +) + +func nativeFSKitStat(root string) (fskitproto.StatFS, error) { + if root == "" { + root = os.TempDir() + } + root, err := filepath.Abs(root) + if err != nil { + return fskitproto.StatFS{}, err + } + var stat unix.Statfs_t + if err := unix.Statfs(root, &stat); err != nil { + return fskitproto.StatFS{}, err + } + blockSize := uint64(stat.Bsize) + total := stat.Blocks * blockSize + available := stat.Bavail * blockSize + free := stat.Bfree * blockSize + return fskitproto.StatFS{ + BlockSize: uint32(stat.Bsize), IOSize: 4 * 1024 * 1024, + TotalBytes: total, AvailableBytes: available, FreeBytes: free, UsedBytes: total - free, + TotalFiles: stat.Files, FreeFiles: stat.Ffree, + }, nil +} diff --git a/internal/mountfs/native_fskit_stat_other.go b/internal/mountfs/native_fskit_stat_other.go new file mode 100644 index 0000000..96507de --- /dev/null +++ b/internal/mountfs/native_fskit_stat_other.go @@ -0,0 +1,19 @@ +//go:build !darwin + +package mountfs + +import ( + "os" + + "github.com/jstar0/codexfold/internal/fskitproto" +) + +func nativeFSKitStat(string) (fskitproto.StatFS, error) { + return fskitproto.StatFS{ + BlockSize: 4096, IOSize: 4 * 1024 * 1024, + TotalBytes: 1 << 40, AvailableBytes: 1 << 39, FreeBytes: 1 << 39, UsedBytes: 1 << 39, + TotalFiles: 1 << 32, FreeFiles: 1 << 31, + }, nil +} + +var _ = os.ErrNotExist diff --git a/internal/mountfs/native_namespace_watch_darwin.go b/internal/mountfs/native_namespace_watch_darwin.go new file mode 100644 index 0000000..38a4396 --- /dev/null +++ b/internal/mountfs/native_namespace_watch_darwin.go @@ -0,0 +1,102 @@ +//go:build darwin + +package mountfs + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "time" + + "golang.org/x/sys/unix" +) + +func (f *Filesystem) WatchNativeNamespace(ctx context.Context) error { + f.mu.RLock() + root := f.nativeRoot + f.mu.RUnlock() + if root == "" { + return nil + } + queue, err := unix.Kqueue() + if err != nil { + return fmt.Errorf("create native namespace kqueue: %w", err) + } + defer unix.Close(queue) + watchers := make(map[int]string) + closeWatchers := func() { + for descriptor := range watchers { + _ = unix.Close(descriptor) + } + clear(watchers) + } + defer closeWatchers() + rescan := func() error { + closeWatchers() + for _, namespace := range []string{"sessions", "archived_sessions"} { + base := filepath.Join(root, namespace) + err := filepath.WalkDir(base, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + if errors.Is(walkErr, os.ErrNotExist) { + return nil + } + return walkErr + } + if !entry.IsDir() { + return nil + } + descriptor, err := unix.Open(name, unix.O_EVTONLY|unix.O_CLOEXEC, 0) + if err != nil { + if errors.Is(err, unix.ENOENT) { + return nil + } + return err + } + event := unix.Kevent_t{ + Ident: uint64(descriptor), Filter: unix.EVFILT_VNODE, + Flags: unix.EV_ADD | unix.EV_ENABLE | unix.EV_CLEAR, + Fflags: unix.NOTE_WRITE | unix.NOTE_DELETE | unix.NOTE_RENAME | + unix.NOTE_ATTRIB | unix.NOTE_LINK | unix.NOTE_REVOKE, + } + if _, err := unix.Kevent(queue, []unix.Kevent_t{event}, nil, nil); err != nil { + _ = unix.Close(descriptor) + return err + } + watchers[descriptor] = name + return nil + }) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("watch native namespace %s: %w", namespace, err) + } + } + return nil + } + if err := rescan(); err != nil { + return err + } + f.bumpNamespaceVersion() + events := make([]unix.Kevent_t, 64) + for { + if err := ctx.Err(); err != nil { + return err + } + timeout := unix.NsecToTimespec((500 * time.Millisecond).Nanoseconds()) + count, err := unix.Kevent(queue, nil, events, &timeout) + if err != nil { + if errors.Is(err, unix.EINTR) { + continue + } + return fmt.Errorf("wait for native namespace event: %w", err) + } + if count == 0 { + continue + } + f.bumpNamespaceVersion() + if err := rescan(); err != nil { + return err + } + } +} diff --git a/internal/mountfs/native_namespace_watch_darwin_test.go b/internal/mountfs/native_namespace_watch_darwin_test.go new file mode 100644 index 0000000..1f2b62d --- /dev/null +++ b/internal/mountfs/native_namespace_watch_darwin_test.go @@ -0,0 +1,55 @@ +//go:build darwin + +package mountfs + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +func TestWatchNativeNamespaceBumpsVersionForExternalEntryChanges(t *testing.T) { + root := t.TempDir() + for _, namespace := range []string{"sessions", "archived_sessions"} { + if err := os.MkdirAll(filepath.Join(root, namespace), 0o700); err != nil { + t.Fatal(err) + } + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- filesystem.WatchNativeNamespace(ctx) }() + initial := filesystem.NamespaceVersion() + readyDeadline := time.Now().Add(5 * time.Second) + for filesystem.NamespaceVersion() == initial && time.Now().Before(readyDeadline) { + time.Sleep(10 * time.Millisecond) + } + if filesystem.NamespaceVersion() == initial { + t.Fatal("native namespace watcher did not become ready") + } + baseline := filesystem.NamespaceVersion() + target := filepath.Join(root, "sessions", "external.jsonl") + if err := os.WriteFile(target, []byte("{}\n"), 0o600); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for filesystem.NamespaceVersion() == baseline && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if filesystem.NamespaceVersion() == baseline { + t.Fatal("external namespace creation did not bump the version") + } + cancel() + select { + case err := <-done: + if err != nil && !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("native namespace watcher did not stop") + } +} diff --git a/internal/mountfs/native_namespace_watch_other.go b/internal/mountfs/native_namespace_watch_other.go new file mode 100644 index 0000000..180636e --- /dev/null +++ b/internal/mountfs/native_namespace_watch_other.go @@ -0,0 +1,10 @@ +//go:build !darwin + +package mountfs + +import "context" + +func (f *Filesystem) WatchNativeNamespace(ctx context.Context) error { + <-ctx.Done() + return ctx.Err() +} diff --git a/internal/mountfs/native_preflight.go b/internal/mountfs/native_preflight.go new file mode 100644 index 0000000..ee37085 --- /dev/null +++ b/internal/mountfs/native_preflight.go @@ -0,0 +1,81 @@ +package mountfs + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "unicode/utf8" +) + +type NativePreflightReport struct { + Files int `json:"files"` + Bytes int64 `json:"bytes"` + ValidatedFiles int `json:"validated_files"` + IncrementalFiles int `json:"incremental_files"` + CachedFiles int `json:"cached_files"` + ValidatedBytes int64 `json:"validated_bytes"` + CachePath string `json:"cache_path,omitempty"` + CacheRebuilt bool `json:"cache_rebuilt,omitempty"` +} + +func (f *Filesystem) ValidateNativeWriterRollouts(ctx context.Context) (NativePreflightReport, error) { + f.mu.RLock() + root := f.nativeRoot + f.mu.RUnlock() + if root == "" { + return NativePreflightReport{}, nil + } + return validateNativeWriterRollouts(ctx, root) +} + +func validateNativeWriterRollouts(ctx context.Context, nativeRoot string) (NativePreflightReport, error) { + return validateNativeWriterRolloutsCached(ctx, nativeRoot) +} + +func validateNativeJSONL(ctx context.Context, filePath string) (int64, error) { + file, err := os.Open(filePath) + if err != nil { + return 0, err + } + reader := bufio.NewReaderSize(file, 1<<20) + var lineNumber int64 + var bytesRead int64 + for { + if err := ctx.Err(); err != nil { + _ = file.Close() + return bytesRead, err + } + line, readErr := reader.ReadBytes('\n') + bytesRead += int64(len(line)) + if len(line) != 0 { + lineNumber++ + } + if errors.Is(readErr, io.EOF) { + if len(line) != 0 { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight %s line %d is missing its final newline", filePath, lineNumber) + } + break + } + if readErr != nil { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight read %s line %d: %w", filePath, lineNumber, readErr) + } + if !utf8.Valid(line) { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight %s line %d is not valid UTF-8", filePath, lineNumber) + } + if !json.Valid(line) { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight %s line %d is not valid JSON", filePath, lineNumber) + } + } + if err := file.Close(); err != nil { + return bytesRead, err + } + return bytesRead, nil +} diff --git a/internal/mountfs/native_preflight_audit.go b/internal/mountfs/native_preflight_audit.go new file mode 100644 index 0000000..3b8267c --- /dev/null +++ b/internal/mountfs/native_preflight_audit.go @@ -0,0 +1,72 @@ +package mountfs + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" +) + +type NativePreflightIssue struct { + Path string `json:"path"` + Message string `json:"message"` +} + +type NativePreflightAudit struct { + NativePreflightReport + Issues []NativePreflightIssue `json:"issues,omitempty"` +} + +func AuditNativeWriterRollouts(ctx context.Context, nativeRoot string) (NativePreflightAudit, error) { + root := filepath.Clean(nativeRoot) + activeRoot := filepath.Join(root, "sessions") + report := NativePreflightAudit{} + err := filepath.WalkDir(activeRoot, func(filePath string, entry os.DirEntry, walkErr error) error { + if err := ctx.Err(); err != nil { + return err + } + if walkErr != nil { + report.Issues = append(report.Issues, NativePreflightIssue{Path: filePath, Message: walkErr.Error()}) + if entry != nil && entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if entry.Type()&os.ModeSymlink != 0 { + report.Issues = append(report.Issues, NativePreflightIssue{Path: filePath, Message: "symlink is not allowed in the active native rollout tree"}) + return nil + } + if entry.IsDir() { + return nil + } + if !entry.Type().IsRegular() { + report.Issues = append(report.Issues, NativePreflightIssue{Path: filePath, Message: "non-regular file is not allowed in the active native rollout tree"}) + return nil + } + if strings.HasPrefix(entry.Name(), "._") || !strings.HasSuffix(entry.Name(), ".jsonl") { + return nil + } + info, err := entry.Info() + if err != nil { + report.Issues = append(report.Issues, NativePreflightIssue{Path: filePath, Message: err.Error()}) + return nil + } + report.Files++ + report.Bytes += info.Size() + validated, err := validateNativeJSONL(ctx, filePath) + report.ValidatedBytes += validated + report.ValidatedFiles++ + if err != nil { + report.Issues = append(report.Issues, NativePreflightIssue{Path: filePath, Message: err.Error()}) + } + return nil + }) + if os.IsNotExist(err) { + return NativePreflightAudit{}, nil + } + if err != nil { + return report, fmt.Errorf("audit active native rollouts: %w", err) + } + return report, nil +} diff --git a/internal/mountfs/native_preflight_cache.go b/internal/mountfs/native_preflight_cache.go new file mode 100644 index 0000000..5af29fa --- /dev/null +++ b/internal/mountfs/native_preflight_cache.go @@ -0,0 +1,276 @@ +package mountfs + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "unicode/utf8" +) + +const ( + nativePreflightCacheVersion = 1 + nativeFingerprintWindow = 64 << 10 +) + +type nativePreflightCache struct { + Version int `json:"version"` + Entries map[string]nativePreflightEntry `json:"entries"` +} + +type nativePreflightEntry struct { + Size int64 `json:"size"` + ModTimeNS int64 `json:"mod_time_ns"` + HeadSHA256 string `json:"head_sha256"` + TailSHA256 string `json:"tail_sha256"` +} + +func validateNativeWriterRolloutsCached(ctx context.Context, nativeRoot string) (NativePreflightReport, error) { + root := filepath.Clean(nativeRoot) + activeRoot := filepath.Join(root, "sessions") + cachePath := filepath.Join(root, ".codexfold-native-preflight-v1.json") + cache, cacheBytes, rebuilt := loadNativePreflightCache(cachePath) + next := nativePreflightCache{Version: nativePreflightCacheVersion, Entries: make(map[string]nativePreflightEntry)} + report := NativePreflightReport{CachePath: cachePath, CacheRebuilt: rebuilt} + + err := filepath.WalkDir(activeRoot, func(filePath string, directoryEntry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := ctx.Err(); err != nil { + return err + } + if directoryEntry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("native writer preflight rejects symlink %s", filePath) + } + if directoryEntry.IsDir() { + return nil + } + if !directoryEntry.Type().IsRegular() { + return fmt.Errorf("native writer preflight rejects non-regular file %s", filePath) + } + if strings.HasPrefix(directoryEntry.Name(), "._") || !strings.HasSuffix(directoryEntry.Name(), ".jsonl") { + return nil + } + info, err := directoryEntry.Info() + if err != nil { + return err + } + relative, err := filepath.Rel(root, filePath) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return fmt.Errorf("native writer preflight path escaped native root: %s", filePath) + } + key := filepath.ToSlash(relative) + current := nativePreflightEntry{Size: info.Size(), ModTimeNS: info.ModTime().UnixNano()} + report.Files++ + report.Bytes += info.Size() + + cached, exists := cache.Entries[key] + if exists && cached.Size == current.Size && cached.ModTimeNS == current.ModTimeNS { + next.Entries[key] = cached + report.CachedFiles++ + return nil + } + + var validatedBytes int64 + incremental := exists && current.Size > cached.Size && nativePrefixMatches(filePath, cached) + if incremental { + validatedBytes, err = validateNativeJSONLFrom(ctx, filePath, cached.Size) + } else { + validatedBytes, err = validateNativeJSONL(ctx, filePath) + } + if err != nil { + return err + } + after, err := os.Stat(filePath) + if err != nil { + return err + } + if after.Size() != current.Size || after.ModTime().UnixNano() != current.ModTimeNS { + return fmt.Errorf("native writer preflight target changed during validation: %s", filePath) + } + current.HeadSHA256, current.TailSHA256, err = nativeFingerprints(filePath, current.Size) + if err != nil { + return err + } + next.Entries[key] = current + report.ValidatedBytes += validatedBytes + if incremental { + report.IncrementalFiles++ + } else { + report.ValidatedFiles++ + } + return nil + }) + if errors.Is(err, os.ErrNotExist) { + err = nil + } + if err != nil { + return report, err + } + if err := writeNativePreflightCache(cachePath, next, cacheBytes); err != nil { + return report, err + } + return report, nil +} + +func loadNativePreflightCache(path string) (nativePreflightCache, []byte, bool) { + empty := nativePreflightCache{Version: nativePreflightCacheVersion, Entries: make(map[string]nativePreflightEntry)} + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return empty, nil, false + } + if err != nil { + return empty, nil, true + } + var cache nativePreflightCache + if json.Unmarshal(data, &cache) != nil || cache.Version != nativePreflightCacheVersion || cache.Entries == nil { + return empty, data, true + } + for key, entry := range cache.Entries { + if key == "" || filepath.IsAbs(key) || strings.HasPrefix(filepath.Clean(key), "..") || entry.Size < 0 || entry.ModTimeNS < 0 || + !validNativeFingerprint(entry.HeadSHA256) || !validNativeFingerprint(entry.TailSHA256) { + return empty, data, true + } + } + return cache, data, false +} + +func validNativeFingerprint(value string) bool { + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == sha256.Size +} + +func writeNativePreflightCache(path string, cache nativePreflightCache, previous []byte) error { + data, err := json.Marshal(cache) + if err != nil { + return err + } + if bytes.Equal(data, previous) { + return nil + } + root := filepath.Dir(path) + if err := os.MkdirAll(root, 0o700); err != nil { + return err + } + temporary, err := os.CreateTemp(root, ".native-preflight-*.tmp") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return err + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + if err := os.Rename(temporaryPath, path); err != nil { + return err + } + return syncDirectory(root) +} + +func nativePrefixMatches(path string, cached nativePreflightEntry) bool { + head, tail, err := nativeFingerprints(path, cached.Size) + return err == nil && head == cached.HeadSHA256 && tail == cached.TailSHA256 +} + +func nativeFingerprints(path string, logicalSize int64) (string, string, error) { + if logicalSize < 0 { + return "", "", errors.New("negative native fingerprint size") + } + file, err := os.Open(path) + if err != nil { + return "", "", err + } + defer file.Close() + window := min(logicalSize, int64(nativeFingerprintWindow)) + head := make([]byte, int(window)) + if _, err := file.ReadAt(head, 0); err != nil && !errors.Is(err, io.EOF) { + return "", "", err + } + tail := make([]byte, int(window)) + if _, err := file.ReadAt(tail, logicalSize-window); err != nil && !errors.Is(err, io.EOF) { + return "", "", err + } + headDigest := sha256.Sum256(head) + tailDigest := sha256.Sum256(tail) + return hex.EncodeToString(headDigest[:]), hex.EncodeToString(tailDigest[:]), nil +} + +func validateNativeJSONLFrom(ctx context.Context, filePath string, offset int64) (int64, error) { + file, err := os.Open(filePath) + if err != nil { + return 0, err + } + if offset < 0 { + _ = file.Close() + return 0, errors.New("negative native preflight offset") + } + if offset > 0 { + boundary := []byte{0} + if _, err := file.ReadAt(boundary, offset-1); err != nil || boundary[0] != '\n' { + _ = file.Close() + return 0, fmt.Errorf("native writer preflight %s cached offset %d is not a JSONL boundary", filePath, offset) + } + } + if _, err := file.Seek(offset, io.SeekStart); err != nil { + _ = file.Close() + return 0, err + } + reader := bufio.NewReaderSize(file, 1<<20) + var lineNumber int64 + var bytesRead int64 + for { + if err := ctx.Err(); err != nil { + _ = file.Close() + return bytesRead, err + } + line, readErr := reader.ReadBytes('\n') + bytesRead += int64(len(line)) + if len(line) != 0 { + lineNumber++ + } + if errors.Is(readErr, io.EOF) { + if len(line) != 0 { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight %s line %d after byte %d is missing its final newline", filePath, lineNumber, offset) + } + break + } + if readErr != nil { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight read %s after byte %d: %w", filePath, offset, readErr) + } + if !utf8.Valid(line) { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight %s line %d after byte %d is not valid UTF-8", filePath, lineNumber, offset) + } + if !json.Valid(line) { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight %s line %d after byte %d is not valid JSON", filePath, lineNumber, offset) + } + } + if err := file.Close(); err != nil { + return bytesRead, err + } + return bytesRead, nil +} diff --git a/internal/mountfs/native_preflight_test.go b/internal/mountfs/native_preflight_test.go new file mode 100644 index 0000000..6b27912 --- /dev/null +++ b/internal/mountfs/native_preflight_test.go @@ -0,0 +1,195 @@ +package mountfs + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestNativeWriterPreflightValidatesOnlyActiveRollouts(t *testing.T) { + root := t.TempDir() + active := filepath.Join(root, "sessions", "2026", "07", "16") + archived := filepath.Join(root, "archived_sessions") + if err := os.MkdirAll(active, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(archived, 0o700); err != nil { + t.Fatal(err) + } + valid := []byte("{\"record\":0}\n{\"record\":1}\n") + if err := os.WriteFile(filepath.Join(active, "rollout-valid.jsonl"), valid, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(active, "._rollout-valid.jsonl"), []byte("not-json"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(archived, "rollout-old.jsonl"), []byte("not-json"), 0o600); err != nil { + t.Fatal(err) + } + report, err := validateNativeWriterRollouts(context.Background(), root) + if err != nil { + t.Fatal(err) + } + if report.Files != 1 || report.Bytes != int64(len(valid)) { + t.Fatalf("preflight report = %#v", report) + } +} + +func TestNativeWriterPreflightRejectsInvalidJSON(t *testing.T) { + root, _ := nativePreflightFixture(t, []byte("{\"record\":0}\nnot-json\n")) + _, err := validateNativeWriterRollouts(context.Background(), root) + if err == nil || !strings.Contains(err.Error(), "line 2") || !strings.Contains(err.Error(), "not valid JSON") { + t.Fatalf("invalid JSON preflight error = %v", err) + } +} + +func TestNativeWriterPreflightRejectsInvalidUTF8(t *testing.T) { + root, _ := nativePreflightFixture(t, []byte{'{', '"', 'x', '"', ':', '"', 0xff, '"', '}', '\n'}) + _, err := validateNativeWriterRollouts(context.Background(), root) + if err == nil || !strings.Contains(err.Error(), "not valid UTF-8") { + t.Fatalf("invalid UTF-8 preflight error = %v", err) + } +} + +func TestNativeWriterPreflightRejectsMissingFinalNewline(t *testing.T) { + root, _ := nativePreflightFixture(t, []byte("{\"record\":0}")) + _, err := validateNativeWriterRollouts(context.Background(), root) + if err == nil || !strings.Contains(err.Error(), "missing its final newline") { + t.Fatalf("missing newline preflight error = %v", err) + } +} + +func TestNativeWriterPreflightRejectsSymlink(t *testing.T) { + root, path := nativePreflightFixture(t, []byte("{\"record\":0}\n")) + link := filepath.Join(filepath.Dir(path), "rollout-link.jsonl") + if err := os.Symlink(path, link); err != nil { + t.Fatal(err) + } + _, err := validateNativeWriterRollouts(context.Background(), root) + if err == nil || !strings.Contains(err.Error(), "rejects symlink") { + t.Fatalf("symlink preflight error = %v", err) + } +} + +func TestNativeWriterPreflightHonorsCancellation(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "sessions"), 0o700); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := validateNativeWriterRollouts(ctx, root) + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled preflight error = %v", err) + } +} + +func TestNativeWriterPreflightCachesAndValidatesOnlyNewTail(t *testing.T) { + root, path := nativePreflightFixture(t, []byte("{\"record\":0}\n")) + first, err := validateNativeWriterRollouts(context.Background(), root) + if err != nil { + t.Fatal(err) + } + if first.ValidatedFiles != 1 || first.CachedFiles != 0 || first.IncrementalFiles != 0 { + t.Fatalf("first preflight = %#v", first) + } + second, err := validateNativeWriterRollouts(context.Background(), root) + if err != nil { + t.Fatal(err) + } + if second.CachedFiles != 1 || second.ValidatedFiles != 0 || second.ValidatedBytes != 0 { + t.Fatalf("cached preflight = %#v", second) + } + + tail := []byte("{\"record\":1}\n") + file, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.Write(tail); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + incremental, err := validateNativeWriterRollouts(context.Background(), root) + if err != nil { + t.Fatal(err) + } + if incremental.IncrementalFiles != 1 || incremental.ValidatedFiles != 0 || incremental.ValidatedBytes != int64(len(tail)) { + t.Fatalf("incremental preflight = %#v", incremental) + } +} + +func TestNativeWriterPreflightFullyRevalidatesSameSizeMutation(t *testing.T) { + root, path := nativePreflightFixture(t, []byte("{\"record\":0}\n")) + if _, err := validateNativeWriterRollouts(context.Background(), root); err != nil { + t.Fatal(err) + } + file, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteAt([]byte("X"), 1); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + future := time.Now().Add(time.Second) + if err := os.Chtimes(path, future, future); err != nil { + t.Fatal(err) + } + if _, err := validateNativeWriterRollouts(context.Background(), root); err == nil || !strings.Contains(err.Error(), "not valid JSON") { + t.Fatalf("same-size mutation preflight error = %v", err) + } +} + +func TestNativeWriterPreflightRebuildsCorruptCache(t *testing.T) { + root, _ := nativePreflightFixture(t, []byte("{\"record\":0}\n")) + cachePath := filepath.Join(root, ".codexfold-native-preflight-v1.json") + if err := os.WriteFile(cachePath, []byte("broken-cache"), 0o600); err != nil { + t.Fatal(err) + } + report, err := validateNativeWriterRollouts(context.Background(), root) + if err != nil { + t.Fatal(err) + } + if !report.CacheRebuilt || report.ValidatedFiles != 1 { + t.Fatalf("rebuilt preflight = %#v", report) + } + if _, _, rebuilt := loadNativePreflightCache(cachePath); rebuilt { + t.Fatal("rewritten preflight cache is still invalid") + } +} + +func TestNativeWriterPreflightExternalRoot(t *testing.T) { + root := os.Getenv("CODEXFOLD_NATIVE_PREFLIGHT_ROOT") + if root == "" { + t.Skip("set CODEXFOLD_NATIVE_PREFLIGHT_ROOT to scan an external native root") + } + report, err := validateNativeWriterRollouts(context.Background(), root) + if err != nil { + t.Fatal(err) + } + t.Logf("validated active native rollouts: files=%d bytes=%d", report.Files, report.Bytes) +} + +func nativePreflightFixture(t *testing.T, data []byte) (string, string) { + t.Helper() + root := t.TempDir() + path := filepath.Join(root, "sessions", "2026", "07", "16", "rollout-fixture.jsonl") + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + return root, path +} diff --git a/internal/mountfs/testdata/codex-real-resume-write.trace b/internal/mountfs/testdata/codex-real-resume-write.trace new file mode 100644 index 0000000..b85940a --- /dev/null +++ b/internal/mountfs/testdata/codex-real-resume-write.trace @@ -0,0 +1,27 @@ +# Sanitized Codex resume write sequence captured on macOS on 2026-07-16. +# Fields are operation, absolute offset, and byte count. No path or content is retained. +open 2 +write 51836 553 +fsync +write 52389 233 +fsync +write 52622 705 +fsync +write 53327 402 +fsync +write 53729 311 +fsync +write 54040 141 +fsync +write 54181 1435 +fsync +write 55616 261 +fsync +write 55877 445 +fsync +write 56322 583 +fsync +write 56905 350 +fsync +flush +release diff --git a/internal/mountfs/xattr_darwin.go b/internal/mountfs/xattr_darwin.go new file mode 100644 index 0000000..efbfcb0 --- /dev/null +++ b/internal/mountfs/xattr_darwin.go @@ -0,0 +1,67 @@ +//go:build darwin + +package mountfs + +import ( + "bytes" + "syscall" + + "github.com/jstar0/codexfold/internal/fskitproto" + "golang.org/x/sys/unix" +) + +func platformSetXattr(path string, attribute string, value []byte, policy fskitproto.XattrPolicy) error { + flags := 0 + switch policy { + case fskitproto.XattrAlwaysSet: + case fskitproto.XattrMustCreate: + flags = unix.XATTR_CREATE + case fskitproto.XattrMustReplace: + flags = unix.XATTR_REPLACE + default: + return syscall.EINVAL + } + return unix.Setxattr(path, attribute, value, flags) +} + +func platformGetXattr(path string, attribute string) ([]byte, error) { + size, err := unix.Getxattr(path, attribute, nil) + if err != nil { + return nil, err + } + value := make([]byte, size) + n, err := unix.Getxattr(path, attribute, value) + if err != nil { + return nil, err + } + return value[:n], nil +} + +func platformListXattrs(path string) ([]string, error) { + size, err := unix.Listxattr(path, nil) + if err != nil { + return nil, err + } + if size == 0 { + return []string{}, nil + } + buffer := make([]byte, size) + n, err := unix.Listxattr(path, buffer) + if err != nil { + return nil, err + } + parts := bytes.Split(bytes.TrimRight(buffer[:n], "\x00"), []byte{0}) + result := make([]string, 0, len(parts)) + for _, part := range parts { + if len(part) != 0 { + result = append(result, string(part)) + } + } + return result, nil +} + +func platformRemoveXattr(path string, attribute string) error { + return unix.Removexattr(path, attribute) +} + +func xattrMissingErrno() syscall.Errno { return syscall.ENOATTR } diff --git a/internal/mountfs/xattr_linux.go b/internal/mountfs/xattr_linux.go new file mode 100644 index 0000000..36e2c57 --- /dev/null +++ b/internal/mountfs/xattr_linux.go @@ -0,0 +1,67 @@ +//go:build linux + +package mountfs + +import ( + "bytes" + "syscall" + + "github.com/jstar0/codexfold/internal/fskitproto" + "golang.org/x/sys/unix" +) + +func platformSetXattr(path string, attribute string, value []byte, policy fskitproto.XattrPolicy) error { + flags := 0 + switch policy { + case fskitproto.XattrAlwaysSet: + case fskitproto.XattrMustCreate: + flags = unix.XATTR_CREATE + case fskitproto.XattrMustReplace: + flags = unix.XATTR_REPLACE + default: + return syscall.EINVAL + } + return unix.Setxattr(path, attribute, value, flags) +} + +func platformGetXattr(path string, attribute string) ([]byte, error) { + size, err := unix.Getxattr(path, attribute, nil) + if err != nil { + return nil, err + } + value := make([]byte, size) + n, err := unix.Getxattr(path, attribute, value) + if err != nil { + return nil, err + } + return value[:n], nil +} + +func platformListXattrs(path string) ([]string, error) { + size, err := unix.Listxattr(path, nil) + if err != nil { + return nil, err + } + if size == 0 { + return []string{}, nil + } + buffer := make([]byte, size) + n, err := unix.Listxattr(path, buffer) + if err != nil { + return nil, err + } + parts := bytes.Split(bytes.TrimRight(buffer[:n], "\x00"), []byte{0}) + result := make([]string, 0, len(parts)) + for _, part := range parts { + if len(part) != 0 { + result = append(result, string(part)) + } + } + return result, nil +} + +func platformRemoveXattr(path string, attribute string) error { + return unix.Removexattr(path, attribute) +} + +func xattrMissingErrno() syscall.Errno { return syscall.ENODATA } diff --git a/internal/mountfs/xattr_other.go b/internal/mountfs/xattr_other.go new file mode 100644 index 0000000..d53d47d --- /dev/null +++ b/internal/mountfs/xattr_other.go @@ -0,0 +1,15 @@ +//go:build !darwin && !linux + +package mountfs + +import ( + "syscall" + + "github.com/jstar0/codexfold/internal/fskitproto" +) + +func platformSetXattr(string, string, []byte, fskitproto.XattrPolicy) error { return syscall.ENOTSUP } +func platformGetXattr(string, string) ([]byte, error) { return nil, syscall.ENOTSUP } +func platformListXattrs(string) ([]string, error) { return nil, syscall.ENOTSUP } +func platformRemoveXattr(string, string) error { return syscall.ENOTSUP } +func xattrMissingErrno() syscall.Errno { return syscall.ENOENT } diff --git a/internal/mountid/identity.go b/internal/mountid/identity.go index 34a3279..c8296f7 100644 --- a/internal/mountid/identity.go +++ b/internal/mountid/identity.go @@ -5,30 +5,58 @@ import ( "encoding/hex" "errors" "strings" + + "github.com/jstar0/codexfold/internal/buildid" ) const ( - Path = ".codexfold-health" - prefix = "codexfold-v1:" + Path = ".codexfold-health" + prefixV1 = "codexfold-v1:" + prefixV2 = "codexfold-v2:" ) -func New() (string, error) { +type Identity struct { + Version int + Nonce string + BuildSHA256 string +} + +func New(buildSHA256 string) (string, error) { + if !buildid.ValidSHA256(buildSHA256) { + return "", errors.New("mount identity build SHA-256 is invalid") + } var random [16]byte if _, err := rand.Read(random[:]); err != nil { return "", err } - return prefix + hex.EncodeToString(random[:]), nil + return prefixV2 + hex.EncodeToString(random[:]) + ":" + buildSHA256, nil } func Validate(value []byte) error { + _, err := Parse(value) + return err +} + +func Parse(value []byte) (Identity, error) { text := string(value) - if !strings.HasPrefix(text, prefix) { - return errors.New("mount identity prefix is invalid") + if strings.HasPrefix(text, prefixV1) { + nonce := strings.TrimPrefix(text, prefixV1) + if !validNonce(nonce) { + return Identity{}, errors.New("mount identity payload is invalid") + } + return Identity{Version: 1, Nonce: nonce}, nil + } + if !strings.HasPrefix(text, prefixV2) { + return Identity{}, errors.New("mount identity prefix is invalid") } - digest := strings.TrimPrefix(text, prefix) - decoded, err := hex.DecodeString(digest) - if err != nil || len(decoded) != 16 { - return errors.New("mount identity payload is invalid") + parts := strings.Split(strings.TrimPrefix(text, prefixV2), ":") + if len(parts) != 2 || !validNonce(parts[0]) || !buildid.ValidSHA256(parts[1]) { + return Identity{}, errors.New("mount identity payload is invalid") } - return nil + return Identity{Version: 2, Nonce: parts[0], BuildSHA256: parts[1]}, nil +} + +func validNonce(value string) bool { + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == 16 } diff --git a/internal/mountid/identity_test.go b/internal/mountid/identity_test.go new file mode 100644 index 0000000..9547f53 --- /dev/null +++ b/internal/mountid/identity_test.go @@ -0,0 +1,40 @@ +package mountid + +import ( + "strings" + "testing" +) + +func TestVersionTwoIdentityCarriesBuildSHA256(t *testing.T) { + build := strings.Repeat("a", 64) + value, err := New(build) + if err != nil { + t.Fatal(err) + } + identity, err := Parse([]byte(value)) + if err != nil { + t.Fatal(err) + } + if identity.Version != 2 || identity.BuildSHA256 != build || len(identity.Nonce) != 32 { + t.Fatalf("identity = %#v", identity) + } +} + +func TestLegacyIdentityRemainsProbeCompatibleWithoutBuild(t *testing.T) { + identity, err := Parse([]byte("codexfold-v1:" + strings.Repeat("a", 32))) + if err != nil { + t.Fatal(err) + } + if identity.Version != 1 || identity.BuildSHA256 != "" { + t.Fatalf("legacy identity = %#v", identity) + } +} + +func TestIdentityRejectsInvalidBuild(t *testing.T) { + if _, err := New("invalid"); err == nil { + t.Fatal("invalid build SHA-256 was accepted") + } + if _, err := Parse([]byte("codexfold-v2:" + strings.Repeat("a", 32) + ":invalid")); err == nil { + t.Fatal("invalid v2 payload was accepted") + } +} diff --git a/internal/pack/build.go b/internal/pack/build.go index 4bd372e..bb5d94d 100644 --- a/internal/pack/build.go +++ b/internal/pack/build.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "math" "os" "path/filepath" "sort" @@ -15,22 +16,25 @@ import ( "time" "github.com/jstar0/codexfold/internal/fold" + "github.com/jstar0/codexfold/internal/storage" "github.com/klauspost/compress/zstd" ) type BuildOptions struct { BlockBytes int64 PackBytes int64 + Budget storage.Checker BeforePublish func() error } type BuildResult struct { - Generation string `json:"generation"` - ObjectCount int `json:"object_count"` - BlockCount int `json:"block_count"` - PackCount int `json:"pack_count"` - RawBytes int64 `json:"raw_bytes"` - StoredBytes int64 `json:"stored_bytes"` + Generation string `json:"generation"` + ObjectCount int `json:"object_count"` + BlockCount int `json:"block_count"` + PackCount int `json:"pack_count"` + RawBytes int64 `json:"raw_bytes"` + StoredBytes int64 `json:"stored_bytes"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` } type packWriter struct { @@ -59,6 +63,27 @@ func Build(ctx context.Context, storeDir string, options BuildOptions) (BuildRes if err != nil { return BuildResult{}, err } + budget := options.Budget + if budget == nil { + guard, err := storage.DefaultGuard(storeDir) + if err != nil { + return BuildResult{}, err + } + budget = guard + } + estimatedBytes, err := estimatedGenerationBytes(refs) + if err != nil { + return BuildResult{}, err + } + storageAssessment, err := budget.Check(ctx, storage.Projection{ + Operation: "pack-build", + AdditionalPersistentBytes: estimatedBytes, + TemporaryBytes: estimatedBytes, + TemporaryPersistentOverlapBytes: estimatedBytes, + }) + if err != nil { + return BuildResult{}, err + } packsDir := filepath.Join(storeDir, "packs") if err := os.MkdirAll(packsDir, 0o755); err != nil { return BuildResult{}, fmt.Errorf("create packs directory: %w", err) @@ -149,9 +174,26 @@ func Build(ctx context.Context, storeDir string, options BuildOptions) (BuildRes if err := publishCurrent(packsDir, generation); err != nil { return BuildResult{}, err } + result.Storage = storage.CompleteAccounting(ctx, storageAssessment, storeDir) return result, nil } +func estimatedGenerationBytes(refs []fold.ObjectRef) (int64, error) { + const fixedOverhead = int64(1 << 20) + var rawBytes int64 + for _, ref := range refs { + if ref.RawBytes < 0 || rawBytes > math.MaxInt64-ref.RawBytes { + return 0, errors.New("pack generation byte estimate overflow") + } + rawBytes += ref.RawBytes + } + compressionOverhead := rawBytes/16 + fixedOverhead + if rawBytes > math.MaxInt64-compressionOverhead { + return 0, errors.New("pack generation byte estimate overflow") + } + return rawBytes + compressionOverhead, nil +} + func referencedObjects(storeDir string) ([]fold.ObjectRef, error) { refs := make(map[string]fold.ObjectRef) err := filepath.WalkDir(filepath.Join(storeDir, "manifests"), func(path string, entry os.DirEntry, walkErr error) error { diff --git a/internal/pack/pack_test.go b/internal/pack/pack_test.go index 8e9a1e3..d52af36 100644 --- a/internal/pack/pack_test.go +++ b/internal/pack/pack_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/jstar0/codexfold/internal/fold" + "github.com/jstar0/codexfold/internal/storage" ) func TestBuildAndResolverReadExactRandomRanges(t *testing.T) { @@ -91,6 +92,48 @@ func TestResolverSupportsOSCacheBypassOption(t *testing.T) { } } +func TestResolverHoldsGenerationLeaseUntilClose(t *testing.T) { + root := t.TempDir() + refs := putObjects(t, root, []byte("leased-generation")) + writeManifest(t, root, "session", refs) + result, err := Build(context.Background(), root, BuildOptions{}) + if err != nil { + t.Fatal(err) + } + resolver, err := Open(root, OpenOptions{}) + if err != nil { + t.Fatal(err) + } + leaseDirectory := filepath.Join(root, "packs", result.Generation, "leases") + active, err := storage.DirectoryHasActiveLease(leaseDirectory, false) + if err != nil || !active { + t.Fatalf("resolver generation lease: active=%t err=%v", active, err) + } + if err := resolver.Close(); err != nil { + t.Fatal(err) + } + active, err = storage.DirectoryHasActiveLease(leaseDirectory, true) + if err != nil || active { + t.Fatalf("closed resolver generation lease: active=%t err=%v", active, err) + } +} + +func TestOpenDoesNotRecreateGenerationMissingFromCurrent(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "packs"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "packs", "CURRENT"), []byte("gen-missing\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Open(root, OpenOptions{}); err == nil { + t.Fatal("resolver unexpectedly opened a missing generation") + } + if _, err := os.Lstat(filepath.Join(root, "packs", "gen-missing")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("missing pack generation was recreated: %v", err) + } +} + func TestBuildInterruptionKeepsPreviousGenerationCurrent(t *testing.T) { root := t.TempDir() refs := putObjects(t, root, []byte("first-generation")) @@ -123,6 +166,35 @@ func TestBuildInterruptionKeepsPreviousGenerationCurrent(t *testing.T) { } } +func TestBuildBudgetRejectsBeforeCreatingCandidateGeneration(t *testing.T) { + root := t.TempDir() + refs := putObjects(t, root, []byte("budgeted-pack-object")) + writeManifest(t, root, "session", refs) + checker := rejectingChecker{} + if _, err := Build(context.Background(), root, BuildOptions{Budget: &checker}); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("Build error = %v, want storage budget rejection", err) + } + if checker.Calls != 1 { + t.Fatalf("budget checks = %d, want 1", checker.Calls) + } + if _, err := os.Stat(filepath.Join(root, "packs")); !os.IsNotExist(err) { + t.Fatalf("candidate packs directory exists after preflight rejection: %v", err) + } +} + +func TestBuildReportsStorageAccounting(t *testing.T) { + root := t.TempDir() + refs := putObjects(t, root, []byte("accounted-pack")) + writeManifest(t, root, "session", refs) + result, err := Build(context.Background(), root, BuildOptions{}) + if err != nil { + t.Fatal(err) + } + if result.Storage == nil || result.Storage.Budget.ProjectedPeakBytes <= result.Storage.Budget.CurrentPhysicalBytes || result.Storage.After.Packs.ApparentBytes == 0 { + t.Fatalf("pack storage accounting is incomplete: %#v", result.Storage) + } +} + func TestResolverAndDoctorDetectPackCorruption(t *testing.T) { root := t.TempDir() refs := putObjects(t, root, bytes.Repeat([]byte("protected"), 10000)) @@ -256,3 +328,12 @@ func loadCurrentIndex(t *testing.T, root string) Index { } return index } + +type rejectingChecker struct { + Calls int +} + +func (c *rejectingChecker) Check(context.Context, storage.Projection) (storage.Assessment, error) { + c.Calls++ + return storage.Assessment{}, storage.ErrBudgetExceeded +} diff --git a/internal/pack/resolver.go b/internal/pack/resolver.go index 06e9f78..dc7f382 100644 --- a/internal/pack/resolver.go +++ b/internal/pack/resolver.go @@ -15,6 +15,7 @@ import ( "sync" "github.com/jstar0/codexfold/internal/fold" + "github.com/jstar0/codexfold/internal/storage" "github.com/klauspost/compress/zstd" ) @@ -29,6 +30,7 @@ type Resolver struct { objects map[string]Object packs map[string]*os.File cache *blockCache + lease *storage.Lease bypassOSCacheApplied bool closeOnce sync.Once } @@ -49,6 +51,16 @@ func Open(storeDir string, options OpenOptions) (*Resolver, error) { } func openGeneration(directory string, cacheBytes int64, bypassOSCache bool) (*Resolver, error) { + lease, err := storage.AcquireLease(filepath.Join(directory, "leases"), "resolver") + if err != nil { + return nil, fmt.Errorf("acquire pack generation lease: %w", err) + } + keepLease := false + defer func() { + if !keepLease { + _ = lease.Close() + } + }() data, err := os.ReadFile(filepath.Join(directory, "index.json")) if err != nil { return nil, fmt.Errorf("read pack index: %w", err) @@ -64,7 +76,7 @@ func openGeneration(directory string, cacheBytes int64, bypassOSCache bool) (*Re if err := validateIndex(index); err != nil { return nil, err } - resolver := &Resolver{directory: directory, index: index, objects: make(map[string]Object, len(index.Objects)), packs: make(map[string]*os.File), cache: newBlockCache(cacheBytes)} + resolver := &Resolver{directory: directory, index: index, objects: make(map[string]Object, len(index.Objects)), packs: make(map[string]*os.File), cache: newBlockCache(cacheBytes), lease: lease} for _, object := range index.Objects { resolver.objects[object.SHA256] = object for _, block := range object.Blocks { @@ -101,6 +113,7 @@ func openGeneration(directory string, cacheBytes int64, bypassOSCache bool) (*Re } } } + keepLease = true return resolver, nil } @@ -197,6 +210,9 @@ func (r *Resolver) Close() error { closeErr = err } } + if err := r.lease.Close(); err != nil { + closeErr = errors.Join(closeErr, err) + } }) return closeErr } diff --git a/internal/prune/remove_contained_test.go b/internal/prune/remove_contained_test.go index a54feee..2c0d509 100644 --- a/internal/prune/remove_contained_test.go +++ b/internal/prune/remove_contained_test.go @@ -168,7 +168,10 @@ func newRemovalFixture(t *testing.T) removalFixture { } contained := codex.Session{ID: "contained", Title: "Contained", CWD: "/workspace", RolloutPath: containedPath, Archived: true} container := codex.Session{ID: "container", Title: "Container", CWD: "/workspace", RolloutPath: containerPath} - if _, err := fold.Fold(context.Background(), contained, fold.FoldOptions{StoreDir: store, Apply: true, FieldThreshold: 4}); err != nil { + if _, err := fold.Fold(context.Background(), fold.Session{ + ID: contained.ID, Title: contained.Title, CWD: contained.CWD, + RolloutPath: contained.RolloutPath, Archived: contained.Archived, + }, fold.FoldOptions{StoreDir: store, Apply: true, FieldThreshold: 4}); err != nil { t.Fatalf("fold contained fixture: %v", err) } diff --git a/internal/reconcile/budget.go b/internal/reconcile/budget.go new file mode 100644 index 0000000..89d9688 --- /dev/null +++ b/internal/reconcile/budget.go @@ -0,0 +1,18 @@ +package reconcile + +import ( + "errors" + "math" +) + +func estimatedOutputBytes(sourceBytes int64) (int64, error) { + const fixedOverhead = int64(1 << 20) + if sourceBytes < 0 { + return 0, errors.New("output byte estimate cannot be negative") + } + overhead := sourceBytes/16 + fixedOverhead + if sourceBytes > math.MaxInt64-overhead { + return 0, errors.New("output byte estimate overflow") + } + return sourceBytes + overhead, nil +} diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go index b4eb192..1111147 100644 --- a/internal/reconcile/reconcile.go +++ b/internal/reconcile/reconcile.go @@ -2,6 +2,7 @@ package reconcile import ( "bufio" + "context" "crypto/sha256" "encoding/hex" "errors" @@ -11,6 +12,8 @@ import ( "path/filepath" "sort" "time" + + "github.com/jstar0/codexfold/internal/storage" ) type SourceSummary struct { @@ -24,16 +27,22 @@ type SourceSummary struct { } type Result struct { - Base SourceSummary `json:"base"` - Branch SourceSummary `json:"branch"` - SharedRecords int64 `json:"shared_records"` - BaseOnlyRecords int64 `json:"base_only_records"` - AddedFromBranch int64 `json:"added_from_branch"` - OutputRecords int64 `json:"output_records"` - OutputBytes int64 `json:"output_bytes,omitempty"` - OutputSHA256 string `json:"output_sha256,omitempty"` - OutputPath string `json:"output_path,omitempty"` - OutputRegressions int64 `json:"output_timestamp_regressions,omitempty"` + Base SourceSummary `json:"base"` + Branch SourceSummary `json:"branch"` + SharedRecords int64 `json:"shared_records"` + BaseOnlyRecords int64 `json:"base_only_records"` + AddedFromBranch int64 `json:"added_from_branch"` + OutputRecords int64 `json:"output_records"` + OutputBytes int64 `json:"output_bytes,omitempty"` + OutputSHA256 string `json:"output_sha256,omitempty"` + OutputPath string `json:"output_path,omitempty"` + OutputRegressions int64 `json:"output_timestamp_regressions,omitempty"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` +} + +type MergeOptions struct { + Context context.Context + Budget storage.Checker } type recordKey struct { @@ -70,6 +79,10 @@ func Analyze(basePath, branchPath string) (Result, error) { } func Merge(basePath, branchPath, outputPath string) (Result, error) { + return MergeWithOptions(basePath, branchPath, outputPath, MergeOptions{}) +} + +func MergeWithOptions(basePath, branchPath, outputPath string, options MergeOptions) (Result, error) { if outputPath == "" { return Result{}, errors.New("output path is required") } @@ -112,6 +125,28 @@ func Merge(basePath, branchPath, outputPath string) (Result, error) { } return records[i].timestamp.Before(records[j].timestamp) }) + ctx := options.Context + if ctx == nil { + ctx = context.Background() + } + var outputBytes int64 + for _, record := range records { + if record.length < 0 || outputBytes > int64(^uint64(0)>>1)-record.length { + return Result{}, errors.New("reconciled output byte count overflow") + } + outputBytes += record.length + } + budget := options.Budget + if budget == nil { + budget = storage.VolumeGuard{Path: filepath.Dir(outputAbs)} + } + storageAssessment, err := budget.Check(ctx, storage.Projection{ + Operation: "reconcile-rollout", AdditionalPersistentBytes: outputBytes, + TemporaryBytes: outputBytes, TemporaryPersistentOverlapBytes: outputBytes, + }) + if err != nil { + return Result{}, err + } baseFile, err := os.Open(baseAbs) if err != nil { @@ -146,6 +181,9 @@ func Merge(basePath, branchPath, outputPath string) (Result, error) { outputHasher := sha256.New() writer := io.MultiWriter(temp, outputHasher) for _, record := range records { + if err := ctx.Err(); err != nil { + return Result{}, err + } source := baseFile if record.source == 1 { source = branchFile @@ -185,6 +223,7 @@ func Merge(basePath, branchPath, outputPath string) (Result, error) { if output.summary.Records != result.OutputRecords || output.summary.SHA256 != result.OutputSHA256 || output.summary.TimestampRegressions != 0 { return Result{}, errors.New("merged output verification failed") } + result.Storage = storage.CompleteAccounting(ctx, storageAssessment, "") return result, nil } diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go index c8e351c..eed857b 100644 --- a/internal/reconcile/reconcile_test.go +++ b/internal/reconcile/reconcile_test.go @@ -1,12 +1,27 @@ package reconcile import ( + "context" + "errors" "os" "path/filepath" "strings" "testing" + + "github.com/jstar0/codexfold/internal/storage" ) +type reconcileRejectingChecker struct { + Calls int + Projection storage.Projection +} + +func (c *reconcileRejectingChecker) Check(_ context.Context, projection storage.Projection) (storage.Assessment, error) { + c.Calls++ + c.Projection = projection + return storage.Assessment{}, storage.ErrBudgetExceeded +} + func TestMergeInsertsBranchOnlyRecordsByTimestamp(t *testing.T) { dir := t.TempDir() base := writeRollout(t, dir, "base.jsonl", []string{ @@ -27,6 +42,9 @@ func TestMergeInsertsBranchOnlyRecordsByTimestamp(t *testing.T) { if result.SharedRecords != 2 || result.AddedFromBranch != 1 || result.OutputRecords != 3 { t.Fatalf("unexpected result: %#v", result) } + if result.Storage == nil || result.Storage.Budget.ProjectedPeakBytes <= 0 || result.Storage.ActualReclaimedBytes != 0 { + t.Fatalf("merge storage accounting is incomplete: %#v", result.Storage) + } data, err := os.ReadFile(output) if err != nil { t.Fatal(err) @@ -41,6 +59,23 @@ func TestMergeInsertsBranchOnlyRecordsByTimestamp(t *testing.T) { } } +func TestMergeBudgetRejectsBeforeCreatingOutput(t *testing.T) { + dir := t.TempDir() + base := writeRollout(t, dir, "base.jsonl", []string{record("2026-07-13T01:00:00Z", "a")}) + branch := writeRollout(t, dir, "branch.jsonl", []string{record("2026-07-13T01:01:00Z", "b")}) + output := filepath.Join(dir, "output", "merged.jsonl") + checker := &reconcileRejectingChecker{} + if _, err := MergeWithOptions(base, branch, output, MergeOptions{Budget: checker}); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("MergeWithOptions error = %v, want storage budget rejection", err) + } + if checker.Calls != 1 || checker.Projection.Operation != "reconcile-rollout" || checker.Projection.AdditionalPersistentBytes <= 0 { + t.Fatalf("unexpected merge budget projection: %#v", checker) + } + if _, err := os.Stat(filepath.Dir(output)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("merge output directory exists after preflight rejection: %v", err) + } +} + func TestMergePreservesExcessDuplicateOccurrence(t *testing.T) { dir := t.TempDir() line := record("2026-07-13T01:00:00Z", "same") diff --git a/internal/reconcile/repair.go b/internal/reconcile/repair.go index d1f0d05..c91a4be 100644 --- a/internal/reconcile/repair.go +++ b/internal/reconcile/repair.go @@ -3,6 +3,7 @@ package reconcile import ( "bufio" "bytes" + "context" "crypto/sha256" "encoding/hex" "encoding/json" @@ -13,6 +14,8 @@ import ( "path/filepath" "regexp" "time" + + "github.com/jstar0/codexfold/internal/storage" ) const maxRepairBufferedBytes = int64(64 * 1024 * 1024) @@ -20,46 +23,63 @@ const maxRepairBufferedBytes = int64(64 * 1024 * 1024) var recordStartPattern = regexp.MustCompile(`\{"timestamp"\s*:`) type RepairResult struct { - SourcePath string `json:"source_path"` - SourceBytes int64 `json:"source_bytes"` - SourceSHA256 string `json:"source_sha256"` - PhysicalLines int64 `json:"physical_lines"` - InvalidPhysicalLines int64 `json:"invalid_physical_lines"` - ReconstructedRecords int64 `json:"reconstructed_records"` - OutputPath string `json:"output_path"` - OutputBytes int64 `json:"output_bytes"` - OutputRecords int64 `json:"output_records"` - OutputSHA256 string `json:"output_sha256"` - TimestampRegressions int64 `json:"timestamp_regressions"` - MaximumBufferedBytes int64 `json:"maximum_buffered_bytes"` - OrphanBytes int64 `json:"orphan_bytes,omitempty"` - OrphanLines int64 `json:"orphan_lines,omitempty"` + SourcePath string `json:"source_path"` + SourceBytes int64 `json:"source_bytes"` + SourceSHA256 string `json:"source_sha256"` + PhysicalLines int64 `json:"physical_lines"` + InvalidPhysicalLines int64 `json:"invalid_physical_lines"` + ReconstructedRecords int64 `json:"reconstructed_records"` + SourceConversationRecords int64 `json:"source_conversation_records"` + PreservedConversationRecords int64 `json:"preserved_conversation_records"` + ReconstructedConversationRecords int64 `json:"reconstructed_conversation_records"` + ConversationIntegrityVerified bool `json:"conversation_integrity_verified"` + OutputPath string `json:"output_path"` + OutputBytes int64 `json:"output_bytes"` + OutputRecords int64 `json:"output_records"` + OutputSHA256 string `json:"output_sha256"` + TimestampRegressions int64 `json:"timestamp_regressions"` + MaximumBufferedBytes int64 `json:"maximum_buffered_bytes"` + OrphanBytes int64 `json:"orphan_bytes,omitempty"` + OrphanLines int64 `json:"orphan_lines,omitempty"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` } type RepairOptions struct { AllowOrphans bool OrphanPath string + Context context.Context + Budget storage.Checker } type repairFrame struct { partial []byte - pending [][]byte + pending []repairRecord startedLine int64 } +type repairRecord struct { + data []byte + sourceValid bool +} + type repairWriter struct { - writer io.Writer - stack []repairFrame - bufferedBytes int64 - maximumBuffered int64 - outputRecords int64 - reconstructed int64 - previousTimestamp time.Time - timestampRegressions int64 - orphanWriter *bufio.Writer - allowOrphans bool - orphanBytes int64 - orphanLines int64 + writer io.Writer + stack []repairFrame + bufferedBytes int64 + maximumBuffered int64 + outputRecords int64 + reconstructed int64 + previousTimestamp time.Time + timestampRegressions int64 + orphanWriter *bufio.Writer + allowOrphans bool + orphanBytes int64 + orphanLines int64 + sourceConversationChain conversationChain + preservedConversationChain conversationChain + sourceConversationRecords int64 + preservedConversationRecords int64 + reconstructedConversationRecords int64 } func Repair(sourcePath, outputPath string) (RepairResult, error) { @@ -86,6 +106,24 @@ func RepairWithOptions(sourcePath, outputPath string, options RepairOptions) (Re } else if !errors.Is(err, os.ErrNotExist) { return RepairResult{}, err } + if options.AllowOrphans { + if options.OrphanPath == "" { + return RepairResult{}, errors.New("orphan path is required when allow orphans is enabled") + } + orphanAbs, err := filepath.Abs(options.OrphanPath) + if err != nil { + return RepairResult{}, err + } + if orphanAbs == sourceAbs || orphanAbs == outputAbs { + return RepairResult{}, errors.New("orphan output must be separate from source and repaired output") + } + if _, err := os.Lstat(orphanAbs); err == nil { + return RepairResult{}, fmt.Errorf("orphan output already exists: %s", orphanAbs) + } else if !errors.Is(err, os.ErrNotExist) { + return RepairResult{}, err + } + options.OrphanPath = orphanAbs + } source, err := os.Open(sourceAbs) if err != nil { @@ -96,6 +134,37 @@ func RepairWithOptions(sourcePath, outputPath string, options RepairOptions) (Re if err != nil { return RepairResult{}, err } + ctx := options.Context + if ctx == nil { + ctx = context.Background() + } + estimatedBytes, err := estimatedOutputBytes(before.Size()) + if err != nil { + return RepairResult{}, err + } + persistentBytes := estimatedBytes + if options.AllowOrphans { + if persistentBytes > int64(^uint64(0)>>1)-estimatedBytes { + return RepairResult{}, errors.New("repair output byte estimate overflow") + } + persistentBytes += estimatedBytes + } + budget := options.Budget + if budget == nil { + budget = storage.VolumeGuard{Path: filepath.Dir(outputAbs)} + } + storageAssessment, err := budget.Check(ctx, storage.Projection{ + Operation: "repair-rollout", AdditionalPersistentBytes: persistentBytes, + TemporaryBytes: estimatedBytes, TemporaryPersistentOverlapBytes: estimatedBytes, + }) + if err != nil { + return RepairResult{}, err + } + if options.AllowOrphans && options.Budget == nil { + if _, err := (storage.VolumeGuard{Path: filepath.Dir(options.OrphanPath)}).Check(ctx, storage.Projection{Operation: "repair-orphans", AdditionalPersistentBytes: estimatedBytes}); err != nil { + return RepairResult{}, err + } + } if err := os.MkdirAll(filepath.Dir(outputAbs), 0o700); err != nil { return RepairResult{}, err } @@ -117,14 +186,19 @@ func RepairWithOptions(sourcePath, outputPath string, options RepairOptions) (Re var orphanFile *os.File var orphanWriter *bufio.Writer if options.AllowOrphans { - if options.OrphanPath == "" { - return RepairResult{}, errors.New("orphan path is required when allow orphans is enabled") + if err := os.MkdirAll(filepath.Dir(options.OrphanPath), 0o700); err != nil { + return RepairResult{}, err } orphanFile, err = os.OpenFile(options.OrphanPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) if err != nil { return RepairResult{}, err } defer orphanFile.Close() + defer func() { + if !committed { + _ = os.Remove(options.OrphanPath) + } + }() orphanWriter = bufio.NewWriterSize(orphanFile, 64*1024) } @@ -134,6 +208,9 @@ func RepairWithOptions(sourcePath, outputPath string, options RepairOptions) (Re processor := repairWriter{writer: io.MultiWriter(temp, outputHasher), orphanWriter: orphanWriter, allowOrphans: options.AllowOrphans} reader := bufio.NewReaderSize(source, 1024*1024) for { + if err := ctx.Err(); err != nil { + return RepairResult{}, err + } line, readErr := reader.ReadBytes('\n') if len(line) > 0 { result.PhysicalLines++ @@ -163,17 +240,12 @@ func RepairWithOptions(sourcePath, outputPath string, options RepairOptions) (Re return RepairResult{}, fmt.Errorf("unresolved interrupted record started at physical line %d", processor.stack[0].startedLine) } if len(processor.stack) != 0 { - for _, frame := range processor.stack { - if err := processor.writeOrphan(frame.partial); err != nil { - return RepairResult{}, err - } - for _, pending := range frame.pending { - if err := processor.writeOrphan(pending); err != nil { - return RepairResult{}, err - } - } + if err := processor.salvageUnresolved(); err != nil { + return RepairResult{}, err } - processor.stack = nil + } + if err := processor.verifyConversationIntegrity(); err != nil { + return RepairResult{}, err } if processor.timestampRegressions != 0 && !options.AllowOrphans { return RepairResult{}, fmt.Errorf("repaired record order still has %d timestamp regressions", processor.timestampRegressions) @@ -220,6 +292,10 @@ func RepairWithOptions(sourcePath, outputPath string, options RepairOptions) (Re result.SourceSHA256 = hex.EncodeToString(sourceHasher.Sum(nil)) result.ReconstructedRecords = processor.reconstructed + result.SourceConversationRecords = processor.sourceConversationRecords + result.PreservedConversationRecords = processor.preservedConversationRecords + result.ReconstructedConversationRecords = processor.reconstructedConversationRecords + result.ConversationIntegrityVerified = true result.OutputBytes = verified.summary.Bytes result.OutputRecords = verified.summary.Records result.OutputSHA256 = outputDigest @@ -227,16 +303,18 @@ func RepairWithOptions(sourcePath, outputPath string, options RepairOptions) (Re result.MaximumBufferedBytes = processor.maximumBuffered result.OrphanBytes = processor.orphanBytes result.OrphanLines = processor.orphanLines + result.Storage = storage.CompleteAccounting(ctx, storageAssessment, "") return result, nil } func (w *repairWriter) acceptValid(record []byte) error { + w.trackSourceConversation(record) if len(w.stack) == 0 { - return w.writeRecord(record) + return w.writeRecord(repairRecord{data: record, sourceValid: true}) } copyOfRecord := append([]byte(nil), record...) top := &w.stack[len(w.stack)-1] - top.pending = append(top.pending, copyOfRecord) + top.pending = append(top.pending, repairRecord{data: copyOfRecord, sourceValid: true}) return w.addBuffered(int64(len(copyOfRecord))) } @@ -298,6 +376,9 @@ func (w *repairWriter) writeOrphan(fragment []byte) error { if w.orphanWriter == nil { return errors.New("orphan writer is not configured") } + if containsCompleteConversationRecord(fragment) { + return errors.New("refusing to orphan a complete user or assistant conversation record") + } if _, err := w.orphanWriter.Write(fragment); err != nil { return err } @@ -314,8 +395,8 @@ func (w *repairWriter) finishTop() error { frame := w.stack[index] w.stack = w.stack[:index] w.reconstructed++ - records := make([][]byte, 0, 1+len(frame.pending)) - records = append(records, frame.partial) + records := make([]repairRecord, 0, 1+len(frame.pending)) + records = append(records, repairRecord{data: frame.partial}) records = append(records, frame.pending...) if len(w.stack) != 0 { parent := &w.stack[len(w.stack)-1] @@ -326,17 +407,33 @@ func (w *repairWriter) finishTop() error { if err := w.writeRecord(record); err != nil { return err } - w.bufferedBytes -= int64(len(record)) + w.bufferedBytes -= int64(len(record.data)) + } + return nil +} + +func (w *repairWriter) salvageUnresolved() error { + for _, frame := range w.stack { + if err := w.writeOrphan(frame.partial); err != nil { + return err + } + for _, pending := range frame.pending { + if err := w.writeRecord(pending); err != nil { + return err + } + } } + w.stack = nil + w.bufferedBytes = 0 return nil } -func (w *repairWriter) writeRecord(record []byte) error { - if !json.Valid(record) { +func (w *repairWriter) writeRecord(record repairRecord) error { + if !json.Valid(record.data) { return errors.New("attempted to emit invalid JSON record") } extractor := newTimestampExtractor() - if err := extractor.Write(record); err != nil { + if err := extractor.Write(record.data); err != nil { return err } timestamp, err := extractor.Timestamp() @@ -347,13 +444,41 @@ func (w *repairWriter) writeRecord(record []byte) error { w.timestampRegressions++ } w.previousTimestamp = timestamp - if _, err := w.writer.Write(record); err != nil { + if _, err := w.writer.Write(record.data); err != nil { return err } if _, err := w.writer.Write([]byte{'\n'}); err != nil { return err } w.outputRecords++ + w.trackOutputConversation(record) + return nil +} + +func (w *repairWriter) trackSourceConversation(record []byte) { + if conversationRecordKind(record) == "" { + return + } + w.sourceConversationChain = w.sourceConversationChain.append(record) + w.sourceConversationRecords++ +} + +func (w *repairWriter) trackOutputConversation(record repairRecord) { + if conversationRecordKind(record.data) == "" { + return + } + if record.sourceValid { + w.preservedConversationChain = w.preservedConversationChain.append(record.data) + w.preservedConversationRecords++ + return + } + w.reconstructedConversationRecords++ +} + +func (w *repairWriter) verifyConversationIntegrity() error { + if w.sourceConversationRecords != w.preservedConversationRecords || w.sourceConversationChain != w.preservedConversationChain { + return fmt.Errorf("conversation integrity verification failed: source=%d preserved=%d", w.sourceConversationRecords, w.preservedConversationRecords) + } return nil } diff --git a/internal/reconcile/repair_test.go b/internal/reconcile/repair_test.go index 7ae7ae3..1d3e533 100644 --- a/internal/reconcile/repair_test.go +++ b/internal/reconcile/repair_test.go @@ -1,10 +1,13 @@ package reconcile import ( + "errors" "os" "path/filepath" "strings" "testing" + + "github.com/jstar0/codexfold/internal/storage" ) func TestRepairRestoresInterruptedRecordBeforeInsertedRecord(t *testing.T) { @@ -25,6 +28,9 @@ func TestRepairRestoresInterruptedRecordBeforeInsertedRecord(t *testing.T) { if result.InvalidPhysicalLines != 2 || result.ReconstructedRecords != 2 || result.OutputRecords != 2 { t.Fatalf("unexpected result: %#v", result) } + if result.Storage == nil || result.Storage.Budget.ProjectedPeakBytes <= 0 || result.Storage.ActualReclaimedBytes != 0 { + t.Fatalf("repair storage accounting is incomplete: %#v", result.Storage) + } data, err := os.ReadFile(output) if err != nil { t.Fatal(err) @@ -34,6 +40,25 @@ func TestRepairRestoresInterruptedRecordBeforeInsertedRecord(t *testing.T) { } } +func TestRepairBudgetRejectsBeforeCreatingOutput(t *testing.T) { + dir := t.TempDir() + input := filepath.Join(dir, "broken.jsonl") + if err := os.WriteFile(input, []byte(recordWithText("2026-07-13T01:00:00Z", "value")+"\n"), 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(dir, "output", "repaired.jsonl") + checker := &reconcileRejectingChecker{} + if _, err := RepairWithOptions(input, output, RepairOptions{Budget: checker}); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("RepairWithOptions error = %v, want storage budget rejection", err) + } + if checker.Calls != 1 || checker.Projection.Operation != "repair-rollout" { + t.Fatalf("unexpected repair budget projection: %#v", checker) + } + if _, err := os.Stat(filepath.Dir(output)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("repair output directory exists after preflight rejection: %v", err) + } +} + func TestRepairBuffersValidPhysicalRecordsWhileOuterRecordIsOpen(t *testing.T) { dir := t.TempDir() outer := recordWithText("2026-07-13T01:00:00Z", "abcdef") @@ -83,6 +108,80 @@ func TestRepairRestoresNestedInterruptions(t *testing.T) { } } +func TestRepairSalvageKeepsValidRecordsAfterUnfinishedFragment(t *testing.T) { + dir := t.TempDir() + before := recordWithText("2026-07-13T01:00:00Z", "before") + unfinished := `{"timestamp":"2026-07-13T01:00:01Z","type":"event_msg","payload":{"text":"unfinished` + afterOne := recordWithText("2026-07-13T01:00:02Z", "after-one") + afterTwo := recordWithText("2026-07-13T01:00:03Z", "after-two") + input := filepath.Join(dir, "broken.jsonl") + physical := before + "\n" + unfinished + "\n" + afterOne + "\n" + afterTwo + "\n" + if err := os.WriteFile(input, []byte(physical), 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(dir, "repaired.jsonl") + orphans := filepath.Join(dir, "orphans.bin") + + result, err := RepairWithOptions(input, output, RepairOptions{AllowOrphans: true, OrphanPath: orphans}) + if err != nil { + t.Fatal(err) + } + if result.OutputRecords != 3 || result.OrphanLines != 1 || result.OrphanBytes != int64(len(unfinished)) { + t.Fatalf("unexpected salvage result: %#v", result) + } + repaired, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if string(repaired) != before+"\n"+afterOne+"\n"+afterTwo+"\n" { + t.Fatalf("salvaged bytes:\n%s", repaired) + } + orphaned, err := os.ReadFile(orphans) + if err != nil { + t.Fatal(err) + } + if string(orphaned) != unfinished+"\n" { + t.Fatalf("orphan bytes: %q", orphaned) + } +} + +func TestRepairSalvageKeepsValidRecordsAcrossNestedUnfinishedFragments(t *testing.T) { + dir := t.TempDir() + outer := `{"timestamp":"2026-07-13T01:00:00Z","type":"event_msg","payload":{"text":"outer` + between := recordWithText("2026-07-13T01:00:01Z", "between") + inner := `{"timestamp":"2026-07-13T01:00:02Z","type":"event_msg","payload":{"text":"inner` + after := recordWithText("2026-07-13T01:00:03Z", "after") + input := filepath.Join(dir, "broken.jsonl") + physical := outer + "\n" + between + "\n" + inner + "\n" + after + "\n" + if err := os.WriteFile(input, []byte(physical), 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(dir, "repaired.jsonl") + orphans := filepath.Join(dir, "orphans.bin") + + result, err := RepairWithOptions(input, output, RepairOptions{AllowOrphans: true, OrphanPath: orphans}) + if err != nil { + t.Fatal(err) + } + if result.OutputRecords != 2 || result.OrphanLines != 2 { + t.Fatalf("unexpected nested salvage result: %#v", result) + } + repaired, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if string(repaired) != between+"\n"+after+"\n" { + t.Fatalf("nested salvaged bytes:\n%s", repaired) + } + orphaned, err := os.ReadFile(orphans) + if err != nil { + t.Fatal(err) + } + if string(orphaned) != outer+"\n"+inner+"\n" { + t.Fatalf("nested orphan bytes: %q", orphaned) + } +} + func recordWithText(timestamp, text string) string { return `{"timestamp":"` + timestamp + `","type":"event_msg","payload":{"text":"` + text + `"}}` } diff --git a/internal/reconcile/semantic.go b/internal/reconcile/semantic.go new file mode 100644 index 0000000..3a28c4f --- /dev/null +++ b/internal/reconcile/semantic.go @@ -0,0 +1,63 @@ +package reconcile + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "regexp" +) + +var conversationRecordStartPattern = regexp.MustCompile(`\{\s*"timestamp"\s*:`) + +type conversationChain [sha256.Size]byte + +func (chain conversationChain) append(record []byte) conversationChain { + recordDigest := sha256.Sum256(record) + var input [sha256.Size * 2]byte + copy(input[:sha256.Size], chain[:]) + copy(input[sha256.Size:], recordDigest[:]) + return sha256.Sum256(input[:]) +} + +func conversationRecordKind(record []byte) string { + if !bytes.Contains(record, []byte(`"type"`)) || !bytes.Contains(record, []byte(`"payload"`)) { + return "" + } + var envelope struct { + Timestamp json.RawMessage `json:"timestamp"` + Type string `json:"type"` + Payload json.RawMessage `json:"payload"` + } + if err := json.Unmarshal(record, &envelope); err != nil || len(envelope.Timestamp) == 0 { + return "" + } + var payload struct { + Type string `json:"type"` + Role string `json:"role"` + } + if err := json.Unmarshal(envelope.Payload, &payload); err != nil { + return "" + } + switch envelope.Type { + case "response_item": + if payload.Type == "message" && (payload.Role == "user" || payload.Role == "assistant") { + return envelope.Type + ":" + payload.Role + } + case "event_msg": + if payload.Type == "user_message" || payload.Type == "agent_message" { + return envelope.Type + ":" + payload.Type + } + } + return "" +} + +func containsCompleteConversationRecord(fragment []byte) bool { + for _, match := range conversationRecordStartPattern.FindAllIndex(fragment, -1) { + decoder := json.NewDecoder(bytes.NewReader(fragment[match[0]:])) + var record json.RawMessage + if err := decoder.Decode(&record); err == nil && conversationRecordKind(record) != "" { + return true + } + } + return false +} diff --git a/internal/reconcile/semantic_test.go b/internal/reconcile/semantic_test.go new file mode 100644 index 0000000..ddeb150 --- /dev/null +++ b/internal/reconcile/semantic_test.go @@ -0,0 +1,71 @@ +package reconcile + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRepairVerifiesConversationRecordsAcrossSalvage(t *testing.T) { + dir := t.TempDir() + user := conversationRecord("2026-07-16T00:00:00Z", "response_item", "message", "user") + unfinished := `{"timestamp":"2026-07-16T00:00:01Z","type":"event_msg","payload":{"text":"unfinished` + agent := conversationRecord("2026-07-16T00:00:02Z", "event_msg", "agent_message", "") + source := filepath.Join(dir, "source.jsonl") + if err := os.WriteFile(source, []byte(user+"\n"+unfinished+"\n"+agent+"\n"), 0o600); err != nil { + t.Fatal(err) + } + result, err := RepairWithOptions(source, filepath.Join(dir, "repaired.jsonl"), RepairOptions{ + AllowOrphans: true, + OrphanPath: filepath.Join(dir, "orphans.bin"), + }) + if err != nil { + t.Fatal(err) + } + if !result.ConversationIntegrityVerified || result.SourceConversationRecords != 2 || result.PreservedConversationRecords != 2 || result.ReconstructedConversationRecords != 0 { + t.Fatalf("unexpected conversation verification: %#v", result) + } +} + +func TestRepairCountsReconstructedConversationRecord(t *testing.T) { + dir := t.TempDir() + record := conversationRecord("2026-07-16T00:00:00Z", "response_item", "message", "assistant") + prefix, suffix := splitAt(t, record, `"role"`) + source := filepath.Join(dir, "source.jsonl") + if err := os.WriteFile(source, []byte(prefix+"\n"+suffix+"\n"), 0o600); err != nil { + t.Fatal(err) + } + result, err := Repair(source, filepath.Join(dir, "repaired.jsonl")) + if err != nil { + t.Fatal(err) + } + if !result.ConversationIntegrityVerified || result.SourceConversationRecords != 0 || result.PreservedConversationRecords != 0 || result.ReconstructedConversationRecords != 1 { + t.Fatalf("unexpected reconstructed conversation verification: %#v", result) + } +} + +func TestRepairRefusesCompleteConversationRecordInOrphan(t *testing.T) { + dir := t.TempDir() + record := conversationRecord("2026-07-16T00:00:00Z", "event_msg", "user_message", "") + source := filepath.Join(dir, "source.jsonl") + if err := os.WriteFile(source, []byte("garbage"+record+"trailing\n"), 0o600); err != nil { + t.Fatal(err) + } + _, err := RepairWithOptions(source, filepath.Join(dir, "repaired.jsonl"), RepairOptions{ + AllowOrphans: true, + OrphanPath: filepath.Join(dir, "orphans.bin"), + }) + if err == nil || !strings.Contains(err.Error(), "refusing to orphan") { + t.Fatalf("RepairWithOptions error = %v, want complete conversation refusal", err) + } +} + +func conversationRecord(timestamp, entryType, payloadType, role string) string { + payload := `{"type":"` + payloadType + `"` + if role != "" { + payload += `,"role":"` + role + `"` + } + payload += `}` + return `{"timestamp":"` + timestamp + `","type":"` + entryType + `","payload":` + payload + `}` +} diff --git a/internal/service/binary_update.go b/internal/service/binary_update.go new file mode 100644 index 0000000..4d3d371 --- /dev/null +++ b/internal/service/binary_update.go @@ -0,0 +1,168 @@ +package service + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/jstar0/codexfold/internal/buildid" +) + +type BinaryUpdate struct { + Target string `json:"target"` + Candidate string `json:"candidate"` + CurrentSHA256 string `json:"current_sha256"` + CandidateSHA256 string `json:"candidate_sha256"` + stagedPath string + backupPath string +} + +func StageBinaryUpdate(candidate string, target string) (*BinaryUpdate, error) { + if !filepath.IsAbs(candidate) || !filepath.IsAbs(target) { + return nil, errors.New("absolute candidate and target binary paths are required") + } + candidate = filepath.Clean(candidate) + target = filepath.Clean(target) + if candidate == target { + return nil, errors.New("candidate binary must be separate from the installed target") + } + targetInfo, err := os.Stat(target) + if err != nil { + return nil, err + } + if !targetInfo.Mode().IsRegular() { + return nil, errors.New("installed service binary is not a regular file") + } + candidateInfo, err := os.Stat(candidate) + if err != nil { + return nil, err + } + if !candidateInfo.Mode().IsRegular() || candidateInfo.Mode().Perm()&0o111 == 0 { + return nil, errors.New("candidate service binary must be a regular executable file") + } + currentSHA256, err := buildid.FileSHA256(target) + if err != nil { + return nil, err + } + candidateSHA256, err := buildid.FileSHA256(candidate) + if err != nil { + return nil, err + } + root := filepath.Dir(target) + stagedPath, err := copyBinaryTemporary(candidate, root, ".codexfold-candidate-*", targetInfo.Mode().Perm()) + if err != nil { + return nil, err + } + backupPath, err := copyBinaryTemporary(target, root, ".codexfold-backup-*", targetInfo.Mode().Perm()) + if err != nil { + _ = os.Remove(stagedPath) + return nil, err + } + if err := syncServiceDirectory(root); err != nil { + _ = os.Remove(stagedPath) + _ = os.Remove(backupPath) + return nil, err + } + return &BinaryUpdate{ + Target: target, Candidate: candidate, CurrentSHA256: currentSHA256, CandidateSHA256: candidateSHA256, + stagedPath: stagedPath, backupPath: backupPath, + }, nil +} + +func (u *BinaryUpdate) Promote() error { + if u == nil || u.stagedPath == "" || u.Target == "" { + return errors.New("staged binary update is required") + } + if err := replaceServiceBinary(u.stagedPath, u.Target); err != nil { + return err + } + u.stagedPath = "" + if err := syncServiceDirectory(filepath.Dir(u.Target)); err != nil { + return err + } + digest, err := buildid.FileSHA256(u.Target) + if err != nil { + return err + } + if digest != u.CandidateSHA256 { + return fmt.Errorf("promoted binary digest=%s expected=%s", digest, u.CandidateSHA256) + } + return nil +} + +func (u *BinaryUpdate) Rollback() error { + if u == nil || u.backupPath == "" || u.Target == "" { + return errors.New("binary update backup is unavailable") + } + if err := replaceServiceBinary(u.backupPath, u.Target); err != nil { + return err + } + u.backupPath = "" + if err := syncServiceDirectory(filepath.Dir(u.Target)); err != nil { + return err + } + digest, err := buildid.FileSHA256(u.Target) + if err != nil { + return err + } + if digest != u.CurrentSHA256 { + return fmt.Errorf("rolled back binary digest=%s expected=%s", digest, u.CurrentSHA256) + } + return nil +} + +func (u *BinaryUpdate) Commit() error { + if u == nil { + return nil + } + var result error + for _, path := range []string{u.stagedPath, u.backupPath} { + if path == "" { + continue + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + result = errors.Join(result, err) + } + } + u.stagedPath = "" + u.backupPath = "" + return errors.Join(result, syncServiceDirectory(filepath.Dir(u.Target))) +} + +func copyBinaryTemporary(source string, directory string, pattern string, mode os.FileMode) (string, error) { + input, err := os.Open(source) + if err != nil { + return "", err + } + temporary, err := os.CreateTemp(directory, pattern) + if err != nil { + _ = input.Close() + return "", err + } + path := temporary.Name() + cleanup := func(operationErr error) (string, error) { + _ = input.Close() + _ = temporary.Close() + _ = os.Remove(path) + return "", operationErr + } + if err := temporary.Chmod(mode); err != nil { + return cleanup(err) + } + if _, err := io.Copy(temporary, input); err != nil { + return cleanup(err) + } + if err := input.Close(); err != nil { + return cleanup(err) + } + if err := temporary.Sync(); err != nil { + return cleanup(err) + } + if err := temporary.Close(); err != nil { + _ = os.Remove(path) + return "", err + } + return path, nil +} diff --git a/internal/service/binary_update_test.go b/internal/service/binary_update_test.go new file mode 100644 index 0000000..b7f9b6f --- /dev/null +++ b/internal/service/binary_update_test.go @@ -0,0 +1,77 @@ +package service + +import ( + "os" + "path/filepath" + "testing" + + "github.com/jstar0/codexfold/internal/buildid" +) + +func TestBinaryUpdatePromotesAndCommitsAtomically(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "codexfold") + candidate := filepath.Join(root, "candidate") + if err := os.WriteFile(target, []byte("old-build"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(candidate, []byte("new-build"), 0o700); err != nil { + t.Fatal(err) + } + update, err := StageBinaryUpdate(candidate, target) + if err != nil { + t.Fatal(err) + } + if err := update.Promote(); err != nil { + t.Fatal(err) + } + if digest, err := buildid.FileSHA256(target); err != nil || digest != update.CandidateSHA256 { + t.Fatalf("promoted digest=%s err=%v", digest, err) + } + if err := update.Commit(); err != nil { + t.Fatal(err) + } + assertNoBinaryUpdateArtifacts(t, root) +} + +func TestBinaryUpdateRollsBackPromotedCandidate(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "codexfold") + candidate := filepath.Join(root, "candidate") + if err := os.WriteFile(target, []byte("old-build"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(candidate, []byte("new-build"), 0o700); err != nil { + t.Fatal(err) + } + update, err := StageBinaryUpdate(candidate, target) + if err != nil { + t.Fatal(err) + } + if err := update.Promote(); err != nil { + t.Fatal(err) + } + if err := update.Rollback(); err != nil { + t.Fatal(err) + } + if digest, err := buildid.FileSHA256(target); err != nil || digest != update.CurrentSHA256 { + t.Fatalf("rolled back digest=%s err=%v", digest, err) + } + if err := update.Commit(); err != nil { + t.Fatal(err) + } + assertNoBinaryUpdateArtifacts(t, root) +} + +func assertNoBinaryUpdateArtifacts(t *testing.T, root string) { + t.Helper() + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if len(entry.Name()) >= len(".codexfold-") && entry.Name()[:len(".codexfold-")] == ".codexfold-" { + t.Fatalf("binary update artifact remained: %s", entry.Name()) + } + } +} diff --git a/internal/service/binary_update_unix.go b/internal/service/binary_update_unix.go new file mode 100644 index 0000000..a8a05bb --- /dev/null +++ b/internal/service/binary_update_unix.go @@ -0,0 +1,22 @@ +//go:build !windows + +package service + +import ( + "errors" + "os" +) + +func replaceServiceBinary(source string, target string) error { + return os.Rename(source, target) +} + +func syncServiceDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + syncErr := directory.Sync() + closeErr := directory.Close() + return errors.Join(syncErr, closeErr) +} diff --git a/internal/service/binary_update_windows.go b/internal/service/binary_update_windows.go new file mode 100644 index 0000000..0f91149 --- /dev/null +++ b/internal/service/binary_update_windows.go @@ -0,0 +1,21 @@ +//go:build windows + +package service + +import ( + "golang.org/x/sys/windows" +) + +func replaceServiceBinary(source string, target string) error { + sourcePath, err := windows.UTF16PtrFromString(source) + if err != nil { + return err + } + targetPath, err := windows.UTF16PtrFromString(target) + if err != nil { + return err + } + return windows.MoveFileEx(sourcePath, targetPath, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH) +} + +func syncServiceDirectory(string) error { return nil } diff --git a/internal/service/build_status.go b/internal/service/build_status.go new file mode 100644 index 0000000..32a0960 --- /dev/null +++ b/internal/service/build_status.go @@ -0,0 +1,351 @@ +package service + +import ( + "encoding/xml" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/jstar0/codexfold/internal/buildid" + "github.com/jstar0/codexfold/internal/mountid" +) + +type BuildStatus struct { + Healthy bool `json:"healthy"` + RunningBuildSHA256 string `json:"running_build_sha256,omitempty"` + ConfiguredBinaryPath string `json:"configured_binary_path,omitempty"` + ConfiguredBuildSHA256 string `json:"configured_build_sha256,omitempty"` + Error string `json:"error,omitempty"` +} + +func InspectBuild(platform Platform, definitionPath string, mountPoint string) BuildStatus { + status := BuildStatus{} + binaryPath, err := DefinitionBinary(platform, definitionPath) + if err != nil { + status.Error = err.Error() + return status + } + status.ConfiguredBinaryPath = binaryPath + status.ConfiguredBuildSHA256, err = buildid.FileSHA256(binaryPath) + if err != nil { + status.Error = err.Error() + return status + } + identityBytes, err := os.ReadFile(filepath.Join(mountPoint, mountid.Path)) + if err != nil { + status.Error = fmt.Sprintf("read running mount build identity: %v", err) + return status + } + identity, err := mountid.Parse(identityBytes) + if err != nil { + status.Error = err.Error() + return status + } + status.RunningBuildSHA256 = identity.BuildSHA256 + if status.RunningBuildSHA256 == "" { + status.Error = "running mount identity does not include a build SHA-256" + return status + } + if status.RunningBuildSHA256 != status.ConfiguredBuildSHA256 { + status.Error = "running daemon build does not match the configured binary on disk" + return status + } + status.Healthy = true + return status +} + +func DefinitionBinary(platform Platform, definitionPath string) (string, error) { + if !filepath.IsAbs(definitionPath) { + return "", errors.New("absolute service definition path is required") + } + definition, err := os.ReadFile(filepath.Clean(definitionPath)) + if err != nil { + return "", err + } + var binary string + switch platform { + case PlatformLaunchd: + binary, err = launchdDefinitionBinary(definition) + case PlatformSystemd: + binary, err = systemdDefinitionBinary(definition) + case PlatformWindows: + var config WindowsConfig + config, err = ParseWindowsConfig(definition) + binary = config.BinaryPath + default: + err = errors.New("unknown service platform") + } + if err != nil { + return "", err + } + if !filepath.IsAbs(binary) && !absoluteWindowsServicePath(binary) { + return "", errors.New("configured service binary path is not absolute") + } + return filepath.Clean(binary), nil +} + +func DefinitionLauncher(platform Platform, definitionPath string) (string, error) { + if !filepath.IsAbs(definitionPath) { + return "", errors.New("absolute service definition path is required") + } + if platform != PlatformLaunchd { + return "", nil + } + definition, err := os.ReadFile(filepath.Clean(definitionPath)) + if err != nil { + return "", err + } + arguments, err := launchdDefinitionArguments(definition) + if err != nil { + return "", err + } + if len(arguments) < 3 || arguments[1] != "--run-helper" { + return "", nil + } + if !filepath.IsAbs(arguments[0]) { + return "", errors.New("configured service launcher path is not absolute") + } + return filepath.Clean(arguments[0]), nil +} + +func DefinitionFrontend(platform Platform, definitionPath string) (string, error) { + if !filepath.IsAbs(definitionPath) { + return "", errors.New("absolute service definition path is required") + } + definition, err := os.ReadFile(filepath.Clean(definitionPath)) + if err != nil { + return "", err + } + if platform != PlatformLaunchd { + return "fuse", nil + } + arguments, err := launchdDefinitionArguments(definition) + if err != nil { + return "", err + } + for index := 0; index < len(arguments); index++ { + if arguments[index] != "--frontend" { + continue + } + if index+1 >= len(arguments) { + return "", errors.New("launchd definition has an incomplete --frontend argument") + } + if arguments[index+1] != "fuse" && arguments[index+1] != "native-fskit" { + return "", fmt.Errorf("launchd definition has unsupported frontend %q", arguments[index+1]) + } + return arguments[index+1], nil + } + return "fuse", nil +} + +func DefinitionFSKitResource(platform Platform, definitionPath string) (string, error) { + frontend, err := DefinitionFrontend(platform, definitionPath) + if err != nil { + return "", err + } + if frontend != "native-fskit" { + return "", nil + } + definition, err := os.ReadFile(filepath.Clean(definitionPath)) + if err != nil { + return "", err + } + arguments, err := launchdDefinitionArguments(definition) + if err != nil { + return "", err + } + for index := 0; index < len(arguments); index++ { + if arguments[index] != "--fskit-resource" { + continue + } + if index+1 >= len(arguments) || !filepath.IsAbs(arguments[index+1]) { + return "", errors.New("launchd definition has an invalid --fskit-resource argument") + } + return filepath.Clean(arguments[index+1]), nil + } + return "", errors.New("native-fskit launchd definition has no --fskit-resource argument") +} + +func DefinitionStore(platform Platform, definitionPath string) (string, error) { + if !filepath.IsAbs(definitionPath) { + return "", errors.New("absolute service definition path is required") + } + if platform != PlatformLaunchd { + return "", errors.New("service store inspection is currently available only for launchd definitions") + } + definition, err := os.ReadFile(filepath.Clean(definitionPath)) + if err != nil { + return "", err + } + arguments, err := launchdDefinitionArguments(definition) + if err != nil { + return "", err + } + for index := 0; index < len(arguments); index++ { + if arguments[index] != "--store" { + continue + } + if index+1 >= len(arguments) || !filepath.IsAbs(arguments[index+1]) { + return "", errors.New("launchd definition has an invalid --store argument") + } + return filepath.Clean(arguments[index+1]), nil + } + return "", errors.New("launchd definition has no --store argument") +} + +func DefinitionLabel(platform Platform, definitionPath string) (string, error) { + if !filepath.IsAbs(definitionPath) { + return "", errors.New("absolute service definition path is required") + } + definition, err := os.ReadFile(filepath.Clean(definitionPath)) + if err != nil { + return "", err + } + var label string + switch platform { + case PlatformLaunchd: + label, err = launchdDefinitionLabel(definition) + case PlatformSystemd: + label = strings.TrimSuffix(filepath.Base(definitionPath), ".service") + case PlatformWindows: + var config WindowsConfig + config, err = ParseWindowsConfig(definition) + label = config.ServiceName + default: + err = errors.New("unknown service platform") + } + if err != nil { + return "", err + } + if !safeLabel(label) { + return "", errors.New("configured service label is invalid") + } + return label, nil +} + +func launchdDefinitionBinary(definition []byte) (string, error) { + arguments, err := launchdDefinitionArguments(definition) + if err != nil { + return "", err + } + if len(arguments) == 0 { + return "", errors.New("launchd definition has no ProgramArguments binary") + } + if len(arguments) >= 3 && arguments[1] == "--run-helper" { + return arguments[2], nil + } + return arguments[0], nil +} + +func launchdDefinitionArguments(definition []byte) ([]string, error) { + decoder := xml.NewDecoder(strings.NewReader(string(definition))) + wantArguments := false + inArguments := false + arguments := make([]string, 0, 16) + for { + token, err := decoder.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, err + } + switch element := token.(type) { + case xml.StartElement: + switch element.Name.Local { + case "key": + var key string + if err := decoder.DecodeElement(&key, &element); err != nil { + return nil, err + } + wantArguments = key == "ProgramArguments" + case "array": + if wantArguments { + inArguments = true + wantArguments = false + } + case "string": + if inArguments { + var argument string + if err := decoder.DecodeElement(&argument, &element); err != nil { + return nil, err + } + arguments = append(arguments, argument) + } + } + case xml.EndElement: + if element.Name.Local == "array" && inArguments { + return arguments, nil + } + } + } + return nil, errors.New("launchd definition has no ProgramArguments array") +} + +func launchdDefinitionLabel(definition []byte) (string, error) { + decoder := xml.NewDecoder(strings.NewReader(string(definition))) + wantLabel := false + for { + token, err := decoder.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return "", err + } + start, ok := token.(xml.StartElement) + if !ok { + continue + } + switch start.Name.Local { + case "key": + var key string + if err := decoder.DecodeElement(&key, &start); err != nil { + return "", err + } + wantLabel = key == "Label" + case "string": + if wantLabel { + var label string + if err := decoder.DecodeElement(&label, &start); err != nil { + return "", err + } + return label, nil + } + } + } + return "", errors.New("launchd definition has no Label") +} + +func systemdDefinitionBinary(definition []byte) (string, error) { + for _, line := range strings.Split(string(definition), "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "ExecStart=:") { + continue + } + value := strings.TrimSpace(strings.TrimPrefix(line, "ExecStart=:")) + if len(value) < 2 || value[0] != '"' { + return "", errors.New("systemd ExecStart binary is not quoted") + } + var binary strings.Builder + for index := 1; index < len(value); index++ { + switch value[index] { + case '"': + return strings.ReplaceAll(binary.String(), "%%", "%"), nil + case '\\': + index++ + if index >= len(value) { + return "", errors.New("systemd ExecStart binary has an incomplete escape") + } + binary.WriteByte(value[index]) + default: + binary.WriteByte(value[index]) + } + } + return "", errors.New("systemd ExecStart binary is missing its closing quote") + } + return "", errors.New("systemd definition has no ExecStart binary") +} diff --git a/internal/service/build_status_test.go b/internal/service/build_status_test.go new file mode 100644 index 0000000..658e0c0 --- /dev/null +++ b/internal/service/build_status_test.go @@ -0,0 +1,154 @@ +package service + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/jstar0/codexfold/internal/buildid" + "github.com/jstar0/codexfold/internal/mountid" +) + +func TestInspectBuildMatchesRunningMountAndConfiguredBinary(t *testing.T) { + root := t.TempDir() + binary := filepath.Join(root, "codexfold") + if err := os.WriteFile(binary, []byte("candidate-binary"), 0o700); err != nil { + t.Fatal(err) + } + definition := filepath.Join(root, "com.codexfold.fs.plist") + plist, err := RenderLaunchd(Options{ + Label: "com.codexfold.fs", BinaryPath: binary, CodexHome: filepath.Join(root, "home"), + StoreDir: filepath.Join(root, "store"), MountPoint: filepath.Join(root, "mount"), + StdoutPath: filepath.Join(root, "stdout.log"), StderrPath: filepath.Join(root, "stderr.log"), + }) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(definition, plist, 0o600); err != nil { + t.Fatal(err) + } + mount := filepath.Join(root, "mount") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + digest, err := buildid.FileSHA256(binary) + if err != nil { + t.Fatal(err) + } + identity, err := mountid.New(digest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(mount, mountid.Path), []byte(identity), 0o600); err != nil { + t.Fatal(err) + } + status := InspectBuild(PlatformLaunchd, definition, mount) + if !status.Healthy || status.RunningBuildSHA256 != digest || status.ConfiguredBuildSHA256 != digest || status.ConfiguredBinaryPath != binary { + t.Fatalf("build status = %#v", status) + } +} + +func TestInspectBuildRejectsStaleRunningDaemon(t *testing.T) { + root := t.TempDir() + binary := filepath.Join(root, "codexfold") + if err := os.WriteFile(binary, []byte("new-binary"), 0o700); err != nil { + t.Fatal(err) + } + definition := filepath.Join(root, "com.codexfold.fs.plist") + plist, err := RenderLaunchd(Options{ + Label: "com.codexfold.fs", BinaryPath: binary, CodexHome: filepath.Join(root, "home"), + StoreDir: filepath.Join(root, "store"), MountPoint: filepath.Join(root, "mount"), + StdoutPath: filepath.Join(root, "stdout.log"), StderrPath: filepath.Join(root, "stderr.log"), + }) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(definition, plist, 0o600); err != nil { + t.Fatal(err) + } + mount := filepath.Join(root, "mount") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + identity, err := mountid.New(strings.Repeat("a", 64)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(mount, mountid.Path), []byte(identity), 0o600); err != nil { + t.Fatal(err) + } + status := InspectBuild(PlatformLaunchd, definition, mount) + if status.Healthy || !strings.Contains(status.Error, "does not match") { + t.Fatalf("stale build status = %#v", status) + } +} + +func TestDefinitionBinaryParsesEveryRenderedPlatform(t *testing.T) { + root := t.TempDir() + options := Options{ + Label: "com.codexfold.fs", BinaryPath: filepath.Join(root, "Codex Fold"), + CodexHome: filepath.Join(root, "home"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "stdout.log"), + StderrPath: filepath.Join(root, "stderr.log"), + } + for _, platform := range []Platform{PlatformLaunchd, PlatformSystemd, PlatformWindows} { + definition, err := RenderDefinition(platform, options) + if err != nil { + t.Fatalf("render %s: %v", platform, err) + } + path := filepath.Join(root, string(platform)+".definition") + if err := os.WriteFile(path, definition, 0o600); err != nil { + t.Fatal(err) + } + binary, err := DefinitionBinary(platform, path) + if err != nil || binary != options.BinaryPath { + t.Fatalf("definition binary %s = %q err=%v", platform, binary, err) + } + } +} + +func TestDefinitionFrontendParsesNativeFSKitLaunchdArguments(t *testing.T) { + root := t.TempDir() + resource := filepath.Join(root, "store", "fs", "native-fskit.resource") + launcher := filepath.Join(root, "CodexFoldFSKit.app", "Contents", "MacOS", "CodexFoldFSKit") + binary := filepath.Join(root, "codexfold") + definition, err := RenderLaunchd(Options{ + Label: "com.codexfold.fs", BinaryPath: binary, LauncherPath: launcher, + CodexHome: filepath.Join(root, "home"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "stdout.log"), + StderrPath: filepath.Join(root, "stderr.log"), CanonicalNamespace: true, + NativeRoot: filepath.Join(root, "native"), Frontend: "native-fskit", FSKitResource: resource, + }) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(root, "com.codexfold.fs.plist") + if err := os.WriteFile(path, definition, 0o600); err != nil { + t.Fatal(err) + } + frontend, err := DefinitionFrontend(PlatformLaunchd, path) + if err != nil || frontend != "native-fskit" { + t.Fatalf("frontend = %q err=%v", frontend, err) + } + gotResource, err := DefinitionFSKitResource(PlatformLaunchd, path) + if err != nil || gotResource != resource { + t.Fatalf("resource = %q err=%v", gotResource, err) + } + gotStore, err := DefinitionStore(PlatformLaunchd, path) + if err != nil || gotStore != filepath.Join(root, "store") { + t.Fatalf("store = %q err=%v", gotStore, err) + } + label, err := DefinitionLabel(PlatformLaunchd, path) + if err != nil || label != "com.codexfold.fs" { + t.Fatalf("label = %q err=%v", label, err) + } + configuredBinary, err := DefinitionBinary(PlatformLaunchd, path) + if err != nil || configuredBinary != binary { + t.Fatalf("wrapped definition binary = %q err=%v", configuredBinary, err) + } + configuredLauncher, err := DefinitionLauncher(PlatformLaunchd, path) + if err != nil || configuredLauncher != launcher { + t.Fatalf("wrapped definition launcher = %q err=%v", configuredLauncher, err) + } +} diff --git a/internal/service/definition_update.go b/internal/service/definition_update.go new file mode 100644 index 0000000..da18ec0 --- /dev/null +++ b/internal/service/definition_update.go @@ -0,0 +1,150 @@ +package service + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/jstar0/codexfold/internal/buildid" +) + +type DefinitionUpdate struct { + Target string `json:"target"` + CurrentSHA256 string `json:"current_sha256,omitempty"` + CandidateSHA256 string `json:"candidate_sha256"` + HadTarget bool `json:"had_target"` + stagedPath string + backupPath string +} + +func StageDefinitionUpdate(target string, definition []byte) (*DefinitionUpdate, error) { + if !filepath.IsAbs(target) || len(definition) == 0 { + return nil, errors.New("absolute definition target and non-empty content are required") + } + target = filepath.Clean(target) + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + return nil, err + } + temporary, err := os.CreateTemp(filepath.Dir(target), ".codexfold-definition-candidate-*") + if err != nil { + return nil, err + } + stagedPath := temporary.Name() + cleanup := func(operationErr error) (*DefinitionUpdate, error) { + _ = temporary.Close() + _ = os.Remove(stagedPath) + return nil, operationErr + } + if err := temporary.Chmod(0o600); err != nil { + return cleanup(err) + } + if _, err := temporary.Write(definition); err != nil { + return cleanup(err) + } + if err := temporary.Sync(); err != nil { + return cleanup(err) + } + if err := temporary.Close(); err != nil { + _ = os.Remove(stagedPath) + return nil, err + } + digest := sha256.Sum256(definition) + update := &DefinitionUpdate{Target: target, CandidateSHA256: hex.EncodeToString(digest[:]), stagedPath: stagedPath} + if info, err := os.Stat(target); err == nil { + if !info.Mode().IsRegular() { + _ = update.Commit() + return nil, errors.New("installed service definition is not a regular file") + } + update.HadTarget = true + update.CurrentSHA256, err = buildid.FileSHA256(target) + if err != nil { + _ = update.Commit() + return nil, err + } + update.backupPath, err = copyBinaryTemporary(target, filepath.Dir(target), ".codexfold-definition-backup-*", info.Mode().Perm()) + if err != nil { + _ = update.Commit() + return nil, err + } + } else if !errors.Is(err, os.ErrNotExist) { + _ = update.Commit() + return nil, err + } + if err := syncServiceDirectory(filepath.Dir(target)); err != nil { + _ = update.Commit() + return nil, err + } + return update, nil +} + +func (u *DefinitionUpdate) Promote() error { + if u == nil || u.Target == "" || u.stagedPath == "" { + return errors.New("staged definition update is required") + } + if err := replaceServiceBinary(u.stagedPath, u.Target); err != nil { + return err + } + u.stagedPath = "" + if err := syncServiceDirectory(filepath.Dir(u.Target)); err != nil { + return err + } + digest, err := buildid.FileSHA256(u.Target) + if err != nil { + return err + } + if digest != u.CandidateSHA256 { + return fmt.Errorf("promoted definition digest=%s expected=%s", digest, u.CandidateSHA256) + } + return nil +} + +func (u *DefinitionUpdate) Rollback() error { + if u == nil || u.Target == "" { + return nil + } + if u.HadTarget { + if u.backupPath == "" { + return errors.New("definition rollback backup is unavailable") + } + if err := replaceServiceBinary(u.backupPath, u.Target); err != nil { + return err + } + u.backupPath = "" + } else if err := os.Remove(u.Target); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := syncServiceDirectory(filepath.Dir(u.Target)); err != nil { + return err + } + if u.HadTarget { + digest, err := buildid.FileSHA256(u.Target) + if err != nil { + return err + } + if digest != u.CurrentSHA256 { + return fmt.Errorf("rolled back definition digest=%s expected=%s", digest, u.CurrentSHA256) + } + } + return nil +} + +func (u *DefinitionUpdate) Commit() error { + if u == nil { + return nil + } + var result error + for _, path := range []string{u.stagedPath, u.backupPath} { + if path == "" { + continue + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + result = errors.Join(result, err) + } + } + u.stagedPath = "" + u.backupPath = "" + return errors.Join(result, syncServiceDirectory(filepath.Dir(u.Target))) +} diff --git a/internal/service/definition_update_test.go b/internal/service/definition_update_test.go new file mode 100644 index 0000000..e32acd7 --- /dev/null +++ b/internal/service/definition_update_test.go @@ -0,0 +1,88 @@ +package service + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestDefinitionUpdatePromotesAndCommitsExistingDefinition(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "service.plist") + if err := os.WriteFile(target, []byte("old-definition"), 0o600); err != nil { + t.Fatal(err) + } + update, err := StageDefinitionUpdate(target, []byte("new-definition")) + if err != nil { + t.Fatal(err) + } + if err := update.Promote(); err != nil { + t.Fatal(err) + } + if data, err := os.ReadFile(target); err != nil || string(data) != "new-definition" { + t.Fatalf("promoted definition=%q err=%v", data, err) + } + if err := update.Commit(); err != nil { + t.Fatal(err) + } + assertNoDefinitionUpdateArtifacts(t, root) +} + +func TestDefinitionUpdateRollsBackExistingAndNewDefinitions(t *testing.T) { + root := t.TempDir() + for _, test := range []struct { + name string + old []byte + }{ + {name: "existing", old: []byte("old-definition")}, + {name: "new"}, + } { + t.Run(test.name, func(t *testing.T) { + dir := filepath.Join(root, test.name) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(dir, "service.plist") + if test.old != nil { + if err := os.WriteFile(target, test.old, 0o600); err != nil { + t.Fatal(err) + } + } + update, err := StageDefinitionUpdate(target, []byte("new-definition")) + if err != nil { + t.Fatal(err) + } + if err := update.Promote(); err != nil { + t.Fatal(err) + } + if err := update.Rollback(); err != nil { + t.Fatal(err) + } + if test.old == nil { + if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("new definition remained after rollback: %v", err) + } + } else if data, err := os.ReadFile(target); err != nil || string(data) != string(test.old) { + t.Fatalf("rolled back definition=%q err=%v", data, err) + } + if err := update.Commit(); err != nil { + t.Fatal(err) + } + assertNoDefinitionUpdateArtifacts(t, dir) + }) + } +} + +func assertNoDefinitionUpdateArtifacts(t *testing.T, root string) { + t.Helper() + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if len(entry.Name()) >= len(".codexfold-definition-") && entry.Name()[:len(".codexfold-definition-")] == ".codexfold-definition-" { + t.Fatalf("definition update artifact remained: %s", entry.Name()) + } + } +} diff --git a/internal/service/fskit_app.go b/internal/service/fskit_app.go new file mode 100644 index 0000000..2bac3f6 --- /dev/null +++ b/internal/service/fskit_app.go @@ -0,0 +1,44 @@ +package service + +import ( + "errors" + "path/filepath" + "strings" +) + +const ( + FSKitAppBundleName = "CodexFoldFSKit.app" + FSKitHostExecutableName = "CodexFoldFSKit" + FSKitHostBundleIdentifier = "vip.jstar.codexfold.fskitprofileprobe" + FSKitModuleBundleName = "CodexFoldFSKitModule.appex" + FSKitModuleIdentifier = "vip.jstar.codexfold.fskitprofileprobe.module" + FSKitAppGroupIdentifier = "group.vip.jstar.codexfold" + FSKitResourceDirectoryName = "native-fskit" +) + +func DefaultFSKitAppPath(userHome string) string { + return filepath.Join(filepath.Clean(userHome), "Applications", FSKitAppBundleName) +} + +func FSKitHostLauncherPath(appPath string) (string, error) { + if !filepath.IsAbs(appPath) { + return "", errors.New("FSKit app path must be absolute") + } + appPath = filepath.Clean(appPath) + if !strings.HasSuffix(filepath.Base(appPath), ".app") { + return "", errors.New("FSKit app path must identify an app bundle") + } + return filepath.Join(appPath, "Contents", "MacOS", FSKitHostExecutableName), nil +} + +func FSKitModulePath(appPath string) (string, error) { + launcher, err := FSKitHostLauncherPath(appPath) + if err != nil { + return "", err + } + return filepath.Join(filepath.Dir(filepath.Dir(launcher)), "Extensions", FSKitModuleBundleName), nil +} + +func DefaultFSKitResourcePath(userHome string) string { + return filepath.Join(filepath.Clean(userHome), "Library", "Group Containers", FSKitAppGroupIdentifier, FSKitResourceDirectoryName) +} diff --git a/internal/service/fskit_app_test.go b/internal/service/fskit_app_test.go new file mode 100644 index 0000000..ad28e21 --- /dev/null +++ b/internal/service/fskit_app_test.go @@ -0,0 +1,33 @@ +package service + +import ( + "path/filepath" + "testing" +) + +func TestFSKitManagedPathsUseStableAppAndAppGroupLocations(t *testing.T) { + home := filepath.Join(t.TempDir(), "user") + app := DefaultFSKitAppPath(home) + if app != filepath.Join(home, "Applications", FSKitAppBundleName) { + t.Fatalf("default app path = %q", app) + } + launcher, err := FSKitHostLauncherPath(app) + if err != nil { + t.Fatal(err) + } + if launcher != filepath.Join(app, "Contents", "MacOS", FSKitHostExecutableName) { + t.Fatalf("launcher path = %q", launcher) + } + resource := DefaultFSKitResourcePath(home) + if resource != filepath.Join(home, "Library", "Group Containers", FSKitAppGroupIdentifier, FSKitResourceDirectoryName) { + t.Fatalf("resource path = %q", resource) + } +} + +func TestFSKitHostLauncherRejectsNonAppAndRelativePaths(t *testing.T) { + for _, path := range []string{"CodexFoldFSKit.app", filepath.Join(t.TempDir(), "CodexFoldFSKit")} { + if _, err := FSKitHostLauncherPath(path); err == nil { + t.Fatalf("FSKitHostLauncherPath(%q) succeeded", path) + } + } +} diff --git a/internal/service/mount_probe_darwin.go b/internal/service/mount_probe_darwin.go index 7865037..f573d55 100644 --- a/internal/service/mount_probe_darwin.go +++ b/internal/service/mount_probe_darwin.go @@ -26,10 +26,8 @@ func defaultMountProbe(path string) error { if actualPath != requestedPath { return errors.New("path is not a mount root") } - macFUSE := strings.Contains(filesystem, "fuse") - fuseT := filesystem == "nfs" && strings.HasPrefix(mountedFrom, "fuse-t:") - if !macFUSE && !fuseT { - return errors.New("mount root is not backed by a supported FUSE provider") + if !validDarwinMountProvider(filesystem, mountedFrom) { + return errors.New("mount root is not backed by CodexFold native FSKit or the supported FUSE-T fallback") } value, err := os.ReadFile(filepath.Join(path, mountid.Path)) if err != nil { @@ -44,6 +42,12 @@ func defaultMountProbe(path string) error { return nil } +func validDarwinMountProvider(filesystem string, mountedFrom string) bool { + filesystem = strings.ToLower(strings.TrimSpace(filesystem)) + mountedFrom = strings.ToLower(strings.TrimSpace(mountedFrom)) + return filesystem == "codexfold" || filesystem == "nfs" && strings.HasPrefix(mountedFrom, "fuse-t:") +} + func canonicalMountPath(path string) string { resolved, err := filepath.EvalSymlinks(path) if err == nil { diff --git a/internal/service/mount_probe_darwin_test.go b/internal/service/mount_probe_darwin_test.go new file mode 100644 index 0000000..cb2a31f --- /dev/null +++ b/internal/service/mount_probe_darwin_test.go @@ -0,0 +1,24 @@ +//go:build darwin + +package service + +import "testing" + +func TestValidDarwinMountProviderAcceptsNativeFSKitAndFallbackOnly(t *testing.T) { + tests := []struct { + filesystem string + mountedFrom string + want bool + }{ + {filesystem: "codexfold", mountedFrom: "file:///private/tmp/resource.bin", want: true}, + {filesystem: "CODEXFOLD", mountedFrom: "FILE:///private/tmp/resource.bin", want: true}, + {filesystem: "nfs", mountedFrom: "fuse-t:/private/tmp/resource", want: true}, + {filesystem: "nfs", mountedFrom: "server:/export", want: false}, + {filesystem: "apfs", mountedFrom: "/dev/disk1s1", want: false}, + } + for _, test := range tests { + if got := validDarwinMountProvider(test.filesystem, test.mountedFrom); got != test.want { + t.Errorf("provider filesystem=%q mountedFrom=%q = %t, want %t", test.filesystem, test.mountedFrom, got, test.want) + } + } +} diff --git a/internal/service/mount_probe_linux.go b/internal/service/mount_probe_linux.go new file mode 100644 index 0000000..cf27ac0 --- /dev/null +++ b/internal/service/mount_probe_linux.go @@ -0,0 +1,54 @@ +//go:build linux + +package service + +import ( + "errors" + "os" + "path/filepath" + "strings" + + "github.com/jstar0/codexfold/internal/mountid" +) + +func defaultMountProbe(path string) error { + data, err := os.ReadFile("/proc/self/mountinfo") + if err != nil { + return err + } + want := filepath.Clean(path) + fuseMount := false + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) < 7 || filepath.Clean(unescapeLinuxMountField(fields[4])) != want { + continue + } + separator := -1 + for index := 6; index < len(fields); index++ { + if fields[index] == "-" { + separator = index + break + } + } + if separator >= 0 && separator+2 < len(fields) && strings.HasPrefix(fields[separator+1], "fuse") && strings.Contains(strings.ToLower(unescapeLinuxMountField(fields[separator+2])), "codexfold") { + fuseMount = true + } + break + } + if !fuseMount { + return errors.New("path is not a CodexFold FUSE mount root") + } + return validateMountIdentity(path) +} + +func unescapeLinuxMountField(value string) string { + return strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`).Replace(value) +} + +func validateMountIdentity(path string) error { + value, err := os.ReadFile(filepath.Join(path, mountid.Path)) + if err != nil { + return err + } + return mountid.Validate(value) +} diff --git a/internal/service/mount_probe_other.go b/internal/service/mount_probe_other.go index 51f5d13..c70649c 100644 --- a/internal/service/mount_probe_other.go +++ b/internal/service/mount_probe_other.go @@ -1,4 +1,4 @@ -//go:build !darwin +//go:build !darwin && !linux && !windows package service diff --git a/internal/service/mount_probe_windows.go b/internal/service/mount_probe_windows.go new file mode 100644 index 0000000..2baf48f --- /dev/null +++ b/internal/service/mount_probe_windows.go @@ -0,0 +1,18 @@ +//go:build windows + +package service + +import ( + "os" + "path/filepath" + + "github.com/jstar0/codexfold/internal/mountid" +) + +func defaultMountProbe(path string) error { + value, err := os.ReadFile(filepath.Join(path, mountid.Path)) + if err != nil { + return err + } + return mountid.Validate(value) +} diff --git a/internal/service/native_fskit_operations_darwin.go b/internal/service/native_fskit_operations_darwin.go new file mode 100644 index 0000000..979f646 --- /dev/null +++ b/internal/service/native_fskit_operations_darwin.go @@ -0,0 +1,100 @@ +//go:build darwin + +package service + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/jstar0/codexfold/internal/fskitproto" + "github.com/jstar0/codexfold/internal/mountid" + "golang.org/x/sys/unix" +) + +type nativeFSKitOperations struct{} + +func defaultNativeFSKitOperations() (NativeFSKitOperations, error) { + return nativeFSKitOperations{}, nil +} + +func (nativeFSKitOperations) DaemonHealthy(ctx context.Context, resourcePath string) error { + if err := ctx.Err(); err != nil { + return err + } + client, err := fskitproto.DialResource(resourcePath, 2*time.Second) + if err != nil { + return err + } + defer client.Close() + _, err = client.Call(fskitproto.OpPing, nil) + return err +} + +func (nativeFSKitOperations) MountState(ctx context.Context, mountPoint string, timeout time.Duration) (NativeFSKitMountState, error) { + var stat unix.Statfs_t + if err := unix.Statfs(mountPoint, &stat); err != nil { + return NativeFSKitMountState{}, err + } + requested := canonicalMountPath(mountPoint) + actual := canonicalMountPath(unix.ByteSliceToString(stat.Mntonname[:])) + if requested != actual { + return NativeFSKitMountState{}, nil + } + filesystem := strings.ToLower(unix.ByteSliceToString(stat.Fstypename[:])) + state := NativeFSKitMountState{Mounted: true, Owned: filesystem == "codexfold"} + if !state.Owned { + return state, nil + } + if timeout <= 0 { + timeout = 2 * time.Second + } + probeCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + for _, directory := range []string{"sessions", "archived_sessions"} { + command := exec.CommandContext(probeCtx, "/usr/bin/stat", "-f", "%HT", filepath.Join(mountPoint, directory)) + if output, err := command.CombinedOutput(); err != nil { + return state, fmt.Errorf("probe native FSKit directory %s: %w: %s", directory, err, strings.TrimSpace(string(output))) + } + } + identity, err := os.ReadFile(filepath.Join(mountPoint, mountid.Path)) + if err != nil { + return state, fmt.Errorf("read native FSKit mount identity: %w", err) + } + if err := mountid.Validate(identity); err != nil { + return state, fmt.Errorf("validate native FSKit mount identity: %w", err) + } + state.Healthy = true + return state, nil +} + +func (nativeFSKitOperations) Mount(ctx context.Context, resourcePath string, mountPoint string) error { + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + return err + } + output, err := exec.CommandContext(ctx, "/sbin/mount", "-t", "codexfoldnative", resourcePath, mountPoint).CombinedOutput() + if err != nil { + return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(output))) + } + return nil +} + +func (nativeFSKitOperations) Unmount(ctx context.Context, mountPoint string, force bool) error { + arguments := []string{mountPoint} + if force { + arguments = []string{"-f", mountPoint} + } + output, err := exec.CommandContext(ctx, "/sbin/umount", arguments...).CombinedOutput() + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(output))) + } + return nil +} diff --git a/internal/service/native_fskit_operations_other.go b/internal/service/native_fskit_operations_other.go new file mode 100644 index 0000000..bdf1f8b --- /dev/null +++ b/internal/service/native_fskit_operations_other.go @@ -0,0 +1,11 @@ +//go:build !darwin + +package service + +import ( + "errors" +) + +func defaultNativeFSKitOperations() (NativeFSKitOperations, error) { + return nil, errors.New("native FSKit supervision is available only on macOS") +} diff --git a/internal/service/native_fskit_supervisor.go b/internal/service/native_fskit_supervisor.go new file mode 100644 index 0000000..4827258 --- /dev/null +++ b/internal/service/native_fskit_supervisor.go @@ -0,0 +1,160 @@ +package service + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "time" +) + +var ErrForeignMount = errors.New("mount point is occupied by a foreign filesystem") + +const NativeFSKitSupervisorLockName = "supervisor.lock" + +type NativeFSKitMountState struct { + Mounted bool + Owned bool + Healthy bool +} + +type NativeFSKitOperations interface { + DaemonHealthy(context.Context, string) error + MountState(context.Context, string, time.Duration) (NativeFSKitMountState, error) + Mount(context.Context, string, string) error + Unmount(context.Context, string, bool) error +} + +type NativeFSKitSupervisorOptions struct { + ResourcePath string + MountPoint string + Interval time.Duration + ProbeTimeout time.Duration + RecoveryTimeout time.Duration + Operations NativeFSKitOperations + Event func(string) +} + +func RunNativeFSKitSupervisor(ctx context.Context, options NativeFSKitSupervisorOptions) error { + if !filepath.IsAbs(options.ResourcePath) || !filepath.IsAbs(options.MountPoint) { + return errors.New("absolute FSKit resource and mount paths are required") + } + if options.Interval <= 0 { + options.Interval = time.Second + } + if options.ProbeTimeout <= 0 { + options.ProbeTimeout = 2 * time.Second + } + if options.RecoveryTimeout <= 0 { + options.RecoveryTimeout = 15 * time.Second + } + operations := options.Operations + if operations == nil { + var err error + operations, err = defaultNativeFSKitOperations() + if err != nil { + return err + } + } + state := nativeFSKitSupervisorState{} + ticker := time.NewTicker(options.Interval) + defer ticker.Stop() + for { + err := reconcileNativeFSKit(ctx, options, operations, &state) + if errors.Is(err, ErrForeignMount) { + return err + } + if err != nil && options.Event != nil { + options.Event(err.Error()) + } + select { + case <-ctx.Done(): + return shutdownNativeFSKit(options, operations) + case <-ticker.C: + } + } +} + +type nativeFSKitSupervisorState struct { + unhealthyOwnedMounts int +} + +func reconcileNativeFSKit( + ctx context.Context, + options NativeFSKitSupervisorOptions, + operations NativeFSKitOperations, + state *nativeFSKitSupervisorState, +) error { + mountState, mountErr := operations.MountState(ctx, options.MountPoint, options.ProbeTimeout) + if mountState.Mounted && !mountState.Owned { + return fmt.Errorf("%w: %s", ErrForeignMount, options.MountPoint) + } + daemonErr := operations.DaemonHealthy(ctx, options.ResourcePath) + if mountState.Owned && mountState.Healthy && daemonErr == nil { + state.unhealthyOwnedMounts = 0 + return nil + } + if mountState.Owned && !mountState.Healthy { + state.unhealthyOwnedMounts++ + if state.unhealthyOwnedMounts < 2 { + return errors.Join(mountErr, daemonErr, errors.New("owned FSKit mount failed its first health probe")) + } + if err := operations.Unmount(ctx, options.MountPoint, true); err != nil { + return errors.Join(mountErr, daemonErr, fmt.Errorf("force-unmount stale FSKit mount: %w", err)) + } + mountState = NativeFSKitMountState{} + state.unhealthyOwnedMounts = 0 + } + if daemonErr != nil { + return errors.Join(mountErr, fmt.Errorf("FSKit daemon unavailable: %w", daemonErr)) + } + if mountErr != nil && mountState.Mounted { + return mountErr + } + if mountState.Owned && mountState.Healthy { + return nil + } + if err := operations.Mount(ctx, options.ResourcePath, options.MountPoint); err != nil { + return fmt.Errorf("mount native FSKit volume: %w", err) + } + return waitForNativeFSKitMount(ctx, options, operations) +} + +func waitForNativeFSKitMount(ctx context.Context, options NativeFSKitSupervisorOptions, operations NativeFSKitOperations) error { + deadline := time.NewTimer(options.RecoveryTimeout) + defer deadline.Stop() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + var lastErr error + for { + mountState, mountErr := operations.MountState(ctx, options.MountPoint, options.ProbeTimeout) + daemonErr := operations.DaemonHealthy(ctx, options.ResourcePath) + if mountState.Mounted && !mountState.Owned { + return fmt.Errorf("%w: %s", ErrForeignMount, options.MountPoint) + } + if mountState.Owned && mountState.Healthy && daemonErr == nil { + return nil + } + lastErr = errors.Join(mountErr, daemonErr) + select { + case <-ctx.Done(): + return ctx.Err() + case <-deadline.C: + return fmt.Errorf("native FSKit mount did not become healthy: %w", lastErr) + case <-ticker.C: + } + } +} + +func shutdownNativeFSKit(options NativeFSKitSupervisorOptions, operations NativeFSKitOperations) error { + ctx, cancel := context.WithTimeout(context.Background(), options.RecoveryTimeout) + defer cancel() + mountState, err := operations.MountState(ctx, options.MountPoint, options.ProbeTimeout) + if err != nil || !mountState.Owned { + return nil + } + if unmountErr := operations.Unmount(ctx, options.MountPoint, false); unmountErr == nil { + return nil + } + return operations.Unmount(ctx, options.MountPoint, true) +} diff --git a/internal/service/native_fskit_supervisor_test.go b/internal/service/native_fskit_supervisor_test.go new file mode 100644 index 0000000..62cec5d --- /dev/null +++ b/internal/service/native_fskit_supervisor_test.go @@ -0,0 +1,165 @@ +package service + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +func TestNativeFSKitSupervisorMountsAndUnmountsOnShutdown(t *testing.T) { + operations := &fakeNativeFSKitOperations{ + daemonHealthy: true, + mounted: make(chan struct{}), forceUnmounted: make(chan struct{}), + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- RunNativeFSKitSupervisor(ctx, NativeFSKitSupervisorOptions{ + ResourcePath: "/tmp/resource", MountPoint: "/tmp/mount", + Interval: time.Millisecond, RecoveryTimeout: 100 * time.Millisecond, + Operations: operations, + }) + }() + + select { + case <-operations.mounted: + case <-time.After(time.Second): + t.Fatal("supervisor did not mount") + } + cancel() + if err := <-done; err != nil { + t.Fatalf("supervisor shutdown: %v", err) + } + operations.mu.Lock() + defer operations.mu.Unlock() + if operations.mountCalls != 1 || operations.unmountCalls != 1 || operations.forceUnmountCalls != 0 { + t.Fatalf("mount calls=%d unmount=%d force=%d", operations.mountCalls, operations.unmountCalls, operations.forceUnmountCalls) + } +} + +func TestNativeFSKitSupervisorForceUnmountsStaleOwnedMountAfterTwoFailures(t *testing.T) { + operations := &fakeNativeFSKitOperations{ + daemonErr: errors.New("daemon unavailable"), + state: NativeFSKitMountState{Mounted: true, Owned: true, Healthy: false}, + mounted: make(chan struct{}), forceUnmounted: make(chan struct{}), + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { + done <- RunNativeFSKitSupervisor(ctx, NativeFSKitSupervisorOptions{ + ResourcePath: "/tmp/resource", MountPoint: "/tmp/mount", + Interval: time.Millisecond, RecoveryTimeout: 100 * time.Millisecond, + Operations: operations, + }) + }() + + select { + case <-operations.forceUnmounted: + cancel() + case <-time.After(time.Second): + t.Fatal("supervisor did not force-unmount stale owned mount") + } + if err := <-done; err != nil { + t.Fatalf("supervisor shutdown: %v", err) + } + operations.mu.Lock() + defer operations.mu.Unlock() + if operations.forceUnmountCalls != 1 || operations.probeCalls < 2 { + t.Fatalf("force unmount=%d probes=%d", operations.forceUnmountCalls, operations.probeCalls) + } +} + +func TestNativeFSKitSupervisorRefusesForeignMount(t *testing.T) { + operations := &fakeNativeFSKitOperations{ + daemonHealthy: true, + state: NativeFSKitMountState{Mounted: true, Owned: false, Healthy: false}, + mounted: make(chan struct{}), forceUnmounted: make(chan struct{}), + } + err := RunNativeFSKitSupervisor(context.Background(), NativeFSKitSupervisorOptions{ + ResourcePath: "/tmp/resource", MountPoint: "/tmp/mount", + Interval: time.Millisecond, RecoveryTimeout: 100 * time.Millisecond, + Operations: operations, + }) + if !errors.Is(err, ErrForeignMount) { + t.Fatalf("foreign mount error = %v", err) + } + operations.mu.Lock() + defer operations.mu.Unlock() + if operations.mountCalls != 0 || operations.unmountCalls != 0 || operations.forceUnmountCalls != 0 { + t.Fatalf("foreign mount was mutated: %#v", operations) + } +} + +type fakeNativeFSKitOperations struct { + mu sync.Mutex + + daemonHealthy bool + daemonErr error + state NativeFSKitMountState + + probeCalls int + mountCalls int + unmountCalls int + forceUnmountCalls int + + mounted chan struct{} + forceUnmounted chan struct{} +} + +func (f *fakeNativeFSKitOperations) DaemonHealthy(context.Context, string) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.daemonErr != nil { + return f.daemonErr + } + if !f.daemonHealthy { + return errors.New("daemon unavailable") + } + return nil +} + +func (f *fakeNativeFSKitOperations) MountState(context.Context, string, time.Duration) (NativeFSKitMountState, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.probeCalls++ + return f.state, nil +} + +func (f *fakeNativeFSKitOperations) Mount(context.Context, string, string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.mountCalls++ + f.state = NativeFSKitMountState{Mounted: true, Owned: true, Healthy: true} + select { + case <-f.mounted: + default: + close(f.mounted) + } + return nil +} + +func (f *fakeNativeFSKitOperations) Unmount(_ context.Context, _ string, force bool) error { + f.mu.Lock() + defer f.mu.Unlock() + f.recordUnmount(force) + if f.state.Owned { + f.state = NativeFSKitMountState{} + } + return nil +} + +func (f *fakeNativeFSKitOperations) recordUnmount(force bool) { + if force { + f.forceUnmountCalls++ + select { + case <-f.forceUnmounted: + default: + close(f.forceUnmounted) + } + return + } + f.unmountCalls++ +} diff --git a/internal/service/platform.go b/internal/service/platform.go new file mode 100644 index 0000000..e43828e --- /dev/null +++ b/internal/service/platform.go @@ -0,0 +1,40 @@ +package service + +import ( + "errors" + "runtime" +) + +type Platform string + +const ( + PlatformLaunchd Platform = "launchd" + PlatformSystemd Platform = "systemd-user" + PlatformWindows Platform = "windows-service" +) + +func CurrentPlatform() (Platform, error) { + switch runtime.GOOS { + case "darwin": + return PlatformLaunchd, nil + case "linux": + return PlatformSystemd, nil + case "windows": + return PlatformWindows, nil + default: + return "", errors.New("transparent filesystem services are supported only on macOS, Linux, and Windows") + } +} + +func RenderDefinition(platform Platform, options Options) ([]byte, error) { + switch platform { + case PlatformLaunchd: + return RenderLaunchd(options) + case PlatformSystemd: + return RenderSystemd(options) + case PlatformWindows: + return RenderWindowsConfig(options) + default: + return nil, errors.New("unknown service platform") + } +} diff --git a/internal/service/process_lock.go b/internal/service/process_lock.go index 373e0d0..4bc2ed8 100644 --- a/internal/service/process_lock.go +++ b/internal/service/process_lock.go @@ -3,14 +3,22 @@ package service import ( "errors" "fmt" + "io" "os" "path/filepath" + "strconv" + "strings" ) type ProcessLock struct { file *os.File } +type ProcessLockStatus struct { + Held bool + PID int +} + func AcquireProcessLock(path string) (*ProcessLock, error) { if !filepath.IsAbs(path) { return nil, errors.New("absolute process lock path is required") @@ -62,3 +70,36 @@ func (l *ProcessLock) Close() error { } return closeErr } + +func InspectProcessLock(path string) (ProcessLockStatus, error) { + if !filepath.IsAbs(path) { + return ProcessLockStatus{}, errors.New("absolute process lock path is required") + } + file, err := os.OpenFile(filepath.Clean(path), os.O_RDWR, 0) + if errors.Is(err, os.ErrNotExist) { + return ProcessLockStatus{}, nil + } + if err != nil { + return ProcessLockStatus{}, err + } + defer file.Close() + locked, err := tryLockProcessFile(file) + if err != nil { + return ProcessLockStatus{}, err + } + if locked { + return ProcessLockStatus{}, unlockProcessFile(file) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return ProcessLockStatus{Held: true}, err + } + value, err := io.ReadAll(io.LimitReader(file, 64)) + if err != nil { + return ProcessLockStatus{Held: true}, err + } + pid, err := strconv.Atoi(strings.TrimSpace(string(value))) + if err != nil || pid <= 1 { + return ProcessLockStatus{Held: true}, errors.New("held process lock has an invalid owner PID") + } + return ProcessLockStatus{Held: true, PID: pid}, nil +} diff --git a/internal/service/process_parent_darwin.go b/internal/service/process_parent_darwin.go new file mode 100644 index 0000000..a87d9da --- /dev/null +++ b/internal/service/process_parent_darwin.go @@ -0,0 +1,24 @@ +//go:build darwin + +package service + +import ( + "errors" + + "golang.org/x/sys/unix" +) + +func ProcessParentPID(pid int) (int, error) { + if pid <= 1 { + return 0, errors.New("valid process PID is required") + } + process, err := unix.SysctlKinfoProc("kern.proc.pid", pid) + if err != nil { + return 0, err + } + parent := int(process.Eproc.Ppid) + if parent <= 0 { + return 0, errors.New("process parent PID is unavailable") + } + return parent, nil +} diff --git a/internal/service/process_parent_darwin_test.go b/internal/service/process_parent_darwin_test.go new file mode 100644 index 0000000..3bbc3c2 --- /dev/null +++ b/internal/service/process_parent_darwin_test.go @@ -0,0 +1,18 @@ +//go:build darwin + +package service + +import ( + "os" + "testing" +) + +func TestProcessParentPIDReportsCurrentParent(t *testing.T) { + parent, err := ProcessParentPID(os.Getpid()) + if err != nil { + t.Fatal(err) + } + if parent != os.Getppid() { + t.Fatalf("parent PID = %d, want %d", parent, os.Getppid()) + } +} diff --git a/internal/service/process_parent_other.go b/internal/service/process_parent_other.go new file mode 100644 index 0000000..9c52456 --- /dev/null +++ b/internal/service/process_parent_other.go @@ -0,0 +1,9 @@ +//go:build !darwin + +package service + +import "errors" + +func ProcessParentPID(int) (int, error) { + return 0, errors.New("process parent inspection is available only on macOS") +} diff --git a/internal/service/service.go b/internal/service/service.go index 693a1bb..65aae0e 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "time" @@ -17,16 +18,23 @@ import ( ) type Options struct { - Label string - BinaryPath string - CodexHome string - StoreDir string - MountPoint string - StdoutPath string - StderrPath string - CanonicalNamespace bool - NativeRoot string - OperationTrace string + Label string + BinaryPath string + LauncherPath string + CodexHome string + StoreDir string + MountPoint string + StdoutPath string + StderrPath string + CanonicalNamespace bool + NativeRoot string + OperationTrace string + EnrollmentInterval time.Duration + EnrollmentStableFor time.Duration + EnrollmentBatchSize int + EnrollmentCanary bool + Frontend string + FSKitResource string } type InstallResult struct { @@ -52,10 +60,15 @@ type Manager struct { } type Status struct { - DaemonRunning bool `json:"daemon_running"` - MountHealthy bool `json:"mount_healthy"` - DaemonError string `json:"daemon_error,omitempty"` - MountError string `json:"mount_error,omitempty"` + DaemonRunning bool `json:"daemon_running"` + DaemonPID int `json:"daemon_pid,omitempty"` + SupervisorRunning bool `json:"supervisor_running,omitempty"` + SupervisorPID int `json:"supervisor_pid,omitempty"` + MountHealthy bool `json:"mount_healthy"` + DaemonError string `json:"daemon_error,omitempty"` + SupervisorError string `json:"supervisor_error,omitempty"` + MountError string `json:"mount_error,omitempty"` + Build BuildStatus `json:"build"` } type UpdateInput struct { @@ -75,24 +88,41 @@ type UpdateDecision struct { } func RenderLaunchd(options Options) ([]byte, error) { + serveArguments, err := ServeArguments(options) + if err != nil { + return nil, err + } + return renderLaunchdJob(options.Label, launchdProgramArguments(options, serveArguments), options.StdoutPath, options.StderrPath), nil +} + +func RenderLaunchdSupervisor(options Options) ([]byte, error) { if err := validateOptions(options); err != nil { return nil, err } - arguments := []string{ - options.BinaryPath, "fs", "serve", "--apply", "--foreground=true", - "--codex-home", options.CodexHome, "--store", options.StoreDir, "--mount", options.MountPoint, + if options.Frontend != "native-fskit" { + return nil, errors.New("native FSKit supervisor requires the native-fskit frontend") } - if options.CanonicalNamespace { - arguments = append(arguments, "--canonical-namespace", "--native-root", options.NativeRoot) + arguments := []string{ + "fs", "supervise", "--apply", + "--resource", options.FSKitResource, "--mount", options.MountPoint, } - if options.OperationTrace != "" { - arguments = append(arguments, "--operation-trace", options.OperationTrace) + return renderLaunchdJob(options.Label+".supervisor", launchdProgramArguments(options, arguments), options.StdoutPath, options.StderrPath), nil +} + +func launchdProgramArguments(options Options, childArguments []string) []string { + if options.Frontend == "native-fskit" { + arguments := []string{options.LauncherPath, "--run-helper", options.BinaryPath} + return append(arguments, childArguments...) } + return append([]string{options.BinaryPath}, childArguments...) +} + +func renderLaunchdJob(label string, arguments []string, stdoutPath string, stderrPath string) []byte { var output bytes.Buffer output.WriteString("\n") output.WriteString("\n") output.WriteString("\n\n") - writePlistString(&output, "Label", options.Label) + writePlistString(&output, "Label", label) output.WriteString(" ProgramArguments\n \n") for _, argument := range arguments { output.WriteString(" ") @@ -100,13 +130,49 @@ func RenderLaunchd(options Options) ([]byte, error) { output.WriteString("\n") } output.WriteString(" \n") - writePlistString(&output, "StandardOutPath", options.StdoutPath) - writePlistString(&output, "StandardErrorPath", options.StderrPath) + writePlistString(&output, "StandardOutPath", stdoutPath) + writePlistString(&output, "StandardErrorPath", stderrPath) output.WriteString(" RunAtLoad\n \n") output.WriteString(" KeepAlive\n \n") output.WriteString(" ProcessType\n Background\n") + output.WriteString(" ThrottleInterval\n 2\n") output.WriteString("\n\n") - return output.Bytes(), nil + return output.Bytes() +} + +func ServeArguments(options Options) ([]string, error) { + if err := validateOptions(options); err != nil { + return nil, err + } + arguments := []string{ + "fs", "serve", "--apply", "--foreground=true", + "--codex-home", options.CodexHome, "--store", options.StoreDir, "--mount", options.MountPoint, + } + frontend := options.Frontend + if frontend == "" { + frontend = "fuse" + } + arguments = append(arguments, "--frontend", frontend) + if frontend == "native-fskit" { + arguments = append(arguments, "--fskit-resource", options.FSKitResource) + } + if options.CanonicalNamespace { + arguments = append(arguments, "--canonical-namespace", "--native-root", options.NativeRoot) + } + if options.OperationTrace != "" { + arguments = append(arguments, "--operation-trace", options.OperationTrace) + } + if options.EnrollmentInterval > 0 { + arguments = append(arguments, + "--enrollment-interval", options.EnrollmentInterval.String(), + "--enrollment-stable-for", options.EnrollmentStableFor.String(), + "--enrollment-batch-size", strconv.Itoa(options.EnrollmentBatchSize), + ) + if options.EnrollmentCanary { + arguments = append(arguments, "--enrollment-canary") + } + } + return arguments, nil } func WriteDefinition(path string, definition []byte, apply bool) (InstallResult, error) { @@ -120,7 +186,7 @@ func WriteDefinition(path string, definition []byte, apply bool) (InstallResult, if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return InstallResult{}, err } - temporary, err := os.CreateTemp(filepath.Dir(path), ".launchd-*.tmp") + temporary, err := os.CreateTemp(filepath.Dir(path), ".service-definition-*.tmp") if err != nil { return InstallResult{}, err } @@ -151,24 +217,55 @@ func (m Manager) Bootstrap(ctx context.Context, plistPath string) error { if !filepath.IsAbs(plistPath) { return errors.New("absolute launchd plist path is required") } - _, err := m.runner().Run(ctx, "launchctl", "bootstrap", m.domain(), plistPath) - return err + output, err := m.runner().Run(ctx, "launchctl", "bootstrap", m.domain(), plistPath) + if err != nil { + return commandFailure("launchctl bootstrap", output, err) + } + return nil } func (m Manager) Bootout(ctx context.Context, plistPath string) error { if !filepath.IsAbs(plistPath) { return errors.New("absolute launchd plist path is required") } - _, err := m.runner().Run(ctx, "launchctl", "bootout", m.domain(), plistPath) - return err + output, err := m.runner().Run(ctx, "launchctl", "bootout", m.domain(), plistPath) + if err != nil { + return commandFailure("launchctl bootout", output, err) + } + return nil } func (m Manager) Kickstart(ctx context.Context, label string) error { if !safeLabel(label) { return errors.New("safe launchd label is required") } - _, err := m.runner().Run(ctx, "launchctl", "kickstart", m.domain()+"/"+label) - return err + output, err := m.runner().Run(ctx, "launchctl", "kickstart", m.domain()+"/"+label) + if err != nil { + return commandFailure("launchctl kickstart", output, err) + } + return nil +} + +func (m Manager) Enable(ctx context.Context, label string) error { + if !safeLabel(label) { + return errors.New("safe launchd label is required") + } + output, err := m.runner().Run(ctx, "launchctl", "enable", m.domain()+"/"+label) + if err != nil { + return commandFailure("launchctl enable", output, err) + } + return nil +} + +func (m Manager) Disable(ctx context.Context, label string) error { + if !safeLabel(label) { + return errors.New("safe launchd label is required") + } + output, err := m.runner().Run(ctx, "launchctl", "disable", m.domain()+"/"+label) + if err != nil { + return commandFailure("launchctl disable", output, err) + } + return nil } func (m Manager) Status(ctx context.Context, label string, mountPoint string) Status { @@ -178,6 +275,18 @@ func (m Manager) Status(ctx context.Context, label string, mountPoint string) St result.DaemonError = err.Error() } else if strings.Contains(string(output), "state = running") { result.DaemonRunning = true + for _, line := range strings.Split(string(output), "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "pid = ") { + continue + } + result.DaemonPID, _ = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "pid = "))) + break + } + if result.DaemonPID <= 1 { + result.DaemonRunning = false + result.DaemonError = "launchd job is running without a valid PID" + } } else { result.DaemonError = "launchd job is loaded but not running" } @@ -252,7 +361,7 @@ func (m Manager) domain() string { func validateOptions(options Options) error { if !safeLabel(options.Label) { - return errors.New("safe launchd label is required") + return errors.New("safe service label is required") } for name, path := range map[string]string{ "binary": options.BinaryPath, "Codex home": options.CodexHome, "store": options.StoreDir, @@ -268,6 +377,38 @@ func validateOptions(options Options) error { if options.OperationTrace != "" && !filepath.IsAbs(options.OperationTrace) { return errors.New("operation trace path must be absolute") } + if options.EnrollmentInterval < 0 || options.EnrollmentStableFor < 0 || options.EnrollmentBatchSize < 0 { + return errors.New("enrollment timing and batch values cannot be negative") + } + if options.EnrollmentInterval > 0 { + if !options.CanonicalNamespace { + return errors.New("periodic enrollment requires the canonical namespace") + } + if options.EnrollmentStableFor <= 0 || options.EnrollmentBatchSize <= 0 { + return errors.New("periodic enrollment requires a positive stable window and batch size") + } + } + if options.EnrollmentCanary && options.EnrollmentInterval <= 0 { + return errors.New("enrollment canary requires periodic enrollment") + } + frontend := options.Frontend + if frontend == "" { + frontend = "fuse" + } + if frontend != "fuse" && frontend != "native-fskit" { + return errors.New("filesystem frontend must be fuse or native-fskit") + } + if frontend == "native-fskit" { + if !options.CanonicalNamespace { + return errors.New("native-fskit frontend requires the canonical namespace") + } + if !filepath.IsAbs(options.LauncherPath) { + return errors.New("native-fskit frontend requires an absolute host launcher path") + } + if !filepath.IsAbs(options.FSKitResource) { + return errors.New("native-fskit frontend requires an absolute resource path") + } + } return nil } diff --git a/internal/service/service_test.go b/internal/service/service_test.go index af21dd4..c95794c 100644 --- a/internal/service/service_test.go +++ b/internal/service/service_test.go @@ -23,6 +23,8 @@ func TestRenderLaunchdUsesAbsoluteArgumentsAndContainsNoSessionContent(t *testin MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "logs", "stdout.log"), StderrPath: filepath.Join(root, "logs", "stderr.log"), CanonicalNamespace: true, NativeRoot: filepath.Join(root, "native"), OperationTrace: filepath.Join(root, "logs", "operations.log"), + EnrollmentInterval: 5 * time.Minute, EnrollmentStableFor: time.Hour, EnrollmentBatchSize: 2, + EnrollmentCanary: true, }) if err != nil { t.Fatalf("RenderLaunchd: %v", err) @@ -32,6 +34,10 @@ func TestRenderLaunchdUsesAbsoluteArgumentsAndContainsNoSessionContent(t *testin "fs", "serve", "--apply", "--canonical-namespace", "--native-root", "--operation-trace", filepath.Join(root, "logs", "operations.log"), + "--enrollment-interval", "5m0s", + "--enrollment-stable-for", "1h0m0s", + "--enrollment-batch-size", "2", + "--enrollment-canary", filepath.Join(root, "store"), filepath.Join(root, "mount"), filepath.Join(root, "native"), } { if !strings.Contains(text, required) { @@ -55,6 +61,123 @@ func TestRenderLaunchdUsesAbsoluteArgumentsAndContainsNoSessionContent(t *testin } } +func TestRenderLaunchdNativeFSKitSeparatesDaemonAndSupervisor(t *testing.T) { + root := t.TempDir() + launcher := filepath.Join(root, "CodexFoldFSKit.app", "Contents", "MacOS", "CodexFoldFSKit") + binary := filepath.Join(root, "bin", "codexfold") + options := Options{ + Label: "com.codexfold.fs", BinaryPath: binary, LauncherPath: launcher, + CodexHome: filepath.Join(root, "codex"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "logs", "stdout.log"), + StderrPath: filepath.Join(root, "logs", "stderr.log"), CanonicalNamespace: true, + NativeRoot: filepath.Join(root, "native"), Frontend: "native-fskit", + FSKitResource: filepath.Join(root, "store", "fs", "native-fskit.resource"), + } + daemon, err := RenderLaunchd(options) + if err != nil { + t.Fatalf("RenderLaunchd: %v", err) + } + for _, required := range []string{ + "" + launcher + "", "--run-helper", "" + binary + "", + "--frontend", "native-fskit", + "--fskit-resource", options.FSKitResource, + } { + if !strings.Contains(string(daemon), required) { + t.Fatalf("native daemon definition missing %q:\n%s", required, daemon) + } + } + + supervisor, err := RenderLaunchdSupervisor(options) + if err != nil { + t.Fatalf("RenderLaunchdSupervisor: %v", err) + } + text := string(supervisor) + for _, required := range []string{ + "com.codexfold.fs.supervisor", + "" + launcher + "", "--run-helper", "" + binary + "", + "fs", "supervise", "--apply", + "--resource", options.FSKitResource, + "--mount", options.MountPoint, + } { + if !strings.Contains(text, required) { + t.Fatalf("native supervisor definition missing %q:\n%s", required, text) + } + } +} + +func TestRenderLaunchdNativeFSKitRequiresHostLauncher(t *testing.T) { + root := t.TempDir() + _, err := RenderLaunchd(Options{ + Label: "com.codexfold.fs", BinaryPath: filepath.Join(root, "codexfold"), + CodexHome: filepath.Join(root, "home"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "stdout.log"), + StderrPath: filepath.Join(root, "stderr.log"), CanonicalNamespace: true, + NativeRoot: filepath.Join(root, "native"), Frontend: "native-fskit", + FSKitResource: filepath.Join(root, "resource"), + }) + if err == nil || !strings.Contains(err.Error(), "launcher") { + t.Fatalf("native FSKit without launcher error = %v", err) + } +} + +func TestRenderSystemdUsesTheSameServeArgumentsAndRestartPolicy(t *testing.T) { + root := filepath.Join(t.TempDir(), "path with space % and $") + options := Options{ + Label: "com.codexfold.fs", BinaryPath: filepath.Join(root, "bin", "codexfold"), + CodexHome: filepath.Join(root, "codex"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "logs", "stdout.log"), + StderrPath: filepath.Join(root, "logs", "stderr.log"), CanonicalNamespace: true, + NativeRoot: filepath.Join(root, "native"), OperationTrace: filepath.Join(root, "logs", "operations.log"), + EnrollmentInterval: 5 * time.Minute, EnrollmentStableFor: time.Hour, EnrollmentBatchSize: 2, + EnrollmentCanary: true, + } + definition, err := RenderSystemd(options) + if err != nil { + t.Fatalf("RenderSystemd: %v", err) + } + text := string(definition) + for _, required := range []string{ + "[Service]", "Type=simple", "Restart=on-failure", "RestartSec=2s", "TimeoutStopSec=30s", + "--canonical-namespace", "--native-root", "--operation-trace", + "--enrollment-interval", "5m0s", "--enrollment-canary", + "ExecStart=:\"", "StandardOutput=append:", "StandardError=append:", "\\x20", "%%", "$", + } { + if !strings.Contains(text, required) { + t.Fatalf("systemd definition missing %q:\n%s", required, text) + } + } + if strings.Contains(text, "session_meta") || strings.Contains(text, "rollout") { + t.Fatalf("definition contains session content: %s", text) + } +} + +func TestRenderWindowsConfigUsesTheSameServeArguments(t *testing.T) { + root := t.TempDir() + definition, err := RenderWindowsConfig(Options{ + Label: "com.codexfold.fs", BinaryPath: filepath.Join(root, "codexfold.exe"), + CodexHome: filepath.Join(root, "codex"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "logs", "stdout.log"), + StderrPath: filepath.Join(root, "logs", "stderr.log"), CanonicalNamespace: true, + NativeRoot: filepath.Join(root, "native"), + }) + if err != nil { + t.Fatalf("RenderWindowsConfig: %v", err) + } + config, err := ParseWindowsConfig(definition) + if err != nil { + t.Fatalf("ParseWindowsConfig: %v", err) + } + if config.Version != 1 || config.ServiceName != "com.codexfold.fs" { + t.Fatalf("unexpected Windows config: %#v", config) + } + joined := strings.Join(config.Arguments, " ") + for _, required := range []string{"fs serve --apply --foreground=true", "--canonical-namespace", "--native-root"} { + if !strings.Contains(joined, required) { + t.Fatalf("Windows config missing %q: %s", required, joined) + } + } +} + func TestManagerUsesOnlyPerUserLaunchctlAndSeparatesDaemonFromMount(t *testing.T) { root := t.TempDir() runner := &recordingRunner{outputs: map[string][]byte{"launchctl print gui/501/com.codexfold.fs": []byte("state = running\npid = 123\n")}} @@ -70,7 +193,7 @@ func TestManagerUsesOnlyPerUserLaunchctlAndSeparatesDaemonFromMount(t *testing.T t.Fatalf("Kickstart: %v", err) } status := manager.Status(context.Background(), "com.codexfold.fs", filepath.Join(root, "mount")) - if !status.DaemonRunning || status.MountHealthy { + if !status.DaemonRunning || status.DaemonPID != 123 || status.MountHealthy { t.Fatalf("status did not separate daemon and mount: %#v", status) } joined := strings.Join(runner.calls, "\n") @@ -79,6 +202,74 @@ func TestManagerUsesOnlyPerUserLaunchctlAndSeparatesDaemonFromMount(t *testing.T } } +func TestSystemdManagerUsesOnlyTheUserManagerAndSeparatesMountHealth(t *testing.T) { + runner := &recordingRunner{outputs: map[string][]byte{ + "systemctl --user show com.codexfold.fs.service --property=ActiveState --property=SubState --no-pager": []byte("ActiveState=active\nSubState=running\n"), + }} + manager := SystemdManager{Runner: runner, MountProbe: func(string) error { return errors.New("mount unavailable") }} + if err := manager.Start(context.Background(), "com.codexfold.fs.service"); err != nil { + t.Fatalf("Start: %v", err) + } + if err := manager.Stop(context.Background(), "com.codexfold.fs.service"); err != nil { + t.Fatalf("Stop: %v", err) + } + status := manager.Status(context.Background(), "com.codexfold.fs.service", filepath.Join(t.TempDir(), "mount")) + if !status.DaemonRunning || status.MountHealthy { + t.Fatalf("status did not separate daemon and mount: %#v", status) + } + joined := strings.Join(runner.calls, "\n") + for _, required := range []string{ + "systemctl --user daemon-reload", + "systemctl --user enable --now com.codexfold.fs.service", + "systemctl --user stop com.codexfold.fs.service", + } { + if !strings.Contains(joined, required) { + t.Fatalf("missing systemd user command %q:\n%s", required, joined) + } + } + if strings.Contains(joined, "sudo") || strings.Contains(joined, "systemctl enable") { + t.Fatalf("system service command leaked into user manager:\n%s", joined) + } +} + +func TestWindowsManagerInstallsStartsStopsAndReportsSCMState(t *testing.T) { + installRunner := &recordingRunner{errors: map[string]error{ + "sc.exe query com.codexfold.fs": errors.New("service does not exist"), + }} + manager := WindowsManager{Runner: installRunner} + binary := `C:\Program Files\CodexFold\codexfold.exe` + definition := `C:\ProgramData\CodexFold\service.json` + if err := manager.Install(context.Background(), "com.codexfold.fs", binary, definition); err != nil { + t.Fatalf("Install: %v", err) + } + joined := strings.Join(installRunner.calls, "\n") + for _, required := range []string{ + "sc.exe create com.codexfold.fs", + `"C:\Program Files\CodexFold\codexfold.exe" fs service run --definition C:\ProgramData\CodexFold\service.json`, + "start= auto", + "sc.exe failure com.codexfold.fs", + } { + if !strings.Contains(joined, required) { + t.Fatalf("missing Windows service command %q:\n%s", required, joined) + } + } + + statusRunner := &recordingRunner{outputs: map[string][]byte{ + "sc.exe queryex com.codexfold.fs": []byte("STATE : 4 RUNNING\n"), + }} + manager = WindowsManager{Runner: statusRunner, MountProbe: func(string) error { return nil }} + if err := manager.Start(context.Background(), "com.codexfold.fs"); err != nil { + t.Fatalf("Start: %v", err) + } + if err := manager.Stop(context.Background(), "com.codexfold.fs"); err != nil { + t.Fatalf("Stop: %v", err) + } + status := manager.Status(context.Background(), "com.codexfold.fs", filepath.Join(t.TempDir(), "mount")) + if !status.DaemonRunning || !status.MountHealthy { + t.Fatalf("Windows service status = %#v", status) + } +} + func TestStatusDoesNotTreatLoadedExitedJobAsRunning(t *testing.T) { runner := &recordingRunner{outputs: map[string][]byte{ "launchctl print gui/501/com.codexfold.fs": []byte("state = exited\nlast exit code = 1\n"), @@ -130,11 +321,19 @@ func TestEvaluateUpdateQuarantinesUnknownVersionsAndRejectsPreviewAutomation(t * func TestProcessLockAllowsOnlyOneFilesystemHost(t *testing.T) { path := filepath.Join(t.TempDir(), "service.lock") + status, err := InspectProcessLock(path) + if err != nil || status.Held { + t.Fatalf("missing process lock status = %#v err=%v", status, err) + } first, err := AcquireProcessLock(path) if err != nil { t.Fatal(err) } defer first.Close() + status, err = InspectProcessLock(path) + if err != nil || !status.Held || status.PID != os.Getpid() { + t.Fatalf("held process lock status = %#v err=%v", status, err) + } if _, err := AcquireProcessLock(path); err == nil { t.Fatal("a second filesystem host acquired the same process lock") @@ -142,6 +341,10 @@ func TestProcessLockAllowsOnlyOneFilesystemHost(t *testing.T) { if err := first.Close(); err != nil { t.Fatal(err) } + status, err = InspectProcessLock(path) + if err != nil || status.Held { + t.Fatalf("released process lock status = %#v err=%v", status, err) + } second, err := AcquireProcessLock(path) if err != nil { t.Fatalf("lock was not released after the first host exited: %v", err) @@ -154,11 +357,15 @@ func TestProcessLockAllowsOnlyOneFilesystemHost(t *testing.T) { type recordingRunner struct { calls []string outputs map[string][]byte + errors map[string]error } func (r *recordingRunner) Run(_ context.Context, name string, args ...string) ([]byte, error) { call := strings.Join(append([]string{name}, args...), " ") r.calls = append(r.calls, call) + if err, ok := r.errors[call]; ok { + return r.outputs[call], err + } if output, ok := r.outputs[call]; ok { return output, nil } diff --git a/internal/service/systemd.go b/internal/service/systemd.go new file mode 100644 index 0000000..e9c0d8f --- /dev/null +++ b/internal/service/systemd.go @@ -0,0 +1,183 @@ +package service + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "time" +) + +type SystemdManager struct { + Runner Runner + MountProbe func(string) error +} + +func RenderSystemd(options Options) ([]byte, error) { + arguments, err := ServeArguments(options) + if err != nil { + return nil, err + } + execStart := make([]string, 0, len(arguments)+1) + for _, argument := range append([]string{options.BinaryPath}, arguments...) { + quoted, err := quoteSystemdArgument(argument) + if err != nil { + return nil, err + } + execStart = append(execStart, quoted) + } + stdout, err := escapeSystemdSettingValue("append:" + options.StdoutPath) + if err != nil { + return nil, err + } + stderr, err := escapeSystemdSettingValue("append:" + options.StderrPath) + if err != nil { + return nil, err + } + + var output bytes.Buffer + output.WriteString("[Unit]\n") + output.WriteString("Description=CodexFold transparent session filesystem\n") + output.WriteString("After=default.target\n\n") + output.WriteString("[Service]\n") + output.WriteString("Type=simple\n") + output.WriteString("ExecStart=:") + output.WriteString(strings.Join(execStart, " ")) + output.WriteByte('\n') + output.WriteString("Restart=on-failure\n") + output.WriteString("RestartSec=2s\n") + output.WriteString("TimeoutStopSec=30s\n") + output.WriteString("KillMode=mixed\n") + output.WriteString("StandardOutput=") + output.WriteString(stdout) + output.WriteByte('\n') + output.WriteString("StandardError=") + output.WriteString(stderr) + output.WriteString("\n\n[Install]\n") + output.WriteString("WantedBy=default.target\n") + return output.Bytes(), nil +} + +func SystemdUnitName(label string) (string, error) { + if !safeLabel(label) { + return "", errors.New("safe service label is required") + } + return label + ".service", nil +} + +func (m SystemdManager) Start(ctx context.Context, unit string) error { + if !safeSystemdUnit(unit) { + return errors.New("safe systemd service unit is required") + } + if output, err := m.runner().Run(ctx, "systemctl", "--user", "daemon-reload"); err != nil { + return commandFailure("systemctl --user daemon-reload", output, err) + } + if output, err := m.runner().Run(ctx, "systemctl", "--user", "enable", "--now", unit); err != nil { + return commandFailure("systemctl --user enable --now", output, err) + } + return nil +} + +func (m SystemdManager) Stop(ctx context.Context, unit string) error { + if !safeSystemdUnit(unit) { + return errors.New("safe systemd service unit is required") + } + output, err := m.runner().Run(ctx, "systemctl", "--user", "stop", unit) + if err != nil { + return commandFailure("systemctl --user stop", output, err) + } + return nil +} + +func (m SystemdManager) Status(ctx context.Context, unit string, mountPoint string) Status { + result := Status{} + if !safeSystemdUnit(unit) { + result.DaemonError = "safe systemd service unit is required" + } else { + output, err := m.runner().Run(ctx, "systemctl", "--user", "show", unit, "--property=ActiveState", "--property=SubState", "--no-pager") + if err != nil { + result.DaemonError = commandFailure("systemctl --user show", output, err).Error() + } else if systemdStateRunning(output) { + result.DaemonRunning = true + } else { + result.DaemonError = "systemd user service is loaded but not running" + } + } + probe := m.MountProbe + if probe == nil { + probe = ProbeMount + } + if err := probe(mountPoint); err != nil { + result.MountError = err.Error() + } else { + result.MountHealthy = true + } + return result +} + +func (m SystemdManager) WaitHealthy(ctx context.Context, unit string, mountPoint string, timeout time.Duration) (Status, error) { + return waitHealthy(ctx, timeout, func() Status { return m.Status(ctx, unit, mountPoint) }) +} + +func (m SystemdManager) runner() Runner { + if m.Runner != nil { + return m.Runner + } + return ExecRunner{} +} + +func quoteSystemdArgument(value string) (string, error) { + if strings.ContainsAny(value, "\x00\r\n") { + return "", errors.New("systemd arguments cannot contain NUL or newlines") + } + value = strings.ReplaceAll(value, "\\", "\\\\") + value = strings.ReplaceAll(value, "\"", "\\\"") + value = strings.ReplaceAll(value, "%", "%%") + return "\"" + value + "\"", nil +} + +func escapeSystemdSettingValue(value string) (string, error) { + if strings.ContainsAny(value, "\x00\r\n") { + return "", errors.New("systemd setting values cannot contain NUL or newlines") + } + var output strings.Builder + for index := 0; index < len(value); index++ { + character := value[index] + switch { + case character == '%': + output.WriteString("%%") + case character <= 0x20 || character == '\\' || character == '"': + _, _ = fmt.Fprintf(&output, "\\x%02x", character) + default: + output.WriteByte(character) + } + } + return output.String(), nil +} + +func safeSystemdUnit(unit string) bool { + return strings.HasSuffix(unit, ".service") && safeLabel(strings.TrimSuffix(unit, ".service")) +} + +func systemdStateRunning(output []byte) bool { + active := false + running := false + for _, line := range strings.Split(string(output), "\n") { + switch strings.TrimSpace(line) { + case "ActiveState=active": + active = true + case "SubState=running": + running = true + } + } + return active && running +} + +func commandFailure(action string, output []byte, err error) error { + message := strings.TrimSpace(string(output)) + if message == "" { + return fmt.Errorf("%s: %w", action, err) + } + return fmt.Errorf("%s: %w: %s", action, err, message) +} diff --git a/internal/service/systemd_linux_test.go b/internal/service/systemd_linux_test.go new file mode 100644 index 0000000..deb010b --- /dev/null +++ b/internal/service/systemd_linux_test.go @@ -0,0 +1,102 @@ +//go:build linux + +package service + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestRenderSystemdPassesSystemdAnalyzeVerify(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_SYSTEMD_USER_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_SYSTEMD_USER_TEST=1 to run systemd verification") + } + root := filepath.Join(t.TempDir(), "path with space % and $") + binary := filepath.Join(root, "bin", "codexfold") + if err := os.MkdirAll(filepath.Dir(binary), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o700); err != nil { + t.Fatal(err) + } + definition, err := RenderSystemd(Options{ + Label: "com.codexfold.fs", BinaryPath: binary, + CodexHome: filepath.Join(root, "codex"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "logs", "stdout.log"), + StderrPath: filepath.Join(root, "logs", "stderr.log"), CanonicalNamespace: true, + NativeRoot: filepath.Join(root, "native"), + }) + if err != nil { + t.Fatal(err) + } + unitPath := filepath.Join(root, "com.codexfold.fs.service") + if err := os.WriteFile(unitPath, definition, 0o600); err != nil { + t.Fatal(err) + } + if output, err := exec.Command("systemd-analyze", "--user", "verify", unitPath).CombinedOutput(); err != nil { + t.Fatalf("systemd-analyze rejected generated unit: %v\n%s", err, output) + } +} + +func TestRealSystemdUserManagerLifecycle(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_SYSTEMD_USER_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_SYSTEMD_USER_TEST=1 to run systemd lifecycle") + } + home, err := os.UserHomeDir() + if err != nil { + t.Fatal(err) + } + unit := fmt.Sprintf("codexfold-validation-%d.service", os.Getpid()) + unitPath := filepath.Join(home, ".config", "systemd", "user", unit) + if err := os.MkdirAll(filepath.Dir(unitPath), 0o700); err != nil { + t.Fatal(err) + } + definition := []byte("[Unit]\nDescription=CodexFold systemd user validation\n\n[Service]\nType=simple\nExecStart=/usr/bin/sleep infinity\n\n[Install]\nWantedBy=default.target\n") + if err := os.WriteFile(unitPath, definition, 0o600); err != nil { + t.Fatal(err) + } + cleanup := func() { + _, _ = exec.Command("systemctl", "--user", "disable", "--now", unit).CombinedOutput() + _ = os.Remove(unitPath) + _, _ = exec.Command("systemctl", "--user", "daemon-reload").CombinedOutput() + } + t.Cleanup(cleanup) + + manager := SystemdManager{MountProbe: func(string) error { return nil }} + mountPoint := filepath.Join(t.TempDir(), "mount") + if err := manager.Start(context.Background(), unit); err != nil { + t.Fatal(err) + } + status, err := manager.WaitHealthy(context.Background(), unit, mountPoint, 10*time.Second) + if err != nil { + t.Fatal(err) + } + if !status.DaemonRunning || !status.MountHealthy { + t.Fatalf("unexpected running status: %#v", status) + } + if err := manager.Stop(context.Background(), unit); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + status = manager.Status(context.Background(), unit, mountPoint) + if !status.DaemonRunning { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("systemd user unit remained running: %#v", status) +} + +func TestLinuxMountProbeRejectsAnOrdinaryDirectory(t *testing.T) { + err := ProbeMount(t.TempDir()) + if err == nil || !strings.Contains(err.Error(), "not a CodexFold FUSE mount root") { + t.Fatalf("ordinary directory probe error = %v", err) + } +} diff --git a/internal/service/wait.go b/internal/service/wait.go new file mode 100644 index 0000000..fdbfb86 --- /dev/null +++ b/internal/service/wait.go @@ -0,0 +1,31 @@ +package service + +import ( + "context" + "fmt" + "time" +) + +func waitHealthy(ctx context.Context, timeout time.Duration, status func() Status) (Status, error) { + if timeout <= 0 { + timeout = 15 * time.Second + } + deadline := time.NewTimer(timeout) + defer deadline.Stop() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + var last Status + for { + last = status() + if last.DaemonRunning && last.MountHealthy { + return last, nil + } + select { + case <-ctx.Done(): + return last, ctx.Err() + case <-deadline.C: + return last, fmt.Errorf("filesystem service did not become healthy: daemon=%t mount=%t daemon_error=%q mount_error=%q", last.DaemonRunning, last.MountHealthy, last.DaemonError, last.MountError) + case <-ticker.C: + } + } +} diff --git a/internal/service/windows.go b/internal/service/windows.go new file mode 100644 index 0000000..3d67769 --- /dev/null +++ b/internal/service/windows.go @@ -0,0 +1,197 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "regexp" + "strings" + "time" +) + +const windowsConfigVersion = 1 + +var windowsRunningState = regexp.MustCompile(`(?m)STATE\s*:\s*4\s+RUNNING\b`) + +type WindowsConfig struct { + Version int `json:"version"` + ServiceName string `json:"service_name"` + BinaryPath string `json:"binary_path,omitempty"` + Arguments []string `json:"arguments"` + StdoutPath string `json:"stdout_path"` + StderrPath string `json:"stderr_path"` +} + +type WindowsManager struct { + Runner Runner + MountProbe func(string) error +} + +func RenderWindowsConfig(options Options) ([]byte, error) { + arguments, err := ServeArguments(options) + if err != nil { + return nil, err + } + return json.MarshalIndent(WindowsConfig{ + Version: windowsConfigVersion, ServiceName: options.Label, BinaryPath: options.BinaryPath, Arguments: arguments, + StdoutPath: options.StdoutPath, StderrPath: options.StderrPath, + }, "", " ") +} + +func ParseWindowsConfig(definition []byte) (WindowsConfig, error) { + var config WindowsConfig + if err := json.Unmarshal(definition, &config); err != nil { + return WindowsConfig{}, err + } + if config.Version != windowsConfigVersion { + return WindowsConfig{}, fmt.Errorf("unsupported Windows service config version %d", config.Version) + } + if !safeLabel(config.ServiceName) { + return WindowsConfig{}, errors.New("safe Windows service name is required") + } + if config.BinaryPath == "" || (!filepath.IsAbs(config.BinaryPath) && !absoluteWindowsServicePath(config.BinaryPath)) { + return WindowsConfig{}, errors.New("Windows service config binary path must be absolute") + } + if len(config.Arguments) < 2 || config.Arguments[0] != "fs" || config.Arguments[1] != "serve" { + return WindowsConfig{}, errors.New("Windows service config must run fs serve") + } + for _, path := range []string{config.StdoutPath, config.StderrPath} { + if !filepath.IsAbs(path) { + return WindowsConfig{}, errors.New("Windows service log paths must be absolute") + } + } + return config, nil +} + +func (m WindowsManager) Install(ctx context.Context, name string, binaryPath string, definitionPath string) error { + if !safeLabel(name) { + return errors.New("safe Windows service name is required") + } + if !absoluteWindowsServicePath(binaryPath) || !absoluteWindowsServicePath(definitionPath) { + return errors.New("Windows service binary and definition paths must be absolute") + } + commandLine := windowsServiceCommand(binaryPath, definitionPath) + _, queryErr := m.runner().Run(ctx, "sc.exe", "query", name) + if queryErr == nil { + output, err := m.runner().Run(ctx, "sc.exe", "config", name, "binPath=", commandLine, "start=", "auto") + if err != nil { + return commandFailure("sc.exe config", output, err) + } + } else { + output, err := m.runner().Run(ctx, "sc.exe", "create", name, "binPath=", commandLine, "start=", "auto", "DisplayName=", "CodexFold Transparent Session Filesystem") + if err != nil { + return commandFailure("sc.exe create", output, err) + } + } + if output, err := m.runner().Run(ctx, "sc.exe", "description", name, "CodexFold transparent Codex session filesystem"); err != nil { + return commandFailure("sc.exe description", output, err) + } + if output, err := m.runner().Run(ctx, "sc.exe", "failure", name, "reset=", "86400", "actions=", "restart/5000/restart/15000/\"\"/0"); err != nil { + return commandFailure("sc.exe failure", output, err) + } + return nil +} + +func (m WindowsManager) Start(ctx context.Context, name string) error { + if !safeLabel(name) { + return errors.New("safe Windows service name is required") + } + output, err := m.runner().Run(ctx, "sc.exe", "start", name) + if err != nil { + return commandFailure("sc.exe start", output, err) + } + return nil +} + +func (m WindowsManager) Stop(ctx context.Context, name string) error { + if !safeLabel(name) { + return errors.New("safe Windows service name is required") + } + output, err := m.runner().Run(ctx, "sc.exe", "stop", name) + if err != nil { + return commandFailure("sc.exe stop", output, err) + } + return nil +} + +func (m WindowsManager) Status(ctx context.Context, name string, mountPoint string) Status { + result := Status{} + if !safeLabel(name) { + result.DaemonError = "safe Windows service name is required" + } else { + output, err := m.runner().Run(ctx, "sc.exe", "queryex", name) + if err != nil { + result.DaemonError = commandFailure("sc.exe queryex", output, err).Error() + } else if windowsRunningState.Match(output) { + result.DaemonRunning = true + } else { + result.DaemonError = "Windows service is installed but not running" + } + } + probe := m.MountProbe + if probe == nil { + probe = ProbeMount + } + if err := probe(mountPoint); err != nil { + result.MountError = err.Error() + } else { + result.MountHealthy = true + } + return result +} + +func (m WindowsManager) WaitHealthy(ctx context.Context, name string, mountPoint string, timeout time.Duration) (Status, error) { + return waitHealthy(ctx, timeout, func() Status { return m.Status(ctx, name, mountPoint) }) +} + +func (m WindowsManager) runner() Runner { + if m.Runner != nil { + return m.Runner + } + return ExecRunner{} +} + +func windowsServiceCommand(binaryPath string, definitionPath string) string { + return strings.Join([]string{ + quoteWindowsCommandLineArgument(binaryPath), "fs", "service", "run", "--definition", + quoteWindowsCommandLineArgument(definitionPath), + }, " ") +} + +func quoteWindowsCommandLineArgument(value string) string { + if value != "" && !strings.ContainsAny(value, " \t\n\v\"") { + return value + } + var output strings.Builder + output.WriteByte('"') + backslashes := 0 + for _, character := range value { + switch character { + case '\\': + backslashes++ + case '"': + output.WriteString(strings.Repeat("\\", backslashes*2+1)) + output.WriteRune(character) + backslashes = 0 + default: + output.WriteString(strings.Repeat("\\", backslashes)) + output.WriteRune(character) + backslashes = 0 + } + } + output.WriteString(strings.Repeat("\\", backslashes*2)) + output.WriteByte('"') + return output.String() +} + +func absoluteWindowsServicePath(path string) bool { + if filepath.IsAbs(path) { + return true + } + if len(path) >= 3 && ((path[0] >= 'a' && path[0] <= 'z') || (path[0] >= 'A' && path[0] <= 'Z')) && path[1] == ':' && (path[2] == '\\' || path[2] == '/') { + return true + } + return strings.HasPrefix(path, `\\`) +} diff --git a/internal/storage/accounting.go b/internal/storage/accounting.go new file mode 100644 index 0000000..87da407 --- /dev/null +++ b/internal/storage/accounting.go @@ -0,0 +1,32 @@ +package storage + +import "context" + +type MutationAccounting struct { + Before Inventory `json:"before"` + Budget BudgetReport `json:"budget"` + After Inventory `json:"after"` + ProjectedReclaimableBytes int64 `json:"projected_reclaimable_bytes"` + ActualReclaimedBytes int64 `json:"actual_reclaimed_bytes"` + AfterInventoryError string `json:"after_inventory_error,omitempty"` +} + +func CompleteAccounting(ctx context.Context, assessment Assessment, storeDir string) *MutationAccounting { + accounting := &MutationAccounting{ + Before: assessment.Inventory, Budget: assessment.Budget, + ProjectedReclaimableBytes: assessment.Budget.ProjectedReclaimableBytes, + } + if storeDir == "" { + return accounting + } + after, err := Scan(ctx, Options{StoreDir: storeDir, AllowMetadataIssues: true}) + if err != nil { + accounting.AfterInventoryError = err.Error() + return accounting + } + accounting.After = after + if assessment.Inventory.StoreDir != "" && assessment.Inventory.TotalPhysicalBytes > after.TotalPhysicalBytes { + accounting.ActualReclaimedBytes = assessment.Inventory.TotalPhysicalBytes - after.TotalPhysicalBytes + } + return accounting +} diff --git a/internal/storage/budget.go b/internal/storage/budget.go new file mode 100644 index 0000000..e94e8c9 --- /dev/null +++ b/internal/storage/budget.go @@ -0,0 +1,103 @@ +package storage + +import ( + "errors" + "fmt" + "math" +) + +var ErrBudgetExceeded = errors.New("storage budget exceeded") + +type RejectionReason string + +const ( + RejectionPhysicalBudget RejectionReason = "physical-budget" + RejectionTemporaryBudget RejectionReason = "temporary-budget" + RejectionFreeSpaceReserve RejectionReason = "free-space-reserve" +) + +type Limits struct { + MaxPhysicalBytes int64 `json:"max_physical_bytes"` + MaxTemporaryBytes int64 `json:"max_temporary_bytes"` + FreeSpaceReserveBytes int64 `json:"free_space_reserve_bytes"` +} + +type BudgetRequest struct { + Operation string `json:"operation"` + CurrentPhysicalBytes int64 `json:"current_physical_bytes"` + AdditionalPersistentBytes int64 `json:"additional_persistent_bytes"` + TemporaryBytes int64 `json:"temporary_bytes"` + TemporaryPersistentOverlapBytes int64 `json:"temporary_persistent_overlap_bytes"` + ReclaimableBytes int64 `json:"reclaimable_bytes"` + AvailableBytes int64 `json:"available_bytes"` +} + +type BudgetReport struct { + Operation string `json:"operation"` + Allowed bool `json:"allowed"` + CurrentPhysicalBytes int64 `json:"current_physical_bytes"` + ProjectedPeakBytes int64 `json:"projected_peak_bytes"` + ProjectedFinalBytes int64 `json:"projected_final_bytes"` + ProjectedReclaimableBytes int64 `json:"projected_reclaimable_bytes"` + AvailableBytes int64 `json:"available_bytes"` + FreeSpaceAfterPeak int64 `json:"free_space_after_peak"` + Rejections []RejectionReason `json:"rejections,omitempty"` +} + +func CheckBudget(request BudgetRequest, limits Limits) (BudgetReport, error) { + if request.Operation == "" { + return BudgetReport{}, errors.New("storage budget operation is required") + } + if request.CurrentPhysicalBytes < 0 || request.AdditionalPersistentBytes < 0 || request.TemporaryBytes < 0 || request.TemporaryPersistentOverlapBytes < 0 || request.ReclaimableBytes < 0 || request.AvailableBytes < 0 { + return BudgetReport{}, errors.New("storage budget byte counts cannot be negative") + } + if request.TemporaryPersistentOverlapBytes > request.TemporaryBytes || request.TemporaryPersistentOverlapBytes > request.AdditionalPersistentBytes { + return BudgetReport{}, errors.New("temporary and persistent overlap exceeds the projected bytes") + } + if limits.MaxPhysicalBytes < 0 || limits.MaxTemporaryBytes < 0 || limits.FreeSpaceReserveBytes < 0 { + return BudgetReport{}, errors.New("storage limits cannot be negative") + } + additional, overflow := addBudgetBytes(request.AdditionalPersistentBytes, request.TemporaryBytes) + if overflow { + return BudgetReport{}, errors.New("storage budget additional byte projection overflow") + } + additional -= request.TemporaryPersistentOverlapBytes + peak, overflow := addBudgetBytes(request.CurrentPhysicalBytes, additional) + if overflow { + return BudgetReport{}, errors.New("storage budget peak byte projection overflow") + } + beforeReclaim, overflow := addBudgetBytes(request.CurrentPhysicalBytes, request.AdditionalPersistentBytes) + if overflow { + return BudgetReport{}, errors.New("storage budget final byte projection overflow") + } + final := beforeReclaim - min(request.ReclaimableBytes, beforeReclaim) + freeAfterPeak := request.AvailableBytes - additional + report := BudgetReport{ + Operation: request.Operation, Allowed: true, + CurrentPhysicalBytes: request.CurrentPhysicalBytes, + ProjectedPeakBytes: peak, ProjectedFinalBytes: final, + ProjectedReclaimableBytes: request.ReclaimableBytes, + AvailableBytes: request.AvailableBytes, FreeSpaceAfterPeak: freeAfterPeak, + } + if limits.MaxPhysicalBytes > 0 && peak > limits.MaxPhysicalBytes { + report.Rejections = append(report.Rejections, RejectionPhysicalBudget) + } + if limits.MaxTemporaryBytes > 0 && request.TemporaryBytes > limits.MaxTemporaryBytes { + report.Rejections = append(report.Rejections, RejectionTemporaryBudget) + } + if freeAfterPeak < limits.FreeSpaceReserveBytes { + report.Rejections = append(report.Rejections, RejectionFreeSpaceReserve) + } + if len(report.Rejections) != 0 { + report.Allowed = false + return report, fmt.Errorf("%w: %s rejected by %v", ErrBudgetExceeded, request.Operation, report.Rejections) + } + return report, nil +} + +func addBudgetBytes(left int64, right int64) (int64, bool) { + if left > math.MaxInt64-right { + return 0, true + } + return left + right, false +} diff --git a/internal/storage/budget_test.go b/internal/storage/budget_test.go new file mode 100644 index 0000000..81902d7 --- /dev/null +++ b/internal/storage/budget_test.go @@ -0,0 +1,105 @@ +package storage + +import ( + "errors" + "testing" +) + +func TestCheckBudgetCalculatesPeakWithoutSubtractingFutureReclamation(t *testing.T) { + report, err := CheckBudget(BudgetRequest{ + Operation: "compact", + CurrentPhysicalBytes: 100, + AdditionalPersistentBytes: 30, + TemporaryBytes: 80, + ReclaimableBytes: 70, + AvailableBytes: 1_000, + }, Limits{ + MaxPhysicalBytes: 500, + MaxTemporaryBytes: 100, + FreeSpaceReserveBytes: 200, + }) + if err != nil { + t.Fatalf("CheckBudget: %v", err) + } + if !report.Allowed || report.ProjectedPeakBytes != 210 || report.ProjectedFinalBytes != 60 || report.ProjectedReclaimableBytes != 70 { + t.Fatalf("unexpected report: %#v", report) + } + if report.FreeSpaceAfterPeak != 890 { + t.Fatalf("free space after peak = %d, want 890", report.FreeSpaceAfterPeak) + } +} + +func TestCheckBudgetDoesNotDoubleCountTemporaryBytesThatBecomePersistent(t *testing.T) { + report, err := CheckBudget(BudgetRequest{ + Operation: "copy-on-write", + CurrentPhysicalBytes: 100, + AdditionalPersistentBytes: 80, + TemporaryBytes: 80, + TemporaryPersistentOverlapBytes: 80, + AvailableBytes: 1_000, + }, Limits{}) + if err != nil { + t.Fatalf("CheckBudget: %v", err) + } + if report.ProjectedPeakBytes != 180 || report.ProjectedFinalBytes != 180 || report.FreeSpaceAfterPeak != 920 { + t.Fatalf("temporary rename projection = %#v", report) + } +} + +func TestCheckBudgetRejectsEveryHardLimit(t *testing.T) { + tests := []struct { + name string + request BudgetRequest + limits Limits + reason RejectionReason + }{ + { + name: "physical footprint", + request: BudgetRequest{ + Operation: "pack", CurrentPhysicalBytes: 90, AdditionalPersistentBytes: 20, + AvailableBytes: 1_000, + }, + limits: Limits{MaxPhysicalBytes: 100}, + reason: RejectionPhysicalBudget, + }, + { + name: "temporary bytes", + request: BudgetRequest{ + Operation: "rollback", CurrentPhysicalBytes: 20, TemporaryBytes: 81, + AvailableBytes: 1_000, + }, + limits: Limits{MaxTemporaryBytes: 80}, + reason: RejectionTemporaryBudget, + }, + { + name: "free space reserve", + request: BudgetRequest{ + Operation: "migrate", CurrentPhysicalBytes: 20, AdditionalPersistentBytes: 30, TemporaryBytes: 40, + AvailableBytes: 100, + }, + limits: Limits{FreeSpaceReserveBytes: 31}, + reason: RejectionFreeSpaceReserve, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + report, err := CheckBudget(test.request, test.limits) + if !errors.Is(err, ErrBudgetExceeded) { + t.Fatalf("error = %v, want ErrBudgetExceeded", err) + } + if report.Allowed || len(report.Rejections) != 1 || report.Rejections[0] != test.reason { + t.Fatalf("unexpected rejection report: %#v", report) + } + }) + } +} + +func TestCheckBudgetRejectsInvalidOrOverflowingProjections(t *testing.T) { + if _, err := CheckBudget(BudgetRequest{Operation: "fold", CurrentPhysicalBytes: -1}, Limits{}); err == nil { + t.Fatal("negative current bytes should fail") + } + if _, err := CheckBudget(BudgetRequest{Operation: "fold", CurrentPhysicalBytes: int64(^uint64(0) >> 1), TemporaryBytes: 1, AvailableBytes: 10}, Limits{}); err == nil { + t.Fatal("overflowing peak projection should fail") + } +} diff --git a/internal/storage/gc.go b/internal/storage/gc.go new file mode 100644 index 0000000..4bf1ec7 --- /dev/null +++ b/internal/storage/gc.go @@ -0,0 +1,699 @@ +package storage + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +type CandidateKind string + +const ( + CandidatePackGeneration CandidateKind = "pack-generation" + CandidateManifestGeneration CandidateKind = "manifest-generation" + CandidateSessionGeneration CandidateKind = "session-generation" + CandidateRetiredState CandidateKind = "retired-state" + CandidateTemporary CandidateKind = "unowned-temporary" +) + +type GCCandidate struct { + Kind CandidateKind `json:"kind"` + Path string `json:"path"` + Files int `json:"files"` + ApparentBytes int64 `json:"apparent_bytes"` + PhysicalBytes int64 `json:"physical_bytes"` + ReclaimableBytes int64 `json:"reclaimable_bytes"` +} + +type GCOptions struct { + StoreDir string + Apply bool + TemporaryGrace time.Duration + KeepPackGenerations int + KeepManifestGenerations int + KeepRetiredPerSession int + Now func() time.Time +} + +type StorageGCResult struct { + StoreDir string `json:"store_dir"` + DryRun bool `json:"dry_run"` + Before Inventory `json:"before"` + After Inventory `json:"after"` + Candidates []GCCandidate `json:"candidates,omitempty"` + CandidateCount int `json:"candidate_count"` + CandidateApparentBytes int64 `json:"candidate_apparent_bytes"` + ProjectedReclaimableBytes int64 `json:"projected_reclaimable_bytes"` + RemovedCount int `json:"removed_count"` + RemovedApparentBytes int64 `json:"removed_apparent_bytes"` + ActualReclaimedBytes int64 `json:"actual_reclaimed_bytes"` +} + +type gcBuilder struct { + ctx context.Context + options GCOptions + scanner *scanner + candidates map[string]CandidateKind +} + +type generationEntry struct { + path string + name string + modTime time.Time + sequence uint64 + sequenced bool +} + +func Collect(ctx context.Context, options GCOptions) (StorageGCResult, error) { + if options.StoreDir == "" { + return StorageGCResult{}, errors.New("storage GC store directory is required") + } + if options.TemporaryGrace < 0 || options.KeepPackGenerations < 0 || options.KeepManifestGenerations < 0 || options.KeepRetiredPerSession < 0 { + return StorageGCResult{}, errors.New("storage GC retention values cannot be negative") + } + if options.TemporaryGrace == 0 { + options.TemporaryGrace = time.Hour + } + if options.KeepPackGenerations == 0 { + options.KeepPackGenerations = 2 + } + if options.KeepManifestGenerations == 0 { + options.KeepManifestGenerations = 2 + } + if options.KeepRetiredPerSession == 0 { + options.KeepRetiredPerSession = 1 + } + if options.Now == nil { + options.Now = time.Now + } + store := cleanAbsolutePath(options.StoreDir) + before, err := Scan(ctx, Options{StoreDir: store, AllowMetadataIssues: true}) + if err != nil { + return StorageGCResult{}, err + } + result := StorageGCResult{StoreDir: store, DryRun: !options.Apply, Before: before, After: before} + metadata, exists, err := prepareScanner(ctx, Options{StoreDir: store, AllowMetadataIssues: true}) + if err != nil { + return StorageGCResult{}, err + } + if !exists { + return result, nil + } + builder := &gcBuilder{ctx: ctx, options: options, scanner: metadata, candidates: make(map[string]CandidateKind)} + if err := builder.discoverPackGenerations(); err != nil { + return StorageGCResult{}, err + } + if err := builder.discoverManifestGenerations(); err != nil { + return StorageGCResult{}, err + } + if err := builder.discoverSessionGenerations(); err != nil { + return StorageGCResult{}, err + } + if err := builder.discoverRetiredState(); err != nil { + return StorageGCResult{}, err + } + if err := builder.discoverTemporaryFiles(); err != nil { + return StorageGCResult{}, err + } + candidates, projected, err := describeCandidates(builder.candidates) + if err != nil { + return StorageGCResult{}, err + } + result.Candidates = candidates + result.CandidateCount = len(candidates) + result.ProjectedReclaimableBytes = projected + for _, candidate := range candidates { + result.CandidateApparentBytes += candidate.ApparentBytes + } + if !options.Apply { + return result, nil + } + for _, candidate := range candidates { + if err := ctx.Err(); err != nil { + return result, err + } + allowed, err := builder.revalidate(candidate) + if err != nil { + return result, err + } + if !allowed { + continue + } + if _, err := os.Lstat(candidate.Path); errors.Is(err, os.ErrNotExist) { + continue + } else if err != nil { + return result, err + } + if err := os.RemoveAll(candidate.Path); err != nil { + return result, fmt.Errorf("remove storage GC candidate %s: %w", candidate.Path, err) + } + result.RemovedCount++ + result.RemovedApparentBytes += candidate.ApparentBytes + } + after, err := Scan(ctx, Options{StoreDir: store, AllowMetadataIssues: true}) + if err != nil { + return result, err + } + result.After = after + if before.TotalPhysicalBytes > after.TotalPhysicalBytes { + result.ActualReclaimedBytes = before.TotalPhysicalBytes - after.TotalPhysicalBytes + } + return result, nil +} + +func (b *gcBuilder) add(path string, kind CandidateKind) error { + path = cleanAbsolutePath(path) + if !pathWithin(b.scanner.store, path) || path == b.scanner.store { + return errors.New("storage GC candidate escapes the store") + } + for existing := range b.candidates { + if pathWithin(existing, path) { + return nil + } + if pathWithin(path, existing) { + delete(b.candidates, existing) + } + } + b.candidates[path] = kind + return nil +} + +func (b *gcBuilder) covered(path string) bool { + for candidate := range b.candidates { + if pathWithin(candidate, path) { + return true + } + } + return false +} + +func (b *gcBuilder) discoverPackGenerations() error { + root := filepath.Join(b.scanner.store, "packs") + entries, err := os.ReadDir(root) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if b.scanner.currentPack == "" { + return nil + } + var generations []generationEntry + for _, entry := range entries { + if !entry.IsDir() || strings.HasPrefix(entry.Name(), ".") { + continue + } + info, err := entry.Info() + if err != nil { + return err + } + generations = append(generations, generationEntry{path: filepath.Join(root, entry.Name()), name: entry.Name(), modTime: info.ModTime()}) + } + sortGenerationEntries(generations) + previousToKeep := max(0, b.options.KeepPackGenerations-1) + for _, generation := range generations { + if generation.name == b.scanner.currentPack { + continue + } + active, err := DirectoryHasActiveLease(filepath.Join(generation.path, "leases"), false) + if err != nil { + return err + } + if active { + continue + } + if previousToKeep > 0 { + previousToKeep-- + continue + } + if err := b.add(generation.path, CandidatePackGeneration); err != nil { + return err + } + } + return nil +} + +func (b *gcBuilder) discoverManifestGenerations() error { + root := filepath.Join(b.scanner.store, "manifests", "generations") + entries, err := os.ReadDir(root) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + sessionID := entry.Name() + directory := filepath.Join(root, sessionID) + if _, ok := b.scanner.managedStates[sessionID]; ok { + maintenance, err := b.sessionMaintenanceActive(sessionID) + if err != nil { + return err + } + if maintenance || b.scanner.journalPending[filepath.Join(b.scanner.store, "fs", "sessions", sessionID)] { + continue + } + } + files, err := os.ReadDir(directory) + if err != nil { + return err + } + var generations []generationEntry + for _, file := range files { + if file.IsDir() || filepath.Ext(file.Name()) != ".json" { + continue + } + info, err := file.Info() + if err != nil { + return err + } + sequence, sequenced := manifestGenerationSequence(file.Name()) + generations = append(generations, generationEntry{path: filepath.Join(directory, file.Name()), name: file.Name(), modTime: info.ModTime(), sequence: sequence, sequenced: sequenced}) + } + if len(generations) <= 1 { + continue + } + current := "" + currentSequence := uint64(0) + currentSequenced := false + if state, ok := b.scanner.managedStates[sessionID]; ok { + current = cleanAbsolutePath(state.ManifestPath) + if pathWithin(directory, current) { + currentSequence, currentSequenced = manifestGenerationSequence(filepath.Base(current)) + } + } else if _, ok := b.scanner.primaryManifests[sessionID]; ok { + current = filepath.Join(b.scanner.store, "manifests", sessionID+".json") + } else { + continue + } + sortGenerationEntries(generations) + previousToKeep := max(0, b.options.KeepManifestGenerations-1) + for _, generation := range generations { + if generation.path == current { + continue + } + if currentSequenced { + if !generation.sequenced { + continue + } + if generation.sequence > currentSequence { + if err := b.add(generation.path, CandidateManifestGeneration); err != nil { + return err + } + continue + } + } + if previousToKeep > 0 { + previousToKeep-- + continue + } + if err := b.add(generation.path, CandidateManifestGeneration); err != nil { + return err + } + } + } + return nil +} + +func (b *gcBuilder) sessionMaintenanceActive(sessionID string) (bool, error) { + directory := filepath.Join(b.scanner.store, "fs", "sessions", sessionID) + writerActive, err := FileHasActiveLock(filepath.Join(directory, "writer.lease")) + if err != nil { + return false, err + } + readerActive, err := treeHasActiveLease(filepath.Join(directory, "leases")) + if err != nil { + return false, err + } + return writerActive || readerActive, nil +} + +func manifestGenerationSequence(name string) (uint64, bool) { + if filepath.Ext(name) != ".json" { + return 0, false + } + sequence, err := strconv.ParseUint(strings.TrimSuffix(name, ".json"), 10, 64) + return sequence, err == nil +} + +func (b *gcBuilder) discoverSessionGenerations() error { + for sessionID, state := range b.scanner.managedStates { + if err := b.ctx.Err(); err != nil { + return err + } + directory := filepath.Join(b.scanner.store, "fs", "sessions", sessionID) + writerActive, err := FileHasActiveLock(filepath.Join(directory, "writer.lease")) + if err != nil { + return err + } + readerActive, err := treeHasActiveLease(filepath.Join(directory, "leases")) + if err != nil { + return err + } + if writerActive || readerActive || b.scanner.journalPending[filepath.Clean(directory)] { + continue + } + current := map[string]struct{}{cleanAbsolutePath(state.DeltaPath): {}} + if state.BackingPath != "" { + current[cleanAbsolutePath(state.BackingPath)] = struct{}{} + } + entries, err := os.ReadDir(directory) + if err != nil { + return err + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + path := filepath.Join(directory, entry.Name()) + if _, keep := current[path]; keep { + continue + } + if _, owned := b.scanner.journalOwned[path]; owned { + continue + } + if isSessionGenerationData(b.scanner.store, path) { + if err := b.add(path, CandidateSessionGeneration); err != nil { + return err + } + } + } + } + return nil +} + +func (b *gcBuilder) discoverRetiredState() error { + root := filepath.Join(b.scanner.store, "fs", "retired") + entries, err := os.ReadDir(root) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + groups := make(map[string][]generationEntry) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + directory := filepath.Join(root, entry.Name()) + data, err := os.ReadFile(filepath.Join(directory, "state.json")) + if err != nil { + continue + } + var state struct { + SessionID string `json:"session_id"` + } + if json.Unmarshal(data, &state) != nil || state.SessionID == "" { + continue + } + active, err := treeHasActiveLease(filepath.Join(directory, "leases")) + if err != nil { + return err + } + pending, err := journalPendingAt(directory) + if err != nil { + return err + } + if active || pending { + continue + } + info, err := entry.Info() + if err != nil { + return err + } + groups[state.SessionID] = append(groups[state.SessionID], generationEntry{path: directory, name: entry.Name(), modTime: info.ModTime()}) + } + for _, states := range groups { + sortGenerationEntries(states) + for index := b.options.KeepRetiredPerSession; index < len(states); index++ { + if err := b.add(states[index].path, CandidateRetiredState); err != nil { + return err + } + } + } + return nil +} + +func (b *gcBuilder) discoverTemporaryFiles() error { + cutoff := b.options.Now().Add(-b.options.TemporaryGrace) + retiredRoot := filepath.Join(b.scanner.store, "fs", "retired") + return filepath.WalkDir(b.scanner.store, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := b.ctx.Err(); err != nil { + return err + } + if path == b.scanner.store { + return nil + } + if b.covered(path) { + if entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if pathWithin(retiredRoot, path) { + if entry.IsDir() && path != retiredRoot { + return filepath.SkipDir + } + return nil + } + if _, owned := b.scanner.journalOwned[cleanAbsolutePath(path)]; owned { + return nil + } + if !isUnownedTemporary(path) { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + if info.ModTime().After(cutoff) { + return nil + } + if err := b.add(path, CandidateTemporary); err != nil { + return err + } + if entry.IsDir() { + return filepath.SkipDir + } + return nil + }) +} + +func (b *gcBuilder) revalidate(candidate GCCandidate) (bool, error) { + switch candidate.Kind { + case CandidatePackGeneration: + data, err := os.ReadFile(filepath.Join(b.scanner.store, "packs", "CURRENT")) + if err != nil { + return false, err + } + if filepath.Base(candidate.Path) == strings.TrimSpace(string(data)) { + return false, nil + } + active, err := DirectoryHasActiveLease(filepath.Join(candidate.Path, "leases"), false) + return !active, err + case CandidateSessionGeneration: + directory := filepath.Dir(candidate.Path) + stateData, err := os.ReadFile(filepath.Join(directory, "state.json")) + if err != nil { + return false, err + } + var state stateRecord + if err := json.Unmarshal(stateData, &state); err != nil { + return false, err + } + if cleanAbsolutePath(state.DeltaPath) == candidate.Path || (state.BackingPath != "" && cleanAbsolutePath(state.BackingPath) == candidate.Path) { + return false, nil + } + writerActive, err := FileHasActiveLock(filepath.Join(directory, "writer.lease")) + if err != nil || writerActive { + return false, err + } + readerActive, err := treeHasActiveLease(filepath.Join(directory, "leases")) + return !readerActive, err + case CandidateTemporary: + info, err := os.Stat(candidate.Path) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + return !info.ModTime().After(b.options.Now().Add(-b.options.TemporaryGrace)), nil + case CandidateRetiredState: + active, err := treeHasActiveLease(filepath.Join(candidate.Path, "leases")) + if err != nil || active { + return false, err + } + pending, err := journalPendingAt(candidate.Path) + return !pending, err + case CandidateManifestGeneration: + for _, state := range b.scanner.managedStates { + if cleanAbsolutePath(state.ManifestPath) == candidate.Path { + return false, nil + } + } + return true, nil + default: + return false, errors.New("unknown storage GC candidate kind") + } +} + +func sortGenerationEntries(entries []generationEntry) { + sort.Slice(entries, func(i, j int) bool { + if entries[i].sequenced && entries[j].sequenced && entries[i].sequence != entries[j].sequence { + return entries[i].sequence > entries[j].sequence + } + if entries[i].sequenced != entries[j].sequenced { + return entries[i].sequenced + } + if entries[i].modTime.Equal(entries[j].modTime) { + return entries[i].name > entries[j].name + } + return entries[i].modTime.After(entries[j].modTime) + }) +} + +func treeHasActiveLease(root string) (bool, error) { + active := false + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { + if errors.Is(walkErr, os.ErrNotExist) { + return filepath.SkipDir + } + if walkErr != nil { + return walkErr + } + if !entry.IsDir() || path == root { + return nil + } + hasLease, err := DirectoryHasActiveLease(path, false) + if err != nil { + return err + } + if hasLease { + active = true + return filepath.SkipAll + } + return nil + }) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return active, err +} + +func journalPendingAt(directory string) (bool, error) { + path := filepath.Join(directory, "journal.jsonl") + file, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + defer file.Close() + decoder := json.NewDecoder(file) + latest := make(map[string]string) + for { + var record journalRecord + if err := decoder.Decode(&record); errors.Is(err, io.EOF) { + break + } else if err != nil { + return false, err + } + latest[record.OperationID] = record.Phase + } + for _, phase := range latest { + if phase != "complete" && phase != "rolled-back" { + return true, nil + } + } + return false, nil +} + +type candidatePhysical struct { + bytes int64 + links uint64 + candidateLinks uint64 + candidates map[int]struct{} +} + +func describeCandidates(paths map[string]CandidateKind) ([]GCCandidate, int64, error) { + ordered := make([]string, 0, len(paths)) + for path := range paths { + ordered = append(ordered, path) + } + sort.Strings(ordered) + candidates := make([]GCCandidate, len(ordered)) + physical := make(map[string]*candidatePhysical) + for index, path := range ordered { + candidate := GCCandidate{Kind: paths[path], Path: path} + localPhysical := make(map[string]struct{}) + err := filepath.Walk(path, func(filePath string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } + if !info.Mode().IsRegular() { + return nil + } + identity, bytes, err := physicalFile(filePath, info) + if err != nil { + return err + } + candidate.Files++ + candidate.ApparentBytes += info.Size() + if _, seen := localPhysical[identity]; !seen { + candidate.PhysicalBytes += bytes + localPhysical[identity] = struct{}{} + } + item := physical[identity] + if item == nil { + links, err := physicalLinkCount(filePath, info) + if err != nil { + return err + } + item = &candidatePhysical{bytes: bytes, links: links, candidates: make(map[int]struct{})} + physical[identity] = item + } + item.candidateLinks++ + item.candidates[index] = struct{}{} + return nil + }) + if err != nil { + return nil, 0, err + } + candidates[index] = candidate + } + var projected int64 + for _, item := range physical { + if item.candidateLinks < item.links { + continue + } + projected += item.bytes + first := len(candidates) + for index := range item.candidates { + if index < first { + first = index + } + } + if first < len(candidates) { + candidates[first].ReclaimableBytes += item.bytes + } + } + return candidates, projected, nil +} diff --git a/internal/storage/gc_test.go b/internal/storage/gc_test.go new file mode 100644 index 0000000..4289aa8 --- /dev/null +++ b/internal/storage/gc_test.go @@ -0,0 +1,211 @@ +package storage + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +func TestCollectBoundsGenerationsRetiredStateAndTemporaryFiles(t *testing.T) { + store := filepath.Join(t.TempDir(), "store") + now := time.Unix(2_000_000, 0) + old := now.Add(-2 * time.Hour) + + writeBytesFile(t, filepath.Join(store, "packs", "CURRENT"), []byte("gen-3\n")) + for _, generation := range []string{"gen-0", "gen-1", "gen-2", "gen-3"} { + path := writeSizedFile(t, filepath.Join(store, "packs", generation, "pack-000001.pack"), 16) + setModTime(t, path, old.Add(time.Duration(generation[len(generation)-1]-'0')*time.Minute)) + writeJSONFile(t, filepath.Join(store, "packs", generation, "index.json"), map[string]any{"generation": generation}) + } + leased, err := AcquireLease(filepath.Join(store, "packs", "gen-0", "leases"), "resolver") + if err != nil { + t.Fatal(err) + } + + manifestRoot := filepath.Join(store, "manifests", "generations", "session") + for generation := 1; generation <= 3; generation++ { + writeJSONFile(t, filepath.Join(manifestRoot, string(rune('0'+generation))+".json"), manifestFixture("session", filepath.Join(store, "native.jsonl"), int64(generation*10))) + } + sessionDir := filepath.Join(store, "fs", "sessions", "session") + currentDelta := writeSizedFile(t, filepath.Join(sessionDir, "delta-00000000000000000003.jsonl"), 3) + oldDelta := writeSizedFile(t, filepath.Join(sessionDir, "delta-00000000000000000001.jsonl"), 7) + oldBacking := writeSizedFile(t, filepath.Join(sessionDir, "backing-00000000000000000002.jsonl"), 9) + writeJSONFile(t, filepath.Join(sessionDir, "state.json"), stateFixture( + "session", filepath.Join(manifestRoot, "3.json"), 30, currentDelta, "", "", + )) + + for index := 1; index <= 3; index++ { + directory := filepath.Join(store, "fs", "retired", "retired-"+string(rune('0'+index))) + writeJSONFile(t, filepath.Join(directory, "state.json"), map[string]any{"session_id": "session"}) + setModTime(t, directory, old.Add(time.Duration(index)*time.Minute)) + } + oldTemp := writeSizedFile(t, filepath.Join(store, "fs", "sessions", "session", ".backing-abandoned.tmp"), 11) + recentTemp := writeSizedFile(t, filepath.Join(store, "fs", "sessions", "session", ".state-recent.tmp"), 13) + setModTime(t, oldTemp, old) + setModTime(t, recentTemp, now.Add(-10*time.Minute)) + setModTime(t, oldDelta, old) + setModTime(t, oldBacking, old) + + options := GCOptions{ + StoreDir: store, TemporaryGrace: time.Hour, Now: func() time.Time { return now }, + KeepPackGenerations: 2, KeepManifestGenerations: 2, KeepRetiredPerSession: 1, + } + dry, err := Collect(context.Background(), options) + if err != nil { + t.Fatalf("Collect dry-run: %v", err) + } + if !dry.DryRun || dry.CandidateCount != 7 || dry.RemovedCount != 0 || dry.ProjectedReclaimableBytes <= 0 || dry.ActualReclaimedBytes != 0 { + t.Fatalf("unexpected dry-run result: %#v", dry) + } + for _, path := range []string{filepath.Join(store, "packs", "gen-1"), oldDelta, oldBacking, oldTemp, filepath.Join(manifestRoot, "1.json")} { + if _, err := os.Stat(path); err != nil { + t.Fatalf("dry-run removed %s: %v", path, err) + } + } + + options.Apply = true + applied, err := Collect(context.Background(), options) + if err != nil { + t.Fatalf("Collect apply: %v", err) + } + if applied.RemovedCount != dry.CandidateCount || applied.ActualReclaimedBytes <= 0 { + t.Fatalf("unexpected apply result: %#v", applied) + } + for _, path := range []string{filepath.Join(store, "packs", "gen-3"), filepath.Join(store, "packs", "gen-2"), filepath.Join(store, "packs", "gen-0"), currentDelta, filepath.Join(manifestRoot, "3.json"), filepath.Join(manifestRoot, "2.json"), filepath.Join(store, "fs", "retired", "retired-3"), recentTemp} { + if _, err := os.Stat(path); err != nil { + t.Fatalf("retained path missing %s: %v", path, err) + } + } + + if err := leased.Close(); err != nil { + t.Fatal(err) + } + second, err := Collect(context.Background(), options) + if err != nil { + t.Fatalf("Collect after lease close: %v", err) + } + if second.RemovedCount != 1 { + t.Fatalf("closed leased generation was not collected: %#v", second) + } + third, err := Collect(context.Background(), options) + if err != nil || third.RemovedCount != 0 { + t.Fatalf("repeated Collect is not idempotent: %#v err=%v", third, err) + } +} + +func TestCollectKeepsSoleRecoveryStateAndJournalOwnedFiles(t *testing.T) { + store := filepath.Join(t.TempDir(), "store") + now := time.Unix(3_000_000, 0) + manifest := filepath.Join(store, "manifests", "generations", "session", "1.json") + writeJSONFile(t, manifest, manifestFixture("session", filepath.Join(store, "native.jsonl"), 5)) + sessionDir := filepath.Join(store, "fs", "sessions", "session") + delta := writeSizedFile(t, filepath.Join(sessionDir, "delta.jsonl"), 5) + scratch := writeSizedFile(t, filepath.Join(sessionDir, ".compact-00000000000000000001.jsonl"), 5) + writeJSONFile(t, filepath.Join(sessionDir, "state.json"), stateFixture("session", manifest, 0, delta, "", "")) + writeJSONLine(t, filepath.Join(sessionDir, "journal.jsonl"), map[string]any{ + "operation_id": "compact-1", "phase": "prepared", "native": map[string]any{"path": scratch}, + }) + writeJSONFile(t, filepath.Join(store, "fs", "retired", "only", "state.json"), map[string]any{"session_id": "session"}) + writeSizedFile(t, filepath.Join(store, "packs", "gen-only", "pack-000001.pack"), 5) + setModTime(t, scratch, now.Add(-24*time.Hour)) + + result, err := Collect(context.Background(), GCOptions{StoreDir: store, Apply: true, TemporaryGrace: time.Hour, Now: func() time.Time { return now }}) + if err != nil { + t.Fatalf("Collect: %v", err) + } + if result.RemovedCount != 0 { + t.Fatalf("sole recovery state was removed: %#v", result) + } + for _, path := range []string{manifest, delta, scratch, filepath.Join(store, "fs", "retired", "only"), filepath.Join(store, "packs", "gen-only")} { + if _, err := os.Stat(path); err != nil { + t.Fatalf("protected path missing %s: %v", path, err) + } + } +} + +func TestCollectKeepsTruePreviousManifestAndRemovesAbandonedFuture(t *testing.T) { + store := filepath.Join(t.TempDir(), "store") + manifestRoot := filepath.Join(store, "manifests", "generations", "session") + for generation := 1; generation <= 3; generation++ { + writeJSONFile(t, filepath.Join(manifestRoot, string(rune('0'+generation))+".json"), manifestFixture("session", filepath.Join(store, "native.jsonl"), int64(generation))) + } + sessionDir := filepath.Join(store, "fs", "sessions", "session") + delta := writeSizedFile(t, filepath.Join(sessionDir, "delta-00000000000000000002.jsonl"), 2) + state := stateFixture("session", filepath.Join(manifestRoot, "2.json"), 2, delta, "", "") + state["generation"] = 2 + writeJSONFile(t, filepath.Join(sessionDir, "state.json"), state) + writeJSONLine(t, filepath.Join(sessionDir, "journal.jsonl"), map[string]any{"operation_id": "compact-2", "phase": "prepared"}) + + blocked, err := Collect(context.Background(), GCOptions{StoreDir: store, Apply: true}) + if err != nil { + t.Fatal(err) + } + if blocked.RemovedCount != 0 { + t.Fatalf("pending compaction allowed manifest cleanup: %#v", blocked) + } + writeJSONLine(t, filepath.Join(sessionDir, "journal.jsonl"), map[string]any{"operation_id": "compact-2", "phase": "rolled-back"}) + collected, err := Collect(context.Background(), GCOptions{StoreDir: store, Apply: true}) + if err != nil { + t.Fatal(err) + } + if collected.RemovedCount != 1 { + t.Fatalf("abandoned future manifest was not collected: %#v", collected) + } + if _, err := os.Stat(filepath.Join(manifestRoot, "3.json")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("abandoned future manifest remains: %v", err) + } + for _, name := range []string{"1.json", "2.json"} { + if _, err := os.Stat(filepath.Join(manifestRoot, name)); err != nil { + t.Fatalf("current/previous manifest missing %s: %v", name, err) + } + } +} + +func TestCollectReportsZeroPhysicalReclamationForRemainingHardLink(t *testing.T) { + store := t.TempDir() + keep := writeSizedFile(t, filepath.Join(store, "keep.bin"), 4096) + temporary := filepath.Join(store, ".backing-abandoned.tmp") + if err := os.Link(keep, temporary); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-2 * time.Hour) + setModTime(t, temporary, old) + result, err := Collect(context.Background(), GCOptions{StoreDir: store, Apply: true, TemporaryGrace: time.Hour, Now: time.Now}) + if err != nil { + t.Fatalf("Collect: %v", err) + } + if result.RemovedCount != 1 || result.ProjectedReclaimableBytes != 0 || result.ActualReclaimedBytes != 0 { + t.Fatalf("hard-link reclamation was overstated: %#v", result) + } + if _, err := os.Stat(keep); err != nil { + t.Fatalf("retained hard link missing: %v", err) + } + if _, err := os.Stat(temporary); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("temporary hard link remains: %v", err) + } +} + +func setModTime(t *testing.T, path string, modTime time.Time) { + t.Helper() + if err := os.Chtimes(path, modTime, modTime); err != nil { + t.Fatal(err) + } +} + +func writeMinimalState(t *testing.T, path string, sessionID string) { + t.Helper() + data, err := json.Marshal(map[string]any{"session_id": sessionID}) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } +} diff --git a/internal/storage/guard.go b/internal/storage/guard.go new file mode 100644 index 0000000..b75fe4b --- /dev/null +++ b/internal/storage/guard.go @@ -0,0 +1,99 @@ +package storage + +import ( + "context" + "errors" + "fmt" + "path/filepath" +) + +type Projection struct { + Operation string `json:"operation"` + AdditionalPersistentBytes int64 `json:"additional_persistent_bytes"` + TemporaryBytes int64 `json:"temporary_bytes"` + TemporaryPersistentOverlapBytes int64 `json:"temporary_persistent_overlap_bytes"` + ReclaimableBytes int64 `json:"reclaimable_bytes"` +} + +type SpaceProbe func(path string) (int64, error) + +type Guard struct { + StoreDir string + Limits Limits + Probe SpaceProbe +} + +type VolumeGuard struct { + Path string + Limits Limits + Probe SpaceProbe +} + +type Assessment struct { + Inventory Inventory `json:"inventory"` + Budget BudgetReport `json:"budget"` +} + +func (g Guard) Check(ctx context.Context, projection Projection) (Assessment, error) { + if err := ctx.Err(); err != nil { + return Assessment{}, err + } + if g.StoreDir == "" { + return Assessment{}, errors.New("storage guard store directory is required") + } + store, err := filepath.Abs(g.StoreDir) + if err != nil { + return Assessment{}, err + } + store = filepath.Clean(store) + inventory, err := Scan(ctx, Options{StoreDir: store, AllowMetadataIssues: true}) + if err != nil { + return Assessment{}, err + } + probe := g.Probe + if probe == nil { + probe = AvailableBytes + } + available, err := probe(store) + if err != nil { + return Assessment{Inventory: inventory}, fmt.Errorf("probe available storage bytes: %w", err) + } + report, err := CheckBudget(BudgetRequest{ + Operation: projection.Operation, + CurrentPhysicalBytes: inventory.TotalPhysicalBytes, + AdditionalPersistentBytes: projection.AdditionalPersistentBytes, + TemporaryBytes: projection.TemporaryBytes, + TemporaryPersistentOverlapBytes: projection.TemporaryPersistentOverlapBytes, + ReclaimableBytes: projection.ReclaimableBytes, + AvailableBytes: available, + }, g.Limits) + return Assessment{Inventory: inventory, Budget: report}, err +} + +func (g VolumeGuard) Check(ctx context.Context, projection Projection) (Assessment, error) { + if err := ctx.Err(); err != nil { + return Assessment{}, err + } + if g.Path == "" { + return Assessment{}, errors.New("volume guard path is required") + } + path := filepath.Clean(g.Path) + probe := g.Probe + if probe == nil { + probe = AvailableBytes + } + available, err := probe(path) + if err != nil { + return Assessment{}, fmt.Errorf("probe available storage bytes: %w", err) + } + limits := g.Limits + if limits == (Limits{}) { + limits = DefaultLimits + } + report, err := CheckBudget(BudgetRequest{ + Operation: projection.Operation, AdditionalPersistentBytes: projection.AdditionalPersistentBytes, + TemporaryBytes: projection.TemporaryBytes, TemporaryPersistentOverlapBytes: projection.TemporaryPersistentOverlapBytes, + ReclaimableBytes: projection.ReclaimableBytes, AvailableBytes: available, + }, limits) + return Assessment{Budget: report}, err +} diff --git a/internal/storage/guard_test.go b/internal/storage/guard_test.go new file mode 100644 index 0000000..4999fe9 --- /dev/null +++ b/internal/storage/guard_test.go @@ -0,0 +1,45 @@ +package storage + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" +) + +func TestGuardScansCurrentFootprintAndChecksLiveFreeSpace(t *testing.T) { + store := filepath.Join(t.TempDir(), "store") + if err := os.MkdirAll(store, 0o700); err != nil { + t.Fatal(err) + } + writeSizedFile(t, filepath.Join(store, "metadata.bin"), 32) + var probed string + guard := Guard{ + StoreDir: store, + Limits: Limits{FreeSpaceReserveBytes: 95}, + Probe: func(path string) (int64, error) { + probed = path + return 100, nil + }, + } + assessment, err := guard.Check(context.Background(), Projection{Operation: "materialize", AdditionalPersistentBytes: 6}) + if !errors.Is(err, ErrBudgetExceeded) { + t.Fatalf("Guard.Check error = %v, want ErrBudgetExceeded", err) + } + if probed != filepath.Clean(store) { + t.Fatalf("space probe path = %q, want %q", probed, filepath.Clean(store)) + } + if assessment.Inventory.TotalPhysicalBytes == 0 || assessment.Budget.CurrentPhysicalBytes != assessment.Inventory.TotalPhysicalBytes { + t.Fatalf("assessment did not use scanned physical bytes: %#v", assessment) + } +} + +func TestGuardPropagatesSpaceProbeFailure(t *testing.T) { + store := t.TempDir() + want := errors.New("probe failed") + guard := Guard{StoreDir: store, Probe: func(string) (int64, error) { return 0, want }} + if _, err := guard.Check(context.Background(), Projection{Operation: "pack"}); !errors.Is(err, want) { + t.Fatalf("Guard.Check error = %v, want %v", err, want) + } +} diff --git a/internal/storage/inventory.go b/internal/storage/inventory.go new file mode 100644 index 0000000..ea47db6 --- /dev/null +++ b/internal/storage/inventory.go @@ -0,0 +1,599 @@ +package storage + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "math" + "os" + "path/filepath" + "strings" +) + +type FileUsage struct { + Files int `json:"files"` + ApparentBytes int64 `json:"apparent_bytes"` + PhysicalBytes int64 `json:"physical_bytes"` +} + +type Inventory struct { + StoreDir string `json:"store_dir"` + LogicalSessionBytes int64 `json:"logical_session_bytes"` + UniqueLooseObjects FileUsage `json:"unique_loose_objects"` + Packs FileUsage `json:"packs"` + NativeSources FileUsage `json:"native_sources"` + RetainedSnapshots FileUsage `json:"retained_snapshots"` + CurrentFallbacks FileUsage `json:"current_fallbacks"` + ActiveDeltas FileUsage `json:"active_deltas"` + WritableBackings FileUsage `json:"writable_backings"` + OldGenerations FileUsage `json:"old_generations"` + RetirementState FileUsage `json:"retirement_state"` + JournalRecovery FileUsage `json:"journal_recovery"` + UnownedTemporary FileUsage `json:"unowned_temporary"` + Metadata FileUsage `json:"metadata"` + TotalFiles int `json:"total_files"` + UniquePhysicalFiles int `json:"unique_physical_files"` + HardlinkAliases int `json:"hardlink_aliases"` + TotalApparentBytes int64 `json:"total_apparent_bytes"` + TotalPhysicalBytes int64 `json:"total_physical_bytes"` + IssueCount int `json:"issue_count"` + Issues []string `json:"issues,omitempty"` +} + +type Options struct { + StoreDir string + AllowMetadataIssues bool +} + +type manifestRecord struct { + Session struct { + ID string `json:"id"` + RolloutPath string `json:"rollout_path"` + } `json:"session"` + Source struct { + Bytes int64 `json:"bytes"` + } `json:"source"` +} + +type stateRecord struct { + SessionID string `json:"session_id"` + Generation uint64 `json:"generation"` + ManifestPath string `json:"manifest_path"` + BaseBytes int64 `json:"base_bytes"` + DeltaPath string `json:"delta_path"` + BackingPath string `json:"backing_path"` + Native struct { + Path string `json:"path"` + } `json:"native_snapshot"` +} + +type journalRecord struct { + OperationID string `json:"operation_id"` + Phase string `json:"phase"` + TempPath string `json:"temp_path"` + FinalPath string `json:"final_path"` + Native struct { + Path string `json:"path"` + } `json:"native"` +} + +type scanner struct { + ctx context.Context + store string + canonicalStore string + nestedMounts map[string]struct{} + result Inventory + physicalFiles map[string]struct{} + primaryManifests map[string]manifestRecord + managedStates map[string]stateRecord + activeDeltas map[string]struct{} + backings map[string]struct{} + snapshots map[string]struct{} + journalOwned map[string]struct{} + journalPending map[string]bool + currentPack string + allowMetadataIssues bool +} + +func Scan(ctx context.Context, options Options) (Inventory, error) { + s, exists, err := prepareScanner(ctx, options) + if err != nil { + return Inventory{}, err + } + if !exists { + return Inventory{StoreDir: cleanAbsolutePath(options.StoreDir)}, nil + } + if err := s.calculateLogicalBytes(); err != nil { + return Inventory{}, err + } + if err := s.walkStore(); err != nil { + return Inventory{}, err + } + if err := s.addExternalReferences(); err != nil { + return Inventory{}, err + } + return s.result, nil +} + +func prepareScanner(ctx context.Context, options Options) (*scanner, bool, error) { + if options.StoreDir == "" { + return nil, false, errors.New("storage inventory store directory is required") + } + store, err := filepath.Abs(options.StoreDir) + if err != nil { + return nil, false, fmt.Errorf("resolve storage inventory root: %w", err) + } + store = filepath.Clean(store) + if info, err := os.Stat(store); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, false, nil + } + return nil, false, fmt.Errorf("stat storage inventory root: %w", err) + } else if !info.IsDir() { + return nil, false, errors.New("storage inventory root is not a directory") + } + canonicalStore := store + if resolved, err := filepath.EvalSymlinks(store); err == nil { + canonicalStore = filepath.Clean(resolved) + } + nestedMounts, err := nestedMountPoints(canonicalStore) + if err != nil { + return nil, false, fmt.Errorf("inspect nested storage mounts: %w", err) + } + s := &scanner{ + ctx: ctx, store: store, result: Inventory{StoreDir: store}, + canonicalStore: canonicalStore, nestedMounts: nestedMounts, + physicalFiles: make(map[string]struct{}), primaryManifests: make(map[string]manifestRecord), + managedStates: make(map[string]stateRecord), activeDeltas: make(map[string]struct{}), + backings: make(map[string]struct{}), snapshots: make(map[string]struct{}), + journalOwned: make(map[string]struct{}), journalPending: make(map[string]bool), + allowMetadataIssues: options.AllowMetadataIssues, + } + if err := s.loadManifests(); err != nil { + return nil, false, err + } + if err := s.loadStatesAndJournals(); err != nil { + return nil, false, err + } + if err := s.loadCurrentPack(); err != nil { + return nil, false, err + } + return s, true, nil +} + +func (s *scanner) loadManifests() error { + root := filepath.Join(s.store, "manifests") + return walkIfPresent(root, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := s.ctx.Err(); err != nil { + return err + } + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { + return nil + } + manifest, err := decodeManifest(path) + if err != nil { + return s.metadataIssue(err) + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + if filepath.Dir(relative) != "." { + return nil + } + if manifest.Session.ID == "" || manifest.Source.Bytes < 0 { + return s.metadataIssue(fmt.Errorf("invalid primary manifest %s", path)) + } + if _, exists := s.primaryManifests[manifest.Session.ID]; exists { + return s.metadataIssue(fmt.Errorf("duplicate primary manifest for session %s", manifest.Session.ID)) + } + s.primaryManifests[manifest.Session.ID] = manifest + return nil + }) +} + +func (s *scanner) loadStatesAndJournals() error { + root := filepath.Join(s.store, "fs", "sessions") + entries, err := os.ReadDir(root) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("read managed session directories: %w", err) + } + for _, entry := range entries { + if err := s.ctx.Err(); err != nil { + return err + } + if !entry.IsDir() { + continue + } + directory := filepath.Join(root, entry.Name()) + statePath := filepath.Join(directory, "state.json") + data, err := os.ReadFile(statePath) + if err != nil { + return fmt.Errorf("read managed session state %s: %w", entry.Name(), err) + } + var state stateRecord + if err := json.Unmarshal(data, &state); err != nil { + return fmt.Errorf("decode managed session state %s: %w", entry.Name(), err) + } + if state.SessionID != entry.Name() || state.Generation == 0 || state.BaseBytes < 0 || state.DeltaPath == "" { + return fmt.Errorf("invalid managed session state %s", statePath) + } + state.DeltaPath, err = cleanPathWithin(directory, state.DeltaPath) + if err != nil { + return fmt.Errorf("invalid managed delta for %s: %w", state.SessionID, err) + } + if state.BackingPath != "" { + state.BackingPath, err = cleanPathWithin(directory, state.BackingPath) + if err != nil { + return fmt.Errorf("invalid writable backing for %s: %w", state.SessionID, err) + } + s.backings[state.BackingPath] = struct{}{} + } else { + s.activeDeltas[state.DeltaPath] = struct{}{} + } + if state.Native.Path != "" { + state.Native.Path = cleanAbsolutePath(state.Native.Path) + s.snapshots[state.Native.Path] = struct{}{} + } + s.managedStates[state.SessionID] = state + if err := s.loadJournal(directory); err != nil { + return err + } + } + return nil +} + +func (s *scanner) loadJournal(directory string) error { + path := filepath.Join(directory, "journal.jsonl") + file, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("open session journal: %w", err) + } + defer file.Close() + latest := make(map[string]journalRecord) + lines := bufio.NewScanner(file) + lines.Buffer(make([]byte, 64*1024), 4*1024*1024) + for lines.Scan() { + var record journalRecord + if err := json.Unmarshal(lines.Bytes(), &record); err != nil { + return fmt.Errorf("decode session journal %s: %w", path, err) + } + if record.OperationID == "" { + return fmt.Errorf("session journal %s contains a record without an operation ID", path) + } + latest[record.OperationID] = record + } + if err := lines.Err(); err != nil { + return fmt.Errorf("read session journal %s: %w", path, err) + } + for _, record := range latest { + if record.Phase == "complete" || record.Phase == "rolled-back" { + continue + } + s.journalPending[filepath.Clean(directory)] = true + for _, candidate := range []string{record.TempPath, record.FinalPath, record.Native.Path} { + if candidate == "" { + continue + } + clean, err := cleanPathWithin(directory, candidate) + if err != nil { + return fmt.Errorf("unsafe journal-owned recovery path: %w", err) + } + s.journalOwned[clean] = struct{}{} + } + } + return nil +} + +func (s *scanner) calculateLogicalBytes() error { + for sessionID, state := range s.managedStates { + var bytes int64 + if state.BackingPath != "" { + info, err := os.Stat(state.BackingPath) + if err != nil { + return fmt.Errorf("stat writable backing for %s: %w", sessionID, err) + } + bytes = info.Size() + } else { + info, err := os.Stat(state.DeltaPath) + if err != nil { + return fmt.Errorf("stat active delta for %s: %w", sessionID, err) + } + var overflow bool + bytes, overflow = addInt64(state.BaseBytes, info.Size()) + if overflow { + return fmt.Errorf("logical bytes overflow for managed session %s", sessionID) + } + } + var overflow bool + s.result.LogicalSessionBytes, overflow = addInt64(s.result.LogicalSessionBytes, bytes) + if overflow { + return errors.New("logical session byte total overflow") + } + } + for sessionID, manifest := range s.primaryManifests { + if _, managed := s.managedStates[sessionID]; managed { + continue + } + var overflow bool + s.result.LogicalSessionBytes, overflow = addInt64(s.result.LogicalSessionBytes, manifest.Source.Bytes) + if overflow { + return errors.New("logical session byte total overflow") + } + } + return nil +} + +func (s *scanner) loadCurrentPack() error { + data, err := os.ReadFile(filepath.Join(s.store, "packs", "CURRENT")) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("read current pack generation: %w", err) + } + s.currentPack = strings.TrimSpace(string(data)) + if s.currentPack == "" || filepath.Base(s.currentPack) != s.currentPack || s.currentPack == "." || s.currentPack == ".." { + s.currentPack = "" + return s.metadataIssue(errors.New("invalid current pack generation")) + } + return nil +} + +func (s *scanner) metadataIssue(err error) error { + if !s.allowMetadataIssues { + return err + } + s.result.Issues = append(s.result.Issues, err.Error()) + s.result.IssueCount = len(s.result.Issues) + return nil +} + +func (s *scanner) walkStore() error { + return filepath.WalkDir(s.store, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := s.ctx.Err(); err != nil { + return err + } + if entry.IsDir() { + if s.isNestedMount(path) { + return filepath.SkipDir + } + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + usage := s.classifyStorePath(path) + return s.addFile(usage, path, info) + }) +} + +func (s *scanner) isNestedMount(path string) bool { + if path == s.store || len(s.nestedMounts) == 0 { + return false + } + relative, err := filepath.Rel(s.store, path) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return false + } + _, mounted := s.nestedMounts[filepath.Clean(filepath.Join(s.canonicalStore, relative))] + return mounted +} + +func (s *scanner) classifyStorePath(path string) *FileUsage { + path = filepath.Clean(path) + if _, ok := s.backings[path]; ok { + return &s.result.WritableBackings + } + if _, ok := s.activeDeltas[path]; ok { + return &s.result.ActiveDeltas + } + if _, ok := s.journalOwned[path]; ok { + return &s.result.JournalRecovery + } + if pathWithin(filepath.Join(s.store, "fs", "retired"), path) { + return &s.result.RetirementState + } + if pathWithin(filepath.Join(s.store, "fs", "snapshots"), path) { + return &s.result.RetainedSnapshots + } + if pathWithin(filepath.Join(s.store, "fs", "fallbacks"), path) && isCurrentFallback(path) { + return &s.result.CurrentFallbacks + } + if isUnownedTemporary(path) { + return &s.result.UnownedTemporary + } + if pathWithin(filepath.Join(s.store, "objects"), path) && filepath.Ext(path) == ".zst" { + return &s.result.UniqueLooseObjects + } + if generation, ok := packGeneration(s.store, path); ok { + if generation == s.currentPack { + return &s.result.Packs + } + return &s.result.OldGenerations + } + if pathWithin(filepath.Join(s.store, "manifests", "generations"), path) { + return &s.result.OldGenerations + } + if isSessionGenerationData(s.store, path) { + return &s.result.OldGenerations + } + return &s.result.Metadata +} + +func (s *scanner) addExternalReferences() error { + for path := range s.snapshots { + if pathWithin(s.store, path) { + continue + } + if err := s.addExternalFile(&s.result.RetainedSnapshots, path, true); err != nil { + return err + } + } + nativePaths := make(map[string]struct{}) + for _, manifest := range s.primaryManifests { + if manifest.Session.RolloutPath != "" { + nativePaths[cleanAbsolutePath(manifest.Session.RolloutPath)] = struct{}{} + } + } + for path := range nativePaths { + if pathWithin(s.store, path) { + continue + } + if err := s.addExternalFile(&s.result.NativeSources, path, false); err != nil { + return err + } + } + return nil +} + +func (s *scanner) addExternalFile(usage *FileUsage, path string, required bool) error { + info, err := os.Stat(path) + if errors.Is(err, os.ErrNotExist) && !required { + return nil + } + if err != nil { + return fmt.Errorf("stat referenced storage file %s: %w", path, err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("referenced storage path is not a regular file: %s", path) + } + return s.addFile(usage, path, info) +} + +func (s *scanner) addFile(usage *FileUsage, path string, info os.FileInfo) error { + identity, physicalBytes, err := physicalFile(path, info) + if err != nil { + return err + } + usage.Files++ + usage.ApparentBytes += info.Size() + s.result.TotalFiles++ + s.result.TotalApparentBytes += info.Size() + if _, exists := s.physicalFiles[identity]; exists { + s.result.HardlinkAliases++ + return nil + } + s.physicalFiles[identity] = struct{}{} + usage.PhysicalBytes += physicalBytes + s.result.UniquePhysicalFiles++ + s.result.TotalPhysicalBytes += physicalBytes + return nil +} + +func decodeManifest(path string) (manifestRecord, error) { + data, err := os.ReadFile(path) + if err != nil { + return manifestRecord{}, fmt.Errorf("read manifest %s: %w", path, err) + } + var manifest manifestRecord + if err := json.Unmarshal(data, &manifest); err != nil { + return manifestRecord{}, fmt.Errorf("decode manifest %s: %w", path, err) + } + return manifest, nil +} + +func walkIfPresent(root string, walk fs.WalkDirFunc) error { + err := filepath.WalkDir(root, walk) + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err +} + +func cleanAbsolutePath(path string) string { + if absolute, err := filepath.Abs(path); err == nil { + return filepath.Clean(absolute) + } + return filepath.Clean(path) +} + +func cleanPathWithin(root string, path string) (string, error) { + clean := cleanAbsolutePath(path) + if !pathWithin(root, clean) { + return "", errors.New("path escapes its managed root") + } + return clean, nil +} + +func pathWithin(root string, path string) bool { + root = cleanAbsolutePath(root) + path = cleanAbsolutePath(path) + relative, err := filepath.Rel(root, path) + return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} + +func packGeneration(store string, path string) (string, bool) { + relative, err := filepath.Rel(filepath.Join(store, "packs"), path) + if err != nil || relative == "." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", false + } + parts := strings.Split(relative, string(filepath.Separator)) + if len(parts) < 2 || parts[0] == "" || strings.HasPrefix(parts[0], ".") { + return "", false + } + return parts[0], true +} + +func isSessionGenerationData(store string, path string) bool { + relative, err := filepath.Rel(filepath.Join(store, "fs", "sessions"), path) + if err != nil || relative == "." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return false + } + parts := strings.Split(relative, string(filepath.Separator)) + if len(parts) != 2 { + return false + } + name := parts[1] + return strings.HasPrefix(name, "delta-") && strings.HasSuffix(name, ".jsonl") || + strings.HasPrefix(name, "backing-") && strings.HasSuffix(name, ".jsonl") || name == "delta.jsonl" +} + +func isCurrentFallback(path string) bool { + switch filepath.Base(path) { + case "fallback-current.jsonl", "quarantine-current.jsonl": + return true + default: + return false + } +} + +func isUnownedTemporary(path string) bool { + name := filepath.Base(path) + if !strings.HasPrefix(name, ".") { + return false + } + return strings.Contains(name, ".tmp") || strings.HasPrefix(name, ".generation-") || + strings.HasPrefix(name, ".compact-") || strings.HasPrefix(name, ".object-") || + strings.HasPrefix(name, ".manifest-") || strings.HasPrefix(name, ".materialize-") || + strings.HasPrefix(name, ".backing-") || strings.HasPrefix(name, ".CURRENT-") +} + +func addInt64(left int64, right int64) (int64, bool) { + if right > 0 && left > math.MaxInt64-right { + return 0, true + } + if right < 0 && left < math.MinInt64-right { + return 0, true + } + return left + right, false +} diff --git a/internal/storage/inventory_test.go b/internal/storage/inventory_test.go new file mode 100644 index 0000000..0caea14 --- /dev/null +++ b/internal/storage/inventory_test.go @@ -0,0 +1,222 @@ +package storage + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" +) + +func TestScanClassifiesManagedStorageAndDeduplicatesHardLinks(t *testing.T) { + root := t.TempDir() + store := filepath.Join(root, "store") + nativeRoot := filepath.Join(root, "native") + + nativeA := writeSizedFile(t, filepath.Join(nativeRoot, "managed-a.jsonl"), 13) + nativeB := writeSizedFile(t, filepath.Join(nativeRoot, "managed-b.jsonl"), 20) + nativeC := writeSizedFile(t, filepath.Join(nativeRoot, "archived.jsonl"), 50) + + manifestA := filepath.Join(store, "manifests", "managed-a.json") + manifestB := filepath.Join(store, "manifests", "managed-b.json") + manifestC := filepath.Join(store, "manifests", "archived.json") + writeJSONFile(t, manifestA, manifestFixture("managed-a", nativeA, 100)) + writeJSONFile(t, manifestB, manifestFixture("managed-b", nativeB, 20)) + writeJSONFile(t, manifestC, manifestFixture("archived", nativeC, 50)) + + writeSizedFile(t, filepath.Join(store, "objects", "aa", "aaaaaaaa.zst"), 3) + writeSizedFile(t, filepath.Join(store, "packs", "gen-current", "pack-000001.pack"), 4) + writeSizedFile(t, filepath.Join(store, "packs", "gen-current", "index.json"), 2) + writeSizedFile(t, filepath.Join(store, "packs", "gen-old", "pack-000001.pack"), 5) + writeSizedFile(t, filepath.Join(store, "packs", "gen-old", "index.json"), 2) + writeBytesFile(t, filepath.Join(store, "packs", "CURRENT"), []byte("gen-current\n")) + + snapshotA := filepath.Join(store, "fs", "snapshots", "managed-a", "native.jsonl") + if err := os.MkdirAll(filepath.Dir(snapshotA), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Link(nativeA, snapshotA); err != nil { + t.Fatalf("hard-link retained snapshot: %v", err) + } + snapshotB := writeSizedFile(t, filepath.Join(store, "fs", "snapshots", "managed-b", "native.jsonl"), 20) + + sessionA := filepath.Join(store, "fs", "sessions", "managed-a") + deltaA := writeSizedFile(t, filepath.Join(sessionA, "delta.jsonl"), 4) + backingA := writeSizedFile(t, filepath.Join(sessionA, "backing-00000000000000000002.jsonl"), 17) + writeJSONFile(t, filepath.Join(sessionA, "state.json"), stateFixture("managed-a", manifestA, 100, deltaA, backingA, snapshotA)) + + sessionB := filepath.Join(store, "fs", "sessions", "managed-b") + deltaB := writeSizedFile(t, filepath.Join(sessionB, "delta.jsonl"), 5) + writeSizedFile(t, filepath.Join(sessionB, "delta-00000000000000000001.jsonl"), 6) + writeJSONFile(t, filepath.Join(sessionB, "state.json"), stateFixture("managed-b", manifestB, 20, deltaB, "", snapshotB)) + + scratch := writeSizedFile(t, filepath.Join(sessionB, ".compact-00000000000000000001.jsonl"), 25) + stateTemp := writeSizedFile(t, filepath.Join(sessionB, ".state-compact-00000000000000000002.tmp"), 7) + writeJSONLine(t, filepath.Join(sessionB, "journal.jsonl"), map[string]any{ + "operation_id": "compact-00000000000000000001", + "phase": "prepared", + "temp_path": stateTemp, + "native": map[string]any{"path": scratch}, + }) + writeSizedFile(t, filepath.Join(sessionB, ".backing-orphan.tmp"), 8) + + writeSizedFile(t, filepath.Join(store, "fs", "fallbacks", "managed-a", "fallback-current.jsonl"), 9) + writeSizedFile(t, filepath.Join(store, "fs", "retired", "managed-a-1", "state.json"), 11) + writeSizedFile(t, filepath.Join(store, "fs", "retired", "managed-a-1", "retained-native", "native.jsonl"), 12) + + inventory, err := Scan(context.Background(), Options{StoreDir: store}) + if err != nil { + t.Fatalf("Scan: %v", err) + } + + if inventory.LogicalSessionBytes != 92 { + t.Fatalf("logical session bytes = %d, want 92", inventory.LogicalSessionBytes) + } + assertUsage(t, "loose objects", inventory.UniqueLooseObjects, 1, 3) + assertUsage(t, "current packs", inventory.Packs, 2, 6) + assertUsage(t, "native sources", inventory.NativeSources, 3, 83) + assertUsage(t, "retained snapshots", inventory.RetainedSnapshots, 2, 33) + assertUsage(t, "current fallbacks", inventory.CurrentFallbacks, 1, 9) + assertUsage(t, "active deltas", inventory.ActiveDeltas, 1, 5) + assertUsage(t, "writable backings", inventory.WritableBackings, 1, 17) + assertUsage(t, "old generations", inventory.OldGenerations, 4, 17) + assertUsage(t, "retirement state", inventory.RetirementState, 2, 23) + assertUsage(t, "journal recovery", inventory.JournalRecovery, 2, 32) + assertUsage(t, "unowned temporary", inventory.UnownedTemporary, 1, 8) + + if inventory.TotalPhysicalBytes <= 0 { + t.Fatalf("total physical bytes = %d", inventory.TotalPhysicalBytes) + } + if inventory.HardlinkAliases != 1 { + t.Fatalf("hard-link aliases = %d, want 1", inventory.HardlinkAliases) + } +} + +func TestScanRejectsPathsOutsideTheDeclaredStore(t *testing.T) { + root := t.TempDir() + store := filepath.Join(root, "store") + manifest := filepath.Join(store, "manifests", "session.json") + native := writeSizedFile(t, filepath.Join(root, "native.jsonl"), 4) + writeJSONFile(t, manifest, manifestFixture("session", native, 4)) + writeJSONFile(t, filepath.Join(store, "fs", "sessions", "session", "state.json"), stateFixture( + "session", + manifest, + 4, + filepath.Join(root, "unsafe-delta.jsonl"), + "", + native, + )) + + if _, err := Scan(context.Background(), Options{StoreDir: store}); err == nil { + t.Fatal("Scan should reject a managed data path outside its session directory") + } +} + +func TestScannerRecognizesCanonicalNestedMount(t *testing.T) { + store := filepath.Join(t.TempDir(), "store") + canonical := filepath.Join(filepath.Dir(store), "canonical-store") + s := &scanner{ + store: store, + canonicalStore: canonical, + nestedMounts: map[string]struct{}{ + filepath.Join(canonical, "nested", "mount"): {}, + }, + } + if !s.isNestedMount(filepath.Join(store, "nested", "mount")) { + t.Fatal("canonical nested mount was not recognized") + } + if s.isNestedMount(filepath.Join(store, "nested")) || s.isNestedMount(store) { + t.Fatal("ordinary storage directory was treated as a mount") + } +} + +func assertUsage(t *testing.T, name string, usage FileUsage, files int, apparentBytes int64) { + t.Helper() + if usage.Files != files || usage.ApparentBytes != apparentBytes { + t.Fatalf("%s usage = %#v, want files=%d apparent_bytes=%d", name, usage, files, apparentBytes) + } +} + +func manifestFixture(sessionID string, rolloutPath string, sourceBytes int64) map[string]any { + return map[string]any{ + "session": map[string]any{"id": sessionID, "rollout_path": rolloutPath}, + "source": map[string]any{"bytes": sourceBytes}, + } +} + +func stateFixture(sessionID string, manifestPath string, baseBytes int64, deltaPath string, backingPath string, snapshotPath string) map[string]any { + return map[string]any{ + "version": 1, + "session_id": sessionID, + "generation": 2, + "manifest_path": manifestPath, + "base_bytes": baseBytes, + "base_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "delta_path": deltaPath, + "backing_path": backingPath, + "native_snapshot": map[string]any{"path": snapshotPath, "bytes": baseBytes, "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + } +} + +func writeJSONFile(t *testing.T, path string, value any) string { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, append(data, '\n'), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func writeJSONLine(t *testing.T, path string, value any) { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, append(data, '\n'), 0o600); err != nil { + t.Fatal(err) + } +} + +func writeSizedFile(t *testing.T, path string, size int64) string { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + buffer := make([]byte, size) + for i := range buffer { + buffer[i] = byte('a' + i%26) + } + if _, err := file.Write(buffer); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + return path +} + +func writeBytesFile(t *testing.T, path string, data []byte) string { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + return path +} diff --git a/internal/storage/lease.go b/internal/storage/lease.go new file mode 100644 index 0000000..96f5773 --- /dev/null +++ b/internal/storage/lease.go @@ -0,0 +1,157 @@ +package storage + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" +) + +type Lease struct { + file *os.File + path string + closeOnce sync.Once + closeErr error +} + +func AcquireLease(directory string, label string) (*Lease, error) { + if directory == "" || label == "" || filepath.Base(label) != label || strings.ContainsAny(label, "/\\\x00") { + return nil, errors.New("lease directory and safe label are required") + } + if err := os.Mkdir(directory, 0o700); err != nil { + if !errors.Is(err, os.ErrExist) { + return nil, fmt.Errorf("create lease directory: %w", err) + } + info, statErr := os.Lstat(directory) + if statErr != nil { + return nil, fmt.Errorf("inspect lease directory: %w", statErr) + } + if !info.IsDir() { + return nil, errors.New("lease path is not a directory") + } + } + random := make([]byte, 8) + if _, err := rand.Read(random); err != nil { + return nil, err + } + path := filepath.Join(directory, fmt.Sprintf(".lease-%s-%d-%s", label, os.Getpid(), hex.EncodeToString(random))) + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("create lease file: %w", err) + } + locked, err := tryLockLease(file) + if err != nil || !locked { + _ = file.Close() + _ = os.Remove(path) + if err == nil { + err = errors.New("new lease file could not be locked") + } + return nil, err + } + if _, err := fmt.Fprintf(file, "%d\n", os.Getpid()); err != nil { + _ = unlockLease(file) + _ = file.Close() + _ = os.Remove(path) + return nil, err + } + if err := file.Sync(); err != nil { + _ = unlockLease(file) + _ = file.Close() + _ = os.Remove(path) + return nil, err + } + return &Lease{file: file, path: path}, nil +} + +func (l *Lease) Close() error { + if l == nil { + return nil + } + l.closeOnce.Do(func() { + var errs []error + if l.file != nil { + if err := unlockLease(l.file); err != nil { + errs = append(errs, err) + } + if err := l.file.Close(); err != nil { + errs = append(errs, err) + } + } + if err := os.Remove(l.path); err != nil && !errors.Is(err, os.ErrNotExist) { + errs = append(errs, err) + } + l.closeErr = errors.Join(errs...) + }) + return l.closeErr +} + +func DirectoryHasActiveLease(directory string, cleanStale bool) (bool, error) { + entries, err := os.ReadDir(directory) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + active := false + for _, entry := range entries { + if entry.IsDir() || !strings.HasPrefix(entry.Name(), ".lease-") { + continue + } + path := filepath.Join(directory, entry.Name()) + file, err := os.OpenFile(path, os.O_RDWR, 0) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return false, err + } + locked, lockErr := tryLockLease(file) + if lockErr != nil { + _ = file.Close() + return false, lockErr + } + if !locked { + active = true + _ = file.Close() + continue + } + unlockErr := unlockLease(file) + closeErr := file.Close() + if unlockErr != nil || closeErr != nil { + return false, errors.Join(unlockErr, closeErr) + } + if cleanStale { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return false, err + } + } + } + return active, nil +} + +func FileHasActiveLock(path string) (bool, error) { + file, err := os.OpenFile(path, os.O_RDWR, 0) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + locked, lockErr := tryLockLease(file) + if lockErr != nil { + _ = file.Close() + return false, lockErr + } + if !locked { + _ = file.Close() + return true, nil + } + unlockErr := unlockLease(file) + closeErr := file.Close() + return false, errors.Join(unlockErr, closeErr) +} diff --git a/internal/storage/lease_other.go b/internal/storage/lease_other.go new file mode 100644 index 0000000..c4074f6 --- /dev/null +++ b/internal/storage/lease_other.go @@ -0,0 +1,16 @@ +//go:build !darwin && !linux && !windows + +package storage + +import ( + "errors" + "os" +) + +func tryLockLease(*os.File) (bool, error) { + return false, errors.New("storage leases are unsupported on this platform") +} + +func unlockLease(*os.File) error { + return errors.New("storage leases are unsupported on this platform") +} diff --git a/internal/storage/lease_test.go b/internal/storage/lease_test.go new file mode 100644 index 0000000..8831020 --- /dev/null +++ b/internal/storage/lease_test.go @@ -0,0 +1,65 @@ +package storage + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestLeaseReportsActiveUntilClosed(t *testing.T) { + directory := filepath.Join(t.TempDir(), "leases") + lease, err := AcquireLease(directory, "generation") + if err != nil { + t.Fatalf("AcquireLease: %v", err) + } + active, err := DirectoryHasActiveLease(directory, true) + if err != nil { + t.Fatalf("DirectoryHasActiveLease: %v", err) + } + if !active { + t.Fatal("held lease was not reported active") + } + if err := lease.Close(); err != nil { + t.Fatalf("close lease: %v", err) + } + active, err = DirectoryHasActiveLease(directory, true) + if err != nil { + t.Fatalf("DirectoryHasActiveLease after close: %v", err) + } + if active { + t.Fatal("closed lease remained active") + } +} + +func TestAcquireLeaseDoesNotCreateMissingAncestorDirectories(t *testing.T) { + root := t.TempDir() + missingParent := filepath.Join(root, "missing-generation") + if _, err := AcquireLease(filepath.Join(missingParent, "leases"), "reader"); err == nil { + t.Fatal("lease unexpectedly created a missing generation tree") + } + if _, err := os.Lstat(missingParent); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("missing generation was recreated: %v", err) + } +} + +func TestDirectoryHasActiveLeaseCleansUnlockedStaleFiles(t *testing.T) { + directory := filepath.Join(t.TempDir(), "leases") + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + stale := filepath.Join(directory, ".lease-stale") + if err := os.WriteFile(stale, []byte("stale\n"), 0o600); err != nil { + t.Fatal(err) + } + active, err := DirectoryHasActiveLease(directory, true) + if err != nil { + t.Fatalf("DirectoryHasActiveLease: %v", err) + } + if active { + t.Fatal("unlocked stale lease was reported active") + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Fatalf("stale lease remains: %v", err) + } +} diff --git a/internal/storage/lease_unix.go b/internal/storage/lease_unix.go new file mode 100644 index 0000000..e4ca92a --- /dev/null +++ b/internal/storage/lease_unix.go @@ -0,0 +1,21 @@ +//go:build darwin || linux + +package storage + +import ( + "errors" + "os" + "syscall" +) + +func tryLockLease(file *os.File) (bool, error) { + err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) { + return false, nil + } + return err == nil, err +} + +func unlockLease(file *os.File) error { + return syscall.Flock(int(file.Fd()), syscall.LOCK_UN) +} diff --git a/internal/storage/lease_windows.go b/internal/storage/lease_windows.go new file mode 100644 index 0000000..4981183 --- /dev/null +++ b/internal/storage/lease_windows.go @@ -0,0 +1,23 @@ +//go:build windows + +package storage + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +func tryLockLease(file *os.File) (bool, error) { + overlapped := new(windows.Overlapped) + err := windows.LockFileEx(windows.Handle(file.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, overlapped) + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) || errors.Is(err, windows.ERROR_IO_PENDING) { + return false, nil + } + return err == nil, err +} + +func unlockLease(file *os.File) error { + return windows.UnlockFileEx(windows.Handle(file.Fd()), 0, 1, 0, new(windows.Overlapped)) +} diff --git a/internal/storage/mountpoints_darwin.go b/internal/storage/mountpoints_darwin.go new file mode 100644 index 0000000..794ee68 --- /dev/null +++ b/internal/storage/mountpoints_darwin.go @@ -0,0 +1,29 @@ +//go:build darwin + +package storage + +import ( + "path/filepath" + + "golang.org/x/sys/unix" +) + +func nestedMountPoints(root string) (map[string]struct{}, error) { + count, err := unix.Getfsstat(nil, unix.MNT_NOWAIT) + if err != nil { + return nil, err + } + stats := make([]unix.Statfs_t, count+16) + count, err = unix.Getfsstat(stats, unix.MNT_NOWAIT) + if err != nil { + return nil, err + } + result := make(map[string]struct{}) + for _, stat := range stats[:count] { + mountPoint := filepath.Clean(unix.ByteSliceToString(stat.Mntonname[:])) + if mountPoint != root && pathWithin(root, mountPoint) { + result[mountPoint] = struct{}{} + } + } + return result, nil +} diff --git a/internal/storage/mountpoints_linux.go b/internal/storage/mountpoints_linux.go new file mode 100644 index 0000000..8a782b1 --- /dev/null +++ b/internal/storage/mountpoints_linux.go @@ -0,0 +1,36 @@ +//go:build linux + +package storage + +import ( + "bufio" + "os" + "path/filepath" + "strings" +) + +func nestedMountPoints(root string) (map[string]struct{}, error) { + file, err := os.Open("/proc/self/mountinfo") + if err != nil { + return nil, err + } + defer file.Close() + result := make(map[string]struct{}) + scanner := bufio.NewScanner(file) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 5 { + continue + } + mountPoint := filepath.Clean(unescapeMountInfoPath(fields[4])) + if mountPoint != root && pathWithin(root, mountPoint) { + result[mountPoint] = struct{}{} + } + } + return result, scanner.Err() +} + +func unescapeMountInfoPath(path string) string { + replacer := strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`) + return replacer.Replace(path) +} diff --git a/internal/storage/mountpoints_other.go b/internal/storage/mountpoints_other.go new file mode 100644 index 0000000..82c43d1 --- /dev/null +++ b/internal/storage/mountpoints_other.go @@ -0,0 +1,7 @@ +//go:build !darwin && !linux + +package storage + +func nestedMountPoints(string) (map[string]struct{}, error) { + return nil, nil +} diff --git a/internal/storage/physical_other.go b/internal/storage/physical_other.go new file mode 100644 index 0000000..fe58f8e --- /dev/null +++ b/internal/storage/physical_other.go @@ -0,0 +1,16 @@ +//go:build !darwin && !linux && !windows + +package storage + +import ( + "os" + "path/filepath" +) + +func physicalFile(path string, info os.FileInfo) (string, int64, error) { + return filepath.Clean(path), info.Size(), nil +} + +func physicalLinkCount(_ string, _ os.FileInfo) (uint64, error) { + return 1, nil +} diff --git a/internal/storage/physical_unix.go b/internal/storage/physical_unix.go new file mode 100644 index 0000000..d21cc47 --- /dev/null +++ b/internal/storage/physical_unix.go @@ -0,0 +1,24 @@ +//go:build darwin || linux + +package storage + +import ( + "fmt" + "os" + "syscall" +) + +func physicalFile(path string, info os.FileInfo) (string, int64, error) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return "", 0, fmt.Errorf("read physical file identity for %s", path) + } + return fmt.Sprintf("%d:%d", stat.Dev, stat.Ino), stat.Blocks * 512, nil +} + +func physicalLinkCount(_ string, info os.FileInfo) (uint64, error) { + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + return uint64(stat.Nlink), nil + } + return 1, nil +} diff --git a/internal/storage/physical_windows.go b/internal/storage/physical_windows.go new file mode 100644 index 0000000..dd97e7b --- /dev/null +++ b/internal/storage/physical_windows.go @@ -0,0 +1,45 @@ +//go:build windows + +package storage + +import ( + "fmt" + "os" + + "golang.org/x/sys/windows" +) + +func physicalFile(path string, info os.FileInfo) (string, int64, error) { + utf16Path, err := windows.UTF16PtrFromString(path) + if err != nil { + return "", 0, err + } + handle, err := windows.CreateFile(utf16Path, 0, windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) + if err != nil { + return "", 0, fmt.Errorf("open physical file identity for %s: %w", path, err) + } + defer windows.CloseHandle(handle) + var identity windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &identity); err != nil { + return "", 0, fmt.Errorf("read physical file identity for %s: %w", path, err) + } + key := fmt.Sprintf("%d:%d:%d", identity.VolumeSerialNumber, identity.FileIndexHigh, identity.FileIndexLow) + return key, info.Size(), nil +} + +func physicalLinkCount(path string, _ os.FileInfo) (uint64, error) { + utf16Path, err := windows.UTF16PtrFromString(path) + if err != nil { + return 0, err + } + handle, err := windows.CreateFile(utf16Path, 0, windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) + if err != nil { + return 0, err + } + defer windows.CloseHandle(handle) + var identity windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &identity); err != nil { + return 0, err + } + return uint64(identity.NumberOfLinks), nil +} diff --git a/internal/storage/policy.go b/internal/storage/policy.go new file mode 100644 index 0000000..aa458d4 --- /dev/null +++ b/internal/storage/policy.go @@ -0,0 +1,70 @@ +package storage + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +const PolicyFilename = "storage-policy.json" + +var DefaultLimits = Limits{ + MaxPhysicalBytes: 512 << 30, + MaxTemporaryBytes: 16 << 30, + FreeSpaceReserveBytes: 5 << 30, +} + +type policyFile struct { + Version int `json:"version"` + Limits Limits `json:"limits"` +} + +type Checker interface { + Check(context.Context, Projection) (Assessment, error) +} + +func LoadLimits(storeDir string) (Limits, error) { + if storeDir == "" { + return Limits{}, errors.New("storage policy store directory is required") + } + path := filepath.Join(filepath.Clean(storeDir), PolicyFilename) + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return DefaultLimits, nil + } + if err != nil { + return Limits{}, fmt.Errorf("read storage policy: %w", err) + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + var policy policyFile + if err := decoder.Decode(&policy); err != nil { + return Limits{}, fmt.Errorf("decode storage policy: %w", err) + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + if err == nil { + err = errors.New("multiple JSON values") + } + return Limits{}, fmt.Errorf("decode storage policy: %w", err) + } + if policy.Version != 1 { + return Limits{}, fmt.Errorf("unsupported storage policy version %d", policy.Version) + } + if policy.Limits.MaxPhysicalBytes <= 0 || policy.Limits.MaxTemporaryBytes <= 0 || policy.Limits.FreeSpaceReserveBytes <= 0 { + return Limits{}, errors.New("storage policy limits must all be positive") + } + return policy.Limits, nil +} + +func DefaultGuard(storeDir string) (Guard, error) { + limits, err := LoadLimits(storeDir) + if err != nil { + return Guard{}, err + } + return Guard{StoreDir: filepath.Clean(storeDir), Limits: limits}, nil +} diff --git a/internal/storage/policy_test.go b/internal/storage/policy_test.go new file mode 100644 index 0000000..738834b --- /dev/null +++ b/internal/storage/policy_test.go @@ -0,0 +1,52 @@ +package storage + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +func TestLoadLimitsUsesBoundedDefaultsAndAllowsStoreOverride(t *testing.T) { + store := t.TempDir() + defaults, err := LoadLimits(store) + if err != nil { + t.Fatalf("LoadLimits defaults: %v", err) + } + if defaults.MaxPhysicalBytes <= 0 || defaults.MaxTemporaryBytes <= 0 || defaults.FreeSpaceReserveBytes <= 0 { + t.Fatalf("default limits are not hard bounds: %#v", defaults) + } + + want := Limits{MaxPhysicalBytes: 900, MaxTemporaryBytes: 80, FreeSpaceReserveBytes: 70} + data, err := json.Marshal(map[string]any{"version": 1, "limits": want}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(store, PolicyFilename), append(data, '\n'), 0o600); err != nil { + t.Fatal(err) + } + got, err := LoadLimits(store) + if err != nil { + t.Fatalf("LoadLimits override: %v", err) + } + if got != want { + t.Fatalf("limits = %#v, want %#v", got, want) + } +} + +func TestLoadLimitsRejectsUnboundedOrUnknownPolicy(t *testing.T) { + tests := []string{ + `{"version":2,"limits":{"max_physical_bytes":1,"max_temporary_bytes":1,"free_space_reserve_bytes":1}}`, + `{"version":1,"limits":{"max_physical_bytes":0,"max_temporary_bytes":1,"free_space_reserve_bytes":1}}`, + `{"version":1,"limits":{"max_physical_bytes":1,"max_temporary_bytes":1,"free_space_reserve_bytes":1},"extra":true}`, + } + for index, data := range tests { + store := t.TempDir() + if err := os.WriteFile(filepath.Join(store, PolicyFilename), []byte(data), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadLimits(store); err == nil { + t.Fatalf("policy %d should fail", index) + } + } +} diff --git a/internal/storage/space.go b/internal/storage/space.go new file mode 100644 index 0000000..3ea3c0f --- /dev/null +++ b/internal/storage/space.go @@ -0,0 +1,23 @@ +package storage + +import ( + "errors" + "os" + "path/filepath" +) + +func existingSpaceProbePath(path string) (string, error) { + path = filepath.Clean(path) + for { + if _, err := os.Stat(path); err == nil { + return path, nil + } else if !errors.Is(err, os.ErrNotExist) { + return "", err + } + parent := filepath.Dir(path) + if parent == path { + return "", os.ErrNotExist + } + path = parent + } +} diff --git a/internal/storage/space_other.go b/internal/storage/space_other.go new file mode 100644 index 0000000..71bd984 --- /dev/null +++ b/internal/storage/space_other.go @@ -0,0 +1,9 @@ +//go:build !darwin && !linux && !windows + +package storage + +import "errors" + +func AvailableBytes(string) (int64, error) { + return 0, errors.New("available-byte probe is unsupported on this platform") +} diff --git a/internal/storage/space_unix.go b/internal/storage/space_unix.go new file mode 100644 index 0000000..34bf00a --- /dev/null +++ b/internal/storage/space_unix.go @@ -0,0 +1,29 @@ +//go:build darwin || linux + +package storage + +import ( + "errors" + "math" + + "golang.org/x/sys/unix" +) + +func AvailableBytes(path string) (int64, error) { + path, err := existingSpaceProbePath(path) + if err != nil { + return 0, err + } + var stat unix.Statfs_t + if err := unix.Statfs(path, &stat); err != nil { + return 0, err + } + if stat.Bsize <= 0 { + return 0, errors.New("filesystem block size is invalid") + } + available := uint64(stat.Bavail) * uint64(stat.Bsize) + if available > math.MaxInt64 { + return math.MaxInt64, nil + } + return int64(available), nil +} diff --git a/internal/storage/space_windows.go b/internal/storage/space_windows.go new file mode 100644 index 0000000..26611bc --- /dev/null +++ b/internal/storage/space_windows.go @@ -0,0 +1,24 @@ +//go:build windows + +package storage + +import "golang.org/x/sys/windows" + +func AvailableBytes(path string) (int64, error) { + path, err := existingSpaceProbePath(path) + if err != nil { + return 0, err + } + utf16Path, err := windows.UTF16PtrFromString(path) + if err != nil { + return 0, err + } + var available uint64 + if err := windows.GetDiskFreeSpaceEx(utf16Path, &available, nil, nil); err != nil { + return 0, err + } + if available > uint64(^uint64(0)>>1) { + return int64(^uint64(0) >> 1), nil + } + return int64(available), nil +} diff --git a/internal/testfs/corpus_test.go b/internal/testfs/corpus_test.go index a137907..b5a6262 100644 --- a/internal/testfs/corpus_test.go +++ b/internal/testfs/corpus_test.go @@ -13,7 +13,6 @@ import ( "testing" "time" - "github.com/jstar0/codexfold/internal/codex" "github.com/jstar0/codexfold/internal/fold" "github.com/jstar0/codexfold/internal/fsctl" "github.com/jstar0/codexfold/internal/pack" @@ -50,7 +49,7 @@ func TestPackedCorpusShadowRandomReadsAndWritableSessionStress(t *testing.T) { } store := filepath.Join(root, "store") for _, fixture := range corpus.Sessions { - _, err := fold.Fold(context.Background(), codex.Session{ID: fixture.ID, RolloutPath: fixture.Path, Archived: true}, fold.FoldOptions{StoreDir: store, Apply: true, FieldThreshold: 32}) + _, err := fold.Fold(context.Background(), fold.Session{ID: fixture.ID, RolloutPath: fixture.Path, Archived: true}, fold.FoldOptions{StoreDir: store, Apply: true, FieldThreshold: 32}) if err != nil { t.Fatalf("fold %s: %v", fixture.ID, err) } diff --git a/internal/testfs/large_test.go b/internal/testfs/large_test.go index a414f51..8d9af5d 100644 --- a/internal/testfs/large_test.go +++ b/internal/testfs/large_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/jstar0/codexfold/internal/codex" "github.com/jstar0/codexfold/internal/fold" "github.com/jstar0/codexfold/internal/fsctl" "github.com/jstar0/codexfold/internal/pack" @@ -44,7 +43,7 @@ func TestLargePreviewBenchmark(t *testing.T) { } store := filepath.Join(root, "store") foldStart := time.Now() - if _, err := fold.Fold(context.Background(), codex.Session{ID: fixture.ID, RolloutPath: fixture.Path, Archived: true}, fold.FoldOptions{StoreDir: store, Apply: true, FieldThreshold: 1 << 20}); err != nil { + if _, err := fold.Fold(context.Background(), fold.Session{ID: fixture.ID, RolloutPath: fixture.Path, Archived: true}, fold.FoldOptions{StoreDir: store, Apply: true, FieldThreshold: 1 << 20}); err != nil { t.Fatal(err) } foldDuration := time.Since(foldStart) diff --git a/internal/vfs/handles.go b/internal/vfs/handles.go index 699b29f..d77ce1c 100644 --- a/internal/vfs/handles.go +++ b/internal/vfs/handles.go @@ -96,8 +96,7 @@ func (h *ReadHandle) ReadAt(ctx context.Context, destination []byte, offset int6 func (h *ReadHandle) Close() error { h.closeOnce.Do(func() { - h.closeErr = h.file.Close() - h.session.releaseReader(h.generation) + h.closeErr = errors.Join(h.file.Close(), h.session.releaseReader(h.generation)) }) return h.closeErr } diff --git a/internal/vfs/recovery_test.go b/internal/vfs/recovery_test.go index 9284500..980be94 100644 --- a/internal/vfs/recovery_test.go +++ b/internal/vfs/recovery_test.go @@ -6,10 +6,12 @@ import ( "errors" "os" "path/filepath" + "strings" "testing" "time" "github.com/jstar0/codexfold/internal/fold" + "github.com/jstar0/codexfold/internal/storage" ) func TestRecoverFinishesPublishedCopyOnWriteGeneration(t *testing.T) { @@ -132,6 +134,70 @@ func TestCompactSwitchesGenerationAndPreservesPinnedReader(t *testing.T) { } } +func TestStorageGCKeepsOldSessionGenerationUntilReaderLeaseCloses(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + writer, err := session.OpenWriter() + if err != nil { + t.Fatal(err) + } + if _, err := writer.Append(context.Background(), []byte("-tail")); err != nil { + t.Fatal(err) + } + _ = writer.Close() + oldDelta := session.State().DeltaPath + oldReader, err := session.OpenReader() + if err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), source...), []byte("-tail")...) + _, err = session.Compact(context.Background(), CompactOptions{Prepare: func(_ context.Context, current NativeFile, _ uint64) (PreparedGeneration, error) { + data, err := os.ReadFile(current.Path) + if err != nil { + return PreparedGeneration{}, err + } + digest := digestBytes(data) + prepared := fold.Manifest{ + Version: fold.ManifestVersion, Kind: fold.ManifestKind, + Session: fold.ManifestSession{ID: "session", RolloutPath: current.Path}, + Source: fold.ManifestSource{Bytes: int64(len(data)), SHA256: digest}, + Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: digest, RawBytes: int64(len(data))}}}, + } + view, err := NewView(prepared, memoryReader{digest: data}) + return PreparedGeneration{ManifestPath: filepath.Join(root, "manifest-generation-2.json"), Manifest: prepared, View: view}, err + }}) + if err != nil { + t.Fatal(err) + } + if got := readHandle(t, oldReader); !bytes.Equal(got, want) { + t.Fatalf("old reader changed: %q", got) + } + blocked, err := storage.Collect(context.Background(), storage.GCOptions{StoreDir: root, Apply: true}) + if err != nil { + t.Fatal(err) + } + if blocked.RemovedCount != 0 { + t.Fatalf("active reader generation was collected: %#v", blocked) + } + if _, err := os.Stat(oldDelta); err != nil { + t.Fatalf("old delta missing while reader lease active: %v", err) + } + if err := oldReader.Close(); err != nil { + t.Fatal(err) + } + collected, err := storage.Collect(context.Background(), storage.GCOptions{StoreDir: root, Apply: true}) + if err != nil { + t.Fatal(err) + } + if collected.RemovedCount != 1 { + t.Fatalf("closed reader generation was not collected: %#v", collected) + } + if _, err := os.Stat(oldDelta); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("old delta remains after reader close: %v", err) + } +} + func TestCompactRejectsDeltaChangedDuringPreparation(t *testing.T) { root := t.TempDir() manifest, reader, _ := sessionFixture(t, root) @@ -166,6 +232,39 @@ func TestCompactRejectsDeltaChangedDuringPreparation(t *testing.T) { } } +func TestCompactBudgetRejectsBeforeScratchOrPreparation(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + checker := &vfsRejectingChecker{} + session, err := OpenSession(context.Background(), SessionOptions{ + Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, Reader: reader, + NativeSnapshot: NativeFile{Path: manifest.Session.RolloutPath, Bytes: manifest.Source.Bytes, SHA256: manifest.Source.SHA256}, + Budget: checker, + }) + if err != nil { + t.Fatal(err) + } + prepared := false + if _, err := session.Compact(context.Background(), CompactOptions{Prepare: func(context.Context, NativeFile, uint64) (PreparedGeneration, error) { + prepared = true + return PreparedGeneration{}, nil + }}); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("Compact error = %v, want storage budget rejection", err) + } + if prepared { + t.Fatal("compact preparation ran after budget rejection") + } + entries, err := os.ReadDir(filepath.Join(root, "fs", "sessions", "session")) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".compact-") { + t.Fatalf("compact scratch exists after preflight rejection: %s", entry.Name()) + } + } +} + func TestCompactRejectsWriterLeaseHeldByAnotherSession(t *testing.T) { root := t.TempDir() manifest, reader, _ := sessionFixture(t, root) diff --git a/internal/vfs/session.go b/internal/vfs/session.go index b3c540e..99fc9aa 100644 --- a/internal/vfs/session.go +++ b/internal/vfs/session.go @@ -13,6 +13,7 @@ import ( "time" "github.com/jstar0/codexfold/internal/fold" + "github.com/jstar0/codexfold/internal/storage" ) type SessionOptions struct { @@ -21,18 +22,21 @@ type SessionOptions struct { Manifest fold.Manifest Reader ObjectReader NativeSnapshot NativeFile + Budget storage.Checker BeforeCOWPhase func(string) error } type Session struct { - mu sync.Mutex - state SessionState - statePath string - directory string - view *View - readerLeases map[uint64]int - writerOpen bool - beforeCOWPhase func(string) error + mu sync.Mutex + state SessionState + statePath string + directory string + view *View + readerLeases map[uint64]int + readerLeaseFiles map[uint64]*storage.Lease + writerOpen bool + budget storage.Checker + beforeCOWPhase func(string) error } type VisibleInfo struct { @@ -59,6 +63,14 @@ func openSession(ctx context.Context, options SessionOptions, reserveWriter bool if options.Root == "" || options.ManifestPath == "" || !safeSessionID(options.Manifest.Session.ID) { return nil, nil, errors.New("session root, manifest path, and safe session ID are required") } + budget := options.Budget + if budget == nil { + guard, err := storage.DefaultGuard(options.Root) + if err != nil { + return nil, nil, err + } + budget = guard + } view, err := NewView(options.Manifest, options.Reader) if err != nil { return nil, nil, err @@ -134,7 +146,11 @@ func openSession(ctx context.Context, options SessionOptions, reserveWriter bool } } } - session := &Session{state: state, statePath: statePath, directory: directory, view: view, readerLeases: make(map[uint64]int), beforeCOWPhase: options.BeforeCOWPhase} + session := &Session{ + state: state, statePath: statePath, directory: directory, view: view, + readerLeases: make(map[uint64]int), readerLeaseFiles: make(map[uint64]*storage.Lease), + budget: budget, beforeCOWPhase: options.BeforeCOWPhase, + } var writer *WriteHandle if reservedLease != nil { session.writerOpen = true @@ -158,6 +174,15 @@ func (s *Session) State() SessionState { return s.state } +func (s *Session) MetadataPath() string { + s.mu.Lock() + defer s.mu.Unlock() + if s.state.BackingPath != "" { + return s.state.BackingPath + } + return s.state.DeltaPath +} + func (s *Session) VisibleInfo() (VisibleInfo, error) { s.mu.Lock() state := s.state @@ -181,6 +206,20 @@ func (s *Session) OpenReader() (*ReadHandle, error) { s.mu.Lock() state := s.state view := s.view + if s.readerLeases[state.Generation] == 0 { + leaseRoot := filepath.Join(s.directory, "leases") + if err := os.Mkdir(leaseRoot, 0o700); err != nil && !errors.Is(err, os.ErrExist) { + s.mu.Unlock() + return nil, fmt.Errorf("create reader lease root: %w", err) + } + leaseDirectory := filepath.Join(s.directory, "leases", fmt.Sprintf("generation-%020d", state.Generation)) + lease, err := storage.AcquireLease(leaseDirectory, "reader") + if err != nil { + s.mu.Unlock() + return nil, fmt.Errorf("acquire reader generation lease: %w", err) + } + s.readerLeaseFiles[state.Generation] = lease + } s.readerLeases[state.Generation]++ s.mu.Unlock() @@ -194,13 +233,13 @@ func (s *Session) OpenReader() (*ReadHandle, error) { } file, err := os.Open(path) if err != nil { - s.releaseReader(state.Generation) + _ = s.releaseReader(state.Generation) return nil, fmt.Errorf("open session reader file: %w", err) } info, err := file.Stat() if err != nil { _ = file.Close() - s.releaseReader(state.Generation) + _ = s.releaseReader(state.Generation) return nil, fmt.Errorf("stat session reader file: %w", err) } handle.file = file @@ -213,14 +252,18 @@ func (s *Session) OpenReader() (*ReadHandle, error) { return handle, nil } -func (s *Session) releaseReader(generation uint64) { +func (s *Session) releaseReader(generation uint64) error { s.mu.Lock() - defer s.mu.Unlock() + var lease *storage.Lease if s.readerLeases[generation] <= 1 { delete(s.readerLeases, generation) + lease = s.readerLeaseFiles[generation] + delete(s.readerLeaseFiles, generation) } else { s.readerLeases[generation]-- } + s.mu.Unlock() + return lease.Close() } func (s *Session) OpenWriter() (*WriteHandle, error) { @@ -285,6 +328,12 @@ func (s *Session) ensureBacking(ctx context.Context) (string, error) { return "", err } defer reader.Close() + if _, err := s.budget.Check(ctx, storage.Projection{ + Operation: "copy-on-write", AdditionalPersistentBytes: reader.Size(), TemporaryBytes: reader.Size(), + TemporaryPersistentOverlapBytes: reader.Size(), + }); err != nil { + return "", err + } temporary, err := os.CreateTemp(s.directory, ".backing-*.tmp") if err != nil { return "", fmt.Errorf("create temporary backing: %w", err) @@ -406,6 +455,20 @@ func (s *Session) MaterializeCurrent(ctx context.Context, target string, overwri return NativeFile{}, err } defer reader.Close() + reclaimableBytes := int64(0) + if overwrite { + if info, err := os.Stat(target); err == nil && info.Mode().IsRegular() { + reclaimableBytes = info.Size() + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return NativeFile{}, err + } + } + if _, err := s.budget.Check(ctx, storage.Projection{ + Operation: "materialize-current", AdditionalPersistentBytes: reader.Size(), TemporaryBytes: reader.Size(), + TemporaryPersistentOverlapBytes: reader.Size(), ReclaimableBytes: reclaimableBytes, + }); err != nil { + return NativeFile{}, err + } if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { return NativeFile{}, err } diff --git a/internal/vfs/session_test.go b/internal/vfs/session_test.go index e75e534..9afc133 100644 --- a/internal/vfs/session_test.go +++ b/internal/vfs/session_test.go @@ -9,9 +9,11 @@ import ( "io" "os" "path/filepath" + "strings" "testing" "github.com/jstar0/codexfold/internal/fold" + "github.com/jstar0/codexfold/internal/storage" ) func TestSessionAppendPersistsWithoutHydratingBase(t *testing.T) { @@ -92,6 +94,45 @@ func TestSessionAllowsOnlyOneWriterLease(t *testing.T) { _ = second.Close() } +func TestSessionReaderHoldsGenerationLeaseUntilClose(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + handle, err := session.OpenReader() + if err != nil { + t.Fatal(err) + } + leaseDirectory := filepath.Join(root, "fs", "sessions", "session", "leases", "generation-00000000000000000001") + active, err := storage.DirectoryHasActiveLease(leaseDirectory, false) + if err != nil || !active { + t.Fatalf("reader generation lease: active=%t err=%v", active, err) + } + if err := handle.Close(); err != nil { + t.Fatal(err) + } + active, err = storage.DirectoryHasActiveLease(leaseDirectory, true) + if err != nil || active { + t.Fatalf("closed reader generation lease: active=%t err=%v", active, err) + } +} + +func TestSessionReaderDoesNotRecreateRetiredStateDirectory(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + stateDirectory := filepath.Dir(session.State().DeltaPath) + retired := stateDirectory + ".retired" + if err := os.Rename(stateDirectory, retired); err != nil { + t.Fatal(err) + } + if _, err := session.OpenReader(); err == nil { + t.Fatal("reader unexpectedly opened after the state directory was retired") + } + if _, err := os.Lstat(stateDirectory); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("retired state directory was recreated: %v", err) + } +} + func TestSessionRandomWriteTransitionsToVerifiedBacking(t *testing.T) { root := t.TempDir() manifest, reader, source := sessionFixture(t, root) @@ -129,6 +170,67 @@ func TestSessionRandomWriteTransitionsToVerifiedBacking(t *testing.T) { } } +func TestSessionBudgetRejectsCopyOnWriteBeforeCreatingBacking(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + checker := &vfsRejectingChecker{} + session, err := OpenSession(context.Background(), SessionOptions{ + Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, Reader: reader, + NativeSnapshot: NativeFile{Path: manifest.Session.RolloutPath, Bytes: manifest.Source.Bytes, SHA256: manifest.Source.SHA256}, + Budget: checker, + }) + if err != nil { + t.Fatalf("OpenSession: %v", err) + } + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + if _, err := writer.WriteAt(context.Background(), []byte("X"), 0); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("WriteAt error = %v, want storage budget rejection", err) + } + _ = writer.Close() + if checker.Calls != 1 || checker.Projection.Operation != "copy-on-write" || checker.Projection.TemporaryBytes != int64(len(source)) { + t.Fatalf("unexpected COW budget projection: %#v", checker) + } + if state := session.State(); state.BackingPath != "" || state.Generation != 1 { + t.Fatalf("budget rejection changed session state: %#v", state) + } + entries, err := os.ReadDir(filepath.Join(root, "fs", "sessions", "session")) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.Contains(entry.Name(), "backing") { + t.Fatalf("backing artifact exists after preflight rejection: %s", entry.Name()) + } + } +} + +func TestSessionBudgetRejectsMaterializeBeforeCreatingTarget(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + checker := &vfsRejectingChecker{} + session, err := OpenSession(context.Background(), SessionOptions{ + Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, Reader: reader, + NativeSnapshot: NativeFile{Path: manifest.Session.RolloutPath, Bytes: manifest.Source.Bytes, SHA256: manifest.Source.SHA256}, + Budget: checker, + }) + if err != nil { + t.Fatalf("OpenSession: %v", err) + } + target := filepath.Join(root, "new", "current.jsonl") + if _, err := session.MaterializeCurrent(context.Background(), target, false); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("MaterializeCurrent error = %v, want storage budget rejection", err) + } + if checker.Calls != 1 || checker.Projection.Operation != "materialize-current" { + t.Fatalf("unexpected materialize budget projection: %#v", checker) + } + if _, err := os.Stat(filepath.Dir(target)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("materialize target directory exists after preflight rejection: %v", err) + } +} + func TestSessionTruncateTransitionsToBacking(t *testing.T) { root := t.TempDir() manifest, reader, source := sessionFixture(t, root) @@ -251,3 +353,14 @@ func digestBytes(data []byte) string { digest := sha256.Sum256(data) return hex.EncodeToString(digest[:]) } + +type vfsRejectingChecker struct { + Calls int + Projection storage.Projection +} + +func (c *vfsRejectingChecker) Check(_ context.Context, projection storage.Projection) (storage.Assessment, error) { + c.Calls++ + c.Projection = projection + return storage.Assessment{}, storage.ErrBudgetExceeded +} diff --git a/platform/darwin/fskit/CodexFoldFSKit.entitlements b/platform/darwin/fskit/CodexFoldFSKit.entitlements new file mode 100644 index 0000000..24446b6 --- /dev/null +++ b/platform/darwin/fskit/CodexFoldFSKit.entitlements @@ -0,0 +1,10 @@ + + + + + com.apple.security.application-groups + + group.vip.jstar.codexfold + + + diff --git a/platform/darwin/fskit/CodexFoldFSKit.xcodeproj/project.pbxproj b/platform/darwin/fskit/CodexFoldFSKit.xcodeproj/project.pbxproj new file mode 100644 index 0000000..1f479d3 --- /dev/null +++ b/platform/darwin/fskit/CodexFoldFSKit.xcodeproj/project.pbxproj @@ -0,0 +1,419 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 77; + objects = { + +/* Begin PBXBuildFile section */ + 54CC8E546672A2B2EB8778AE /* ProfileModule.swift in Sources */ = {isa = PBXBuildFile; fileRef = A321270CF90754EFB5FB4F82 /* ProfileModule.swift */; }; + 5A3E15ED0688984A0382281F /* Host.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0299A73CFDD46513958B3173 /* Host.swift */; }; + AC5E7B1EB1279405828432A1 /* Wire.swift in Sources */ = {isa = PBXBuildFile; fileRef = 442319991E8D02494E8E8966 /* Wire.swift */; }; + D146B22FE62339A25BA1F531 /* CodexFoldFSKitModule.appex in Embed ExtensionKit Extensions */ = {isa = PBXBuildFile; fileRef = 0EB68F3D6D26A8642A137511 /* CodexFoldFSKitModule.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + AE33723DBF591F025AAAE018 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 708C773DF138FE6212E0FC9A /* Project object */; + proxyType = 1; + remoteGlobalIDString = 00016AAD22571AA1F375B40A; + remoteInfo = CodexFoldFSKitModule; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXCopyFilesBuildPhase section */ + 7F086318D92AAA862935176E /* Embed ExtensionKit Extensions */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = "$(EXTENSIONS_FOLDER_PATH)"; + dstSubfolderSpec = 16; + files = ( + D146B22FE62339A25BA1F531 /* CodexFoldFSKitModule.appex in Embed ExtensionKit Extensions */, + ); + name = "Embed ExtensionKit Extensions"; + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXCopyFilesBuildPhase section */ + +/* Begin PBXFileReference section */ + 0299A73CFDD46513958B3173 /* Host.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Host.swift; sourceTree = ""; }; + 0EB68F3D6D26A8642A137511 /* CodexFoldFSKitModule.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.extensionkit-extension"; includeInIndex = 0; path = CodexFoldFSKitModule.appex; sourceTree = BUILT_PRODUCTS_DIR; }; + 12B2D089AE4A8AE24344C2F8 /* CodexFoldFSKit.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = CodexFoldFSKit.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 442319991E8D02494E8E8966 /* Wire.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Wire.swift; sourceTree = ""; }; + 45646F5D9CFE136ED2DC3D6E /* CodexFoldFSKitModule.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = CodexFoldFSKitModule.entitlements; sourceTree = ""; }; + 683C4E970391FE59129607C0 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = Info.plist; sourceTree = ""; }; + 8B37F589115C048AA0DA2C76 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = Info.plist; sourceTree = ""; }; + A321270CF90754EFB5FB4F82 /* ProfileModule.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileModule.swift; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXGroup section */ + 00AEBFA549CEA3A2EF2670ED /* Products */ = { + isa = PBXGroup; + children = ( + 12B2D089AE4A8AE24344C2F8 /* CodexFoldFSKit.app */, + 0EB68F3D6D26A8642A137511 /* CodexFoldFSKitModule.appex */, + ); + name = Products; + sourceTree = ""; + }; + 4DED45333D2D92F454B627C0 = { + isa = PBXGroup; + children = ( + B470688CF65E5B72C59AEBB3 /* Extension */, + A88D2555D5D45400A73FD862 /* Host */, + 00AEBFA549CEA3A2EF2670ED /* Products */, + ); + sourceTree = ""; + }; + A88D2555D5D45400A73FD862 /* Host */ = { + isa = PBXGroup; + children = ( + 0299A73CFDD46513958B3173 /* Host.swift */, + 8B37F589115C048AA0DA2C76 /* Info.plist */, + ); + path = Host; + sourceTree = ""; + }; + B470688CF65E5B72C59AEBB3 /* Extension */ = { + isa = PBXGroup; + children = ( + 45646F5D9CFE136ED2DC3D6E /* CodexFoldFSKitModule.entitlements */, + 683C4E970391FE59129607C0 /* Info.plist */, + A321270CF90754EFB5FB4F82 /* ProfileModule.swift */, + 442319991E8D02494E8E8966 /* Wire.swift */, + ); + path = Extension; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 00016AAD22571AA1F375B40A /* CodexFoldFSKitModule */ = { + isa = PBXNativeTarget; + buildConfigurationList = 2AA9F9298B9AF2C59CCB9ED4 /* Build configuration list for PBXNativeTarget "CodexFoldFSKitModule" */; + buildPhases = ( + C77A846ECB9F07A35A0A7952 /* Sources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = CodexFoldFSKitModule; + packageProductDependencies = ( + ); + productName = CodexFoldFSKitModule; + productReference = 0EB68F3D6D26A8642A137511 /* CodexFoldFSKitModule.appex */; + productType = "com.apple.product-type.extensionkit-extension"; + }; + 07E85731B7BEF398A8C4E4EC /* CodexFoldFSKit */ = { + isa = PBXNativeTarget; + buildConfigurationList = 830559E5E3522260B246A010 /* Build configuration list for PBXNativeTarget "CodexFoldFSKit" */; + buildPhases = ( + 0E5DA05244BDCF249A088B8C /* Sources */, + 7F086318D92AAA862935176E /* Embed ExtensionKit Extensions */, + ); + buildRules = ( + ); + dependencies = ( + 74BE85F55E9506D33987AFBF /* PBXTargetDependency */, + ); + name = CodexFoldFSKit; + packageProductDependencies = ( + ); + productName = CodexFoldFSKit; + productReference = 12B2D089AE4A8AE24344C2F8 /* CodexFoldFSKit.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 708C773DF138FE6212E0FC9A /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = YES; + LastUpgradeCheck = 1430; + TargetAttributes = { + 00016AAD22571AA1F375B40A = { + DevelopmentTeam = Y987FUR837; + ProvisioningStyle = Automatic; + }; + 07E85731B7BEF398A8C4E4EC = { + DevelopmentTeam = Y987FUR837; + ProvisioningStyle = Automatic; + }; + }; + }; + buildConfigurationList = 16739CA6CFCBE5CD8461F74B /* Build configuration list for PBXProject "CodexFoldFSKit" */; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + Base, + en, + ); + mainGroup = 4DED45333D2D92F454B627C0; + minimizedProjectReferenceProxies = 1; + preferredProjectObjectVersion = 77; + productRefGroup = 00AEBFA549CEA3A2EF2670ED /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 07E85731B7BEF398A8C4E4EC /* CodexFoldFSKit */, + 00016AAD22571AA1F375B40A /* CodexFoldFSKitModule */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXSourcesBuildPhase section */ + 0E5DA05244BDCF249A088B8C /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 5A3E15ED0688984A0382281F /* Host.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + C77A846ECB9F07A35A0A7952 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 54CC8E546672A2B2EB8778AE /* ProfileModule.swift in Sources */, + AC5E7B1EB1279405828432A1 /* Wire.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 74BE85F55E9506D33987AFBF /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 00016AAD22571AA1F375B40A /* CodexFoldFSKitModule */; + targetProxy = AE33723DBF591F025AAAE018 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin XCBuildConfiguration section */ + 07A6AEE6F5B5E6DA978DAA00 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGN_ENTITLEMENTS = CodexFoldFSKit.entitlements; + COMBINE_HIDPI_IMAGES = YES; + INFOPLIST_FILE = Host/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = vip.jstar.codexfold.fskitprofileprobe; + REGISTER_APP_GROUPS = YES; + SDKROOT = macosx; + }; + name = Release; + }; + 109F1AB456D7EEE08D96BA16 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + CODE_SIGN_ENTITLEMENTS = Extension/CodexFoldFSKitModule.entitlements; + COMBINE_HIDPI_IMAGES = YES; + ENABLE_APP_SANDBOX = YES; + INFOPLIST_FILE = Extension/Info.plist; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @executable_path/../../../../Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = vip.jstar.codexfold.fskitprofileprobe.module; + REGISTER_APP_GROUPS = YES; + SDKROOT = macosx; + SKIP_INSTALL = YES; + }; + name = Debug; + }; + 2ABE4C480A0163130E06B856 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + CODE_SIGN_STYLE = Automatic; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + DEVELOPMENT_TEAM = Y987FUR837; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "$(inherited)", + "DEBUG=1", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + MACOSX_DEPLOYMENT_TARGET = 27.0; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_VERSION = 5.0; + }; + name = Debug; + }; + 39F4FAD3CDE67351EA64468B /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGN_ENTITLEMENTS = CodexFoldFSKit.entitlements; + COMBINE_HIDPI_IMAGES = YES; + INFOPLIST_FILE = Host/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = vip.jstar.codexfold.fskitprofileprobe; + REGISTER_APP_GROUPS = YES; + SDKROOT = macosx; + }; + name = Debug; + }; + B61FA346F14A61E65CC7F596 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + CODE_SIGN_STYLE = Automatic; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + DEVELOPMENT_TEAM = Y987FUR837; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + MACOSX_DEPLOYMENT_TARGET = 27.0; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SWIFT_COMPILATION_MODE = wholemodule; + SWIFT_OPTIMIZATION_LEVEL = "-O"; + SWIFT_VERSION = 5.0; + }; + name = Release; + }; + ED8C7963C890ACC5C7FF9A70 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + CODE_SIGN_ENTITLEMENTS = Extension/CodexFoldFSKitModule.entitlements; + COMBINE_HIDPI_IMAGES = YES; + ENABLE_APP_SANDBOX = YES; + INFOPLIST_FILE = Extension/Info.plist; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @executable_path/../../../../Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = vip.jstar.codexfold.fskitprofileprobe.module; + REGISTER_APP_GROUPS = YES; + SDKROOT = macosx; + SKIP_INSTALL = YES; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 16739CA6CFCBE5CD8461F74B /* Build configuration list for PBXProject "CodexFoldFSKit" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 2ABE4C480A0163130E06B856 /* Debug */, + B61FA346F14A61E65CC7F596 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Debug; + }; + 2AA9F9298B9AF2C59CCB9ED4 /* Build configuration list for PBXNativeTarget "CodexFoldFSKitModule" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 109F1AB456D7EEE08D96BA16 /* Debug */, + ED8C7963C890ACC5C7FF9A70 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Debug; + }; + 830559E5E3522260B246A010 /* Build configuration list for PBXNativeTarget "CodexFoldFSKit" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 39F4FAD3CDE67351EA64468B /* Debug */, + 07A6AEE6F5B5E6DA978DAA00 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Debug; + }; +/* End XCConfigurationList section */ + }; + rootObject = 708C773DF138FE6212E0FC9A /* Project object */; +} diff --git a/platform/darwin/fskit/CodexFoldFSKit.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/platform/darwin/fskit/CodexFoldFSKit.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000..919434a --- /dev/null +++ b/platform/darwin/fskit/CodexFoldFSKit.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/platform/darwin/fskit/Extension/CodexFoldFSKitModule.entitlements b/platform/darwin/fskit/Extension/CodexFoldFSKitModule.entitlements new file mode 100644 index 0000000..f7535bd --- /dev/null +++ b/platform/darwin/fskit/Extension/CodexFoldFSKitModule.entitlements @@ -0,0 +1,16 @@ + + + + + com.apple.developer.fskit.fsmodule + + com.apple.security.app-sandbox + + com.apple.security.application-groups + + group.vip.jstar.codexfold + + com.apple.security.network.client + + + diff --git a/platform/darwin/fskit/Extension/Info.plist b/platform/darwin/fskit/Extension/Info.plist new file mode 100644 index 0000000..5a45bc8 --- /dev/null +++ b/platform/darwin/fskit/Extension/Info.plist @@ -0,0 +1,54 @@ + + + + + CFBundleDisplayName + CodexFold Native FSKit Module + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + XPC! + CFBundleShortVersionString + 0.1.0 + CFBundleVersion + 1 + EXAppExtensionAttributes + + EXExtensionPointIdentifier + com.apple.fskit.fsmodule + FSActivateOptionSyntax + + shortOptions + o: + + FSMediaTypes + + FSPersonalities + + FSRequiresSecurityScopedPathURLResources + + FSShortName + codexfoldnative + FSSupportedSchemes + + codexfoldnative + + FSSupportsBlockResources + + FSSupportsGenericURLResources + + FSSupportsPathURLs + + FSSupportsServerURLs + + + LSMinimumSystemVersion + $(MACOSX_DEPLOYMENT_TARGET) + + diff --git a/platform/darwin/fskit/Extension/ProfileModule.swift b/platform/darwin/fskit/Extension/ProfileModule.swift new file mode 100644 index 0000000..f8d6f2b --- /dev/null +++ b/platform/darwin/fskit/Extension/ProfileModule.swift @@ -0,0 +1,940 @@ +import Darwin +import Dispatch +import ExtensionFoundation +import Foundation +import FSKit +import OSLog + +@main +struct CodexFoldFSKitModule: UnaryFileSystemExtension { + var fileSystem: FSUnaryFileSystem & FSUnaryFileSystemOperations { + CodexFoldFileSystem() + } +} + +final class CodexFoldFileSystem: FSUnaryFileSystem, FSUnaryFileSystemOperations { + private let logger = Logger( + subsystem: "vip.jstar.codexfold.fskitprofileprobe.module", + category: "resource" + ) + private let volumeID = FSVolume.Identifier(uuid: UUID(uuidString: "5D0CF927-75A7-48B0-BDAE-621D8F2E695B")!) + private let lock = NSLock() + private var activeResourceURL: URL? + private weak var activeVolume: CodexFoldVolume? + + func probeResource( + resource: FSResource, + replyHandler: @escaping (FSProbeResult?, (any Error)?) -> Void + ) { + let containerID = FSContainerIdentifier(uuid: volumeID.uuid) + replyHandler(.usable(name: "CodexFold", containerID: containerID), nil) + } + + func loadResource( + resource: FSResource, + options: FSTaskOptions, + replyHandler: @escaping (FSVolume?, (any Error)?) -> Void + ) { + guard let pathResource = resource as? FSPathURLResource else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + let url = pathResource.url + let scoped = url.startAccessingSecurityScopedResource() + logger.notice("loadResource started scoped=\(scoped, privacy: .public)") + do { + var isDirectory = ObjCBool(false) + guard FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory) else { + logger.error("resource path is missing") + throw POSIXError(.ENOENT) + } + logger.notice("resource inspected directory=\(isDirectory.boolValue, privacy: .public)") + let descriptorURL = isDirectory.boolValue + ? url.appendingPathComponent("descriptor.bin", isDirectory: false) + : url + let descriptorData = try Data(contentsOf: descriptorURL) + logger.notice("descriptor read bytes=\(descriptorData.count, privacy: .public)") + let descriptor = try WireDescriptor(data: descriptorData) + logger.notice("descriptor decoded generation=\(descriptor.generation, privacy: .public)") + logger.notice("connecting to daemon socket") + let client = try DaemonClient(descriptor: descriptor) + logger.notice("daemon socket connected; sending ping") + try client.ping() + logger.notice("daemon ping succeeded") + let volume = try CodexFoldVolume(volumeID: volumeID, client: client) + lock.lock() + activeResourceURL = scoped ? url : nil + activeVolume = volume + lock.unlock() + containerStatus = .ready + replyHandler(volume, nil) + } catch { + logger.error("loadResource failed: \(String(describing: error), privacy: .public)") + if scoped { + url.stopAccessingSecurityScopedResource() + } + replyHandler(nil, error) + } + } + + func unloadResource(resource: FSResource, options: FSTaskOptions) async throws { + let (volume, url) = lock.withLock { () -> (CodexFoldVolume?, URL?) in + let volume = activeVolume + let url = activeResourceURL + activeVolume = nil + activeResourceURL = nil + return (volume, url) + } + try volume?.synchronizeNow() + url?.stopAccessingSecurityScopedResource() + containerStatus = .notReady(status: POSIXError(.EAGAIN)) + } +} + +private final class CodexFoldIO { + private static let readAheadBytes = 1 * 1024 * 1024 + + let connection: WireConnection + let handle: UInt64 + let writable: Bool + private let cacheLock = NSLock() + private var readCacheOffset: Int64 = 0 + private var readCache = Data() + + init(connection: WireConnection, handle: UInt64, writable: Bool) { + self.connection = connection + self.handle = handle + self.writable = writable + } + + func read(client: DaemonClient, offset: Int64, length: Int) throws -> Data { + guard offset >= 0, length >= 0 else { throw POSIXError(.EINVAL) } + guard length > 0 else { return Data() } + if writable { + return try client.read(handle: handle, offset: offset, length: length, connection: connection) + } + if length >= Self.readAheadBytes { + return try client.read(handle: handle, offset: offset, length: length, connection: connection) + } + + cacheLock.lock() + defer { cacheLock.unlock() } + if offset >= readCacheOffset { + let start = offset - readCacheOffset + if start <= Int64(readCache.count), Int64(length) <= Int64(readCache.count) - start { + let lower = Int(start) + return readCache.subdata(in: lower..<(lower + length)) + } + } + + let blockSize = Int64(Self.readAheadBytes) + let fetchOffset = offset / blockSize * blockSize + let requestedEnd = offset - fetchOffset + Int64(length) + let fetchLength = Int(max(blockSize, requestedEnd)) + let fetched = try client.read(handle: handle, offset: fetchOffset, length: fetchLength, connection: connection) + readCacheOffset = fetchOffset + readCache = fetched + + let lower = Int(offset - fetchOffset) + guard lower < fetched.count else { return Data() } + return fetched.subdata(in: lower.. CodexFoldIO? { + lock.lock() + defer { lock.unlock() } + return storedIO + } + + func replaceIO(_ io: CodexFoldIO?) -> CodexFoldIO? { + lock.lock() + let previous = storedIO + storedIO = io + lock.unlock() + return previous + } + + func invalidateReadCache() { + lock.lock() + let current = storedIO + lock.unlock() + current?.invalidateReadCache() + } +} + +private final class CodexFoldVolume: FSVolume, FSVolume.Handler, FSVolume.ReadWriteHandler, FSVolume.DataCacheHandler, FSVolume.XattrHandler { + private let client: DaemonClient + private let itemLock = NSLock() + private var items: [UInt64: CodexFoldItem] = [:] + private var rootItem: CodexFoldItem + private var namespaceVersion: UInt64 + private var namespaceTimer: DispatchSourceTimer? + + init(volumeID: FSVolume.Identifier, client: DaemonClient) throws { + self.client = client + let rootEntry = try client.getattr("/") + rootItem = CodexFoldItem(entry: rootEntry) + namespaceVersion = rootEntry.namespaceID + items[rootEntry.nodeID] = rootItem + super.init(volumeID: volumeID, volumeName: FSFileName(string: "CodexFold")) + } + + var supportedVolumeCapabilities: FSVolume.SupportedCapabilities { + let capabilities = FSVolume.SupportedCapabilities() + capabilities.supportsPersistentObjectIDs = false + capabilities.supportsSymbolicLinks = false + capabilities.supportsHardLinks = false + capabilities.supportsJournal = true + capabilities.supportsActiveJournal = true + capabilities.supportsSparseFiles = false + capabilities.supportsFastStatFS = true + capabilities.supports2TBFiles = true + capabilities.supports64BitObjectIDs = true + capabilities.supportsHiddenFiles = true + capabilities.caseFormat = .sensitive + return capabilities + } + + var volumeStatistics: FSStatFSResult { + let result = FSStatFSResult(fileSystemTypeName: "codexfold") + do { + let stat = try client.statfs() + result.blockSize = Int(stat.blockSize) + result.ioSize = Int(stat.ioSize) + result.totalBytes = stat.totalBytes + result.availableBytes = stat.availableBytes + result.freeBytes = stat.freeBytes + result.usedBytes = stat.usedBytes + result.totalFiles = stat.totalFiles + result.freeFiles = stat.freeFiles + } catch { + result.blockSize = 4096 + result.ioSize = 4 * 1024 * 1024 + result.totalBytes = 1 << 40 + result.availableBytes = 1 << 39 + result.freeBytes = 1 << 39 + result.usedBytes = 1 << 39 + result.totalFiles = 1 << 32 + result.freeFiles = 1 << 31 + } + return result + } + + var maximumLinkCount: Int { 1 } + var maximumNameLength: Int { 255 } + var maximumFileSize: UInt64 { UInt64.max >> 1 } + var maximumXattrSize: Int { 16 * 1024 * 1024 - 8192 } + var restrictsOwnershipChanges: Bool { false } + var truncatesLongNames: Bool { false } + var enableOpenUnlinkEmulation: Bool { true } + + func activate( + options: FSTaskOptions, + replyHandler: @escaping (FSActivateResult?, (any Error)?) -> Void + ) { + do { + let entry = try client.getattr("/") + rootItem.update(entry) + itemLock.lock() + items = [entry.nodeID: rootItem] + namespaceVersion = entry.namespaceID + itemLock.unlock() + startNamespaceMonitor() + replyHandler(FSActivateResult(rootItem: rootItem), nil) + } catch { + replyHandler(nil, error) + } + } + + func deactivate(options: FSDeactivateOptions, replyHandler: @escaping ((any Error)?) -> Void) { + stopNamespaceMonitor() + closeAllItems() + replyHandler(nil) + } + + func mount(options: FSTaskOptions, replyHandler: @escaping ((any Error)?) -> Void) { + do { + try client.ping() + replyHandler(nil) + } catch { + replyHandler(error) + } + } + + func unmount(replyHandler: @escaping () -> Void) { + stopNamespaceMonitor() + try? synchronizeNow() + closeAllItems() + replyHandler() + } + + func synchronize(flags: FSSyncFlags, replyHandler: @escaping ((any Error)?) -> Void) { + do { + try synchronizeNow() + replyHandler(nil) + } catch { + replyHandler(error) + } + } + + func synchronizeNow() throws { + try client.sync() + } + + func lookupItem( + named name: FSFileName, + in directory: FSItem, + context: FSContext, + replyHandler: @escaping (FSLookupItemResult?, (any Error)?) -> Void + ) { + guard let directory = directory as? CodexFoldItem, directory.entry.type == .directory else { + replyHandler(nil, POSIXError(.ENOTDIR)) + return + } + guard let nameString = name.string else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let entry = try client.getattr(join(directory.entry.path, nameString)) + let item = item(for: entry) + replyHandler( + FSLookupItemResult(foundItem: item, itemName: FSFileName(string: entry.name), itemAttributes: attributes(for: entry)), + nil + ) + } catch { + replyHandler(nil, error) + } + } + + func reclaimItem(_ item: FSItem, replyHandler: @escaping ((any Error)?) -> Void) { + guard let item = item as? CodexFoldItem else { + replyHandler(POSIXError(.EINVAL)) + return + } + itemLock.lock() + let reclaimed = item.tryReclaim { [self] in + closeIO(item.replaceIO(nil)) + if items[item.entry.nodeID] === item && item !== rootItem { + items.removeValue(forKey: item.entry.nodeID) + } + } + itemLock.unlock() + replyHandler(reclaimed ? nil : nil) + } + + func createItem( + named name: FSFileName, + type: FSItem.ItemType, + in directory: FSItem, + attributes newAttributes: FSItem.SetAttributesRequest, + context: FSContext, + replyHandler: @escaping (FSCreateItemResult?, (any Error)?) -> Void + ) { + guard let directory = directory as? CodexFoldItem, directory.entry.type == .directory else { + replyHandler(nil, POSIXError(.ENOTDIR)) + return + } + guard let nameString = name.string else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + let itemPath = join(directory.entry.path, nameString) + do { + var entry: WireEntry + switch type { + case .file: + let created = try client.create(itemPath, flags: O_RDWR | O_APPEND) + try client.handleOperation(.release, handle: created.1, connection: created.0) + created.0.close() + entry = created.2 + case .directory: + try client.mkdir(itemPath, mode: newAttributes.isValid(.mode) ? newAttributes.mode : 0o700) + entry = try client.getattr(itemPath) + default: + throw POSIXError(.ENOTSUP) + } + try applyAttributes(newAttributes, path: itemPath, type: type) + entry = try client.getattr(itemPath) + let item = item(for: entry) + let directoryEntry = try client.getattr(directory.entry.path) + directory.update(directoryEntry) + replyHandler( + FSCreateItemResult( + newItem: item, + newItemName: FSFileName(string: entry.name), + newItemAttributes: attributes(for: entry), + directoryAttributes: attributes(for: directoryEntry), + freeSpace: nil + ), + nil + ) + } catch { + replyHandler(nil, error) + } + } + + func createSymbolicLink( + named name: FSFileName, + in directory: FSItem, + attributes: FSItem.SetAttributesRequest, + linkContents: FSFileName, + context: FSContext, + replyHandler: @escaping (FSCreateSymlinkResult?, (any Error)?) -> Void + ) { + replyHandler(nil, POSIXError(.ENOTSUP)) + } + + func createLink( + to item: FSItem, + named name: FSFileName, + in directory: FSItem, + context: FSContext, + replyHandler: @escaping (FSCreateLinkResult?, (any Error)?) -> Void + ) { + replyHandler(nil, POSIXError(.ENOTSUP)) + } + + func renameItem( + _ item: FSItem, + inDirectory sourceDirectory: FSItem, + named sourceName: FSFileName, + to destinationName: FSFileName, + inDirectory destinationDirectory: FSItem, + overItem: FSItem?, + context: FSContext, + replyHandler: @escaping (FSRenameItemResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem, + let sourceDirectory = sourceDirectory as? CodexFoldItem, + let destinationDirectory = destinationDirectory as? CodexFoldItem else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + guard let sourceNameString = sourceName.string, let destinationNameString = destinationName.string else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + let sourcePath = join(sourceDirectory.entry.path, sourceNameString) + let destinationPath = join(destinationDirectory.entry.path, destinationNameString) + do { + try client.rename(sourcePath, destinationPath) + let renamedEntry = try client.getattr(destinationPath) + item.update(renamedEntry) + let sourceEntry = try client.getattr(sourceDirectory.entry.path) + let destinationEntry = sourceDirectory === destinationDirectory ? sourceEntry : try client.getattr(destinationDirectory.entry.path) + sourceDirectory.update(sourceEntry) + destinationDirectory.update(destinationEntry) + if let overItem = overItem as? CodexFoldItem { + overItem.markDeleted() + removeCached(overItem) + } + replyHandler( + FSRenameItemResult( + newName: FSFileName(string: renamedEntry.name), + renamedItemAttributes: attributes(for: renamedEntry), + sourceDirectoryAttributes: attributes(for: sourceEntry), + destinationDirectoryAttributes: attributes(for: destinationEntry), + overItemAttributes: nil, + freeSpace: nil + ), + nil + ) + } catch { + replyHandler(nil, error) + } + } + + func removeItem( + _ item: FSItem, + named name: FSFileName, + from directory: FSItem, + context: FSContext, + replyHandler: @escaping (FSRemoveItemResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem, let directory = directory as? CodexFoldItem else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + guard let nameString = name.string else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + let removedEntry = item.entry + do { + try client.remove(join(directory.entry.path, nameString), directory: removedEntry.type == .directory) + item.markDeleted() + closeIO(item.replaceIO(nil)) + removeCached(item) + let directoryEntry = try client.getattr(directory.entry.path) + directory.update(directoryEntry) + replyHandler( + FSRemoveItemResult( + itemAttributes: attributes(for: removedEntry), + directoryAttributes: attributes(for: directoryEntry), + freeSpace: nil + ), + nil + ) + } catch { + replyHandler(nil, error) + } + } + + func getAttributes( + _ desiredAttributes: FSItem.GetAttributesRequest, + of item: FSItem, + context: FSContext, + replyHandler: @escaping (FSGetAttributesResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let entry = try client.getattr(item.entry.path) + item.update(entry) + replyHandler(FSGetAttributesResult(attributes: attributes(for: entry)), nil) + } catch { + replyHandler(nil, error) + } + } + + func setAttributes( + _ newAttributes: FSItem.SetAttributesRequest, + on item: FSItem, + context: FSContext, + replyHandler: @escaping (FSSetAttributesResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + item.invalidateReadCache() + try applyAttributes(newAttributes, path: item.entry.path, type: itemType(item.entry.type)) + let entry = try client.getattr(item.entry.path) + item.update(entry) + replyHandler(FSSetAttributesResult(attributes: attributes(for: entry), freeSpace: nil), nil) + } catch { + replyHandler(nil, error) + } + } + + func enumerateDirectory( + _ directory: FSItem, + startingAt cookie: FSDirectoryCookie, + verifier: FSDirectoryVerifier, + attributes desiredAttributes: FSItem.GetAttributesRequest?, + packer: FSDirectoryEntryPacker, + context: FSContext, + replyHandler: @escaping (FSEnumerateDirectoryResult?, (any Error)?) -> Void + ) { + guard let directory = directory as? CodexFoldItem, directory.entry.type == .directory else { + replyHandler(nil, POSIXError(.ENOTDIR)) + return + } + do { + let entries = try client.readDir(directory.entry.path) + let currentVersion = try client.namespaceVersion() + let start = Int(cookie.rawValue) + guard start >= 0, start <= entries.count else { + throw POSIXError(.EINVAL) + } + if verifier != .initial, verifier.rawValue != currentVersion { + throw POSIXError(.ESTALE) + } + for index in start.. Void + ) { + replyHandler(nil, POSIXError(.ENOTSUP)) + } + + func getXattr( + named name: FSFileName, + of item: FSItem, + context: FSContext, + replyHandler: @escaping (FSGetXattrResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem, let nameString = name.string else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let value = try client.getXattr(item.entry.path, name: nameString) + guard let result = FSGetXattrResult(xattrValue: value) else { + throw POSIXError(.EIO) + } + replyHandler(result, nil) + } catch { + replyHandler(nil, error) + } + } + + func setXattr( + named name: FSFileName, + to value: Data?, + on item: FSItem, + policy: FSVolume.SetXattrPolicy, + context: FSContext, + replyHandler: @escaping (FSSetXattrResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem, let nameString = name.string else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + try client.setXattr(item.entry.path, name: nameString, value: value ?? Data(), policy: UInt32(policy.rawValue)) + if let entry = try? client.getattr(item.entry.path) { + item.update(entry) + } + guard let result = FSSetXattrResult(freeSpace: nil) else { + throw POSIXError(.EIO) + } + replyHandler(result, nil) + } catch { + replyHandler(nil, error) + } + } + + func listXattrs( + of item: FSItem, + context: FSContext, + replyHandler: @escaping (FSListXattrsResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let names = try client.listXattrs(item.entry.path).map { FSFileName(string: $0) } + guard let result = FSListXattrsResult(xattrNames: names) else { + throw POSIXError(.EIO) + } + replyHandler(result, nil) + } catch { + replyHandler(nil, error) + } + } + + func read( + from item: FSItem, + at offset: off_t, + length: Int, + into buffer: FSMutableFileDataBuffer, + replyHandler: @escaping (FSReadFileResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem, item.entry.type == .file, offset >= 0 else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let io = try ensureIO(item, writable: false) + let data = try io.read(client: client, offset: Int64(offset), length: length) + _ = buffer.withUnsafeMutableBytes { destination in + data.copyBytes(to: destination.bindMemory(to: UInt8.self)) + } + let entry = item.entry + replyHandler(FSReadFileResult(bytesRead: data.count, itemAttributes: attributes(for: entry)), nil) + } catch { + replyHandler(nil, error) + } + } + + func write( + contents: Data, + to item: FSItem, + at offset: off_t, + replyHandler: @escaping (FSWriteFileResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem, item.entry.type == .file, offset >= 0 else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let io = try ensureIO(item, writable: true) + io.invalidateReadCache() + let count = try client.write(handle: io.handle, offset: Int64(offset), data: contents, connection: io.connection) + let entry = try client.getattr(item.entry.path) + item.update(entry) + replyHandler(FSWriteFileResult(bytesWritten: count, itemAttributes: attributes(for: entry), freeSpace: nil), nil) + } catch { + replyHandler(nil, error) + } + } + + func open( + _ item: FSItem, + modes: FSVolume.OpenModes, + cacheMode: FSVolume.DataCacheMode, + context: FSContext, + replyHandler: @escaping (FSOpenItemResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let writable = modes.contains(.write) + if item.entry.type == .file { + _ = try ensureIO(item, writable: writable) + } + let coherency: FSVolume.KernelCacheCoherencyType = !writable && cacheMode != .none ? .readCache : .noCache + replyHandler(FSOpenItemResult(grantedCoherency: coherency), nil) + } catch { + replyHandler(nil, error) + } + } + + func close(_ item: FSItem, context: FSContext, replyHandler: @escaping () -> Void) { + if let item = item as? CodexFoldItem { + let closed = item.replaceIO(nil) + closeIO(closed) + if closed?.writable == true { + if let entry = try? client.getattr(item.entry.path) { + item.update(entry) + } + _ = setCacheState(for: item, cacheMode: .none, coherencyType: .noCache, action: .revoke) + } + } + replyHandler() + } + + func upgrade( + _ item: FSItem, + cacheMode: FSVolume.DataCacheMode, + context: FSContext, + replyHandler: @escaping (FSUpgradeItemResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let writable = cacheMode == .readWriteWithCache + if item.entry.type == .file { + _ = try ensureIO(item, writable: writable) + } + let coherency: FSVolume.KernelCacheCoherencyType = writable ? .noCache : .readCache + replyHandler(FSUpgradeItemResult(grantedCoherency: coherency), nil) + } catch { + replyHandler(nil, error) + } + } + + private func item(for entry: WireEntry) -> CodexFoldItem { + itemLock.lock() + defer { itemLock.unlock() } + if let existing = items[entry.nodeID] { + existing.update(entry) + return existing + } + let item = CodexFoldItem(entry: entry) + items[entry.nodeID] = item + return item + } + + private func removeCached(_ item: CodexFoldItem) { + itemLock.lock() + if items[item.entry.nodeID] === item { + items.removeValue(forKey: item.entry.nodeID) + } + itemLock.unlock() + } + + private func ensureIO(_ item: CodexFoldItem, writable: Bool) throws -> CodexFoldIO { + if let existing = item.io(), !writable || existing.writable { + return existing + } + closeIO(item.replaceIO(nil)) + let connection = try client.newConnection() + var flags: Int32 = writable ? O_RDWR : O_RDONLY + if writable && item.entry.path.hasSuffix(".jsonl") && !item.entry.name.hasPrefix("._") { + flags |= O_APPEND + flags |= Int32(bitPattern: 1 << 31) + } + do { + let handle = try client.open(item.entry.path, flags: flags, connection: connection) + let io = CodexFoldIO(connection: connection, handle: handle, writable: writable) + closeIO(item.replaceIO(io)) + return io + } catch { + connection.close() + throw error + } + } + + private func applyAttributes( + _ request: FSItem.SetAttributesRequest, + path: String, + type: FSItem.ItemType + ) throws { + if type == .file, request.isValid(.size) { + try client.truncate(path, size: request.size) + request.consumedAttributes.insert(.size) + } + + var valid: UInt32 = 0 + if request.isValid(.mode) { valid |= 1 << 0 } + if request.isValid(.uid) { valid |= 1 << 1 } + if request.isValid(.gid) { valid |= 1 << 2 } + if request.isValid(.accessTime) { valid |= 1 << 3 } + if request.isValid(.modifyTime) { valid |= 1 << 4 } + guard valid != 0 else { return } + + try client.setAttributes( + path, + valid: valid, + mode: request.mode, + uid: request.uid, + gid: request.gid, + accessTime: request.accessTime, + modifyTime: request.modifyTime + ) + if request.isValid(.mode) { request.consumedAttributes.insert(.mode) } + if request.isValid(.uid) { request.consumedAttributes.insert(.uid) } + if request.isValid(.gid) { request.consumedAttributes.insert(.gid) } + if request.isValid(.accessTime) { request.consumedAttributes.insert(.accessTime) } + if request.isValid(.modifyTime) { request.consumedAttributes.insert(.modifyTime) } + } + + private func closeIO(_ io: CodexFoldIO?) { + guard let io else { return } + if io.writable { + try? client.handleOperation(.fsync, handle: io.handle, connection: io.connection) + } + try? client.handleOperation(.release, handle: io.handle, connection: io.connection) + io.connection.close() + } + + private func closeAllItems() { + itemLock.lock() + let snapshot = Array(items.values) + itemLock.unlock() + for item in snapshot { + closeIO(item.replaceIO(nil)) + } + } + + private func startNamespaceMonitor() { + stopNamespaceMonitor() + let timer = DispatchSource.makeTimerSource(queue: DispatchQueue(label: "vip.jstar.codexfold.fskit.namespace")) + timer.schedule(deadline: .now() + .milliseconds(250), repeating: .milliseconds(500), leeway: .milliseconds(100)) + timer.setEventHandler { [weak self] in self?.refreshNamespace() } + namespaceTimer = timer + timer.resume() + } + + private func stopNamespaceMonitor() { + namespaceTimer?.cancel() + namespaceTimer = nil + } + + private func refreshNamespace() { + guard let current = try? client.namespaceVersion() else { return } + itemLock.lock() + if current == namespaceVersion { + itemLock.unlock() + return + } + namespaceVersion = current + let staleItems = items.values.filter { $0 !== rootItem } + items = [rootItem.entry.nodeID: rootItem] + itemLock.unlock() + if let rootEntry = try? client.getattr("/") { + rootItem.update(rootEntry) + } + for item in staleItems { + item.invalidateReadCache() + _ = setCacheState(for: item, cacheMode: .none, coherencyType: .noCache, action: .revoke) + } + } + + private func attributes(for entry: WireEntry) -> FSItem.Attributes { + let result = FSItem.Attributes() + result.uid = entry.uid + result.gid = entry.gid + result.linkCount = 1 + result.fileID = FSItem.Identifier(rawValue: entry.nodeID)! + result.parentID = FSItem.Identifier(rawValue: entry.parentID)! + result.mode = entry.mode + result.type = itemType(entry.type) + result.size = entry.type == .directory ? 0 : entry.size + result.allocSize = entry.type == .directory ? 0 : entry.allocSize + result.modifyTime = entry.modifyTime + result.changeTime = entry.changeTime + result.accessTime = entry.accessTime + return result + } + + private func itemType(_ type: WireEntryType) -> FSItem.ItemType { + switch type { + case .file: return .file + case .directory: return .directory + case .symlink: return .symlink + case .unknown: return .unknown + } + } + + private func join(_ directory: String, _ name: String) -> String { + if directory == "/" { return "/" + name } + return directory + "/" + name + } +} diff --git a/platform/darwin/fskit/Extension/Wire.swift b/platform/darwin/fskit/Extension/Wire.swift new file mode 100644 index 0000000..4095741 --- /dev/null +++ b/platform/darwin/fskit/Extension/Wire.swift @@ -0,0 +1,622 @@ +import Darwin +import Foundation + +private let wireMagic = Data([0x43, 0x46, 0x53, 0x50]) +private let descriptorMagic = Data([0x43, 0x46, 0x53, 0x52]) +private let wireVersion: UInt16 = 2 +private let wireHeaderSize = 40 +private let defaultMaxPayload = 16 * 1024 * 1024 + +enum WireOperation: UInt8 { + case hello = 1 + case ping + case getattr + case readDir + case open + case create + case read + case write + case fsync + case flush + case release + case truncate + case mkdir + case rename + case unlink + case rmdir + case statfs + case sync + case namespaceVersion + case setattr + case getXattr + case setXattr + case listXattrs +} + +enum WireEntryType: UInt8 { + case unknown = 0 + case file + case directory + case symlink +} + +struct WireDescriptor { + let generation: UInt64 + let socketPath: String + let token: Data + + init(resourceURL: URL) throws { + var isDirectory = ObjCBool(false) + guard FileManager.default.fileExists(atPath: resourceURL.path, isDirectory: &isDirectory) else { + throw POSIXError(.ENOENT) + } + let descriptorURL = isDirectory.boolValue + ? resourceURL.appendingPathComponent("descriptor.bin", isDirectory: false) + : resourceURL + try self.init(data: Data(contentsOf: descriptorURL)) + } + + init(data: Data) throws { + var reader = WireReader(data) + guard try reader.raw(count: 4) == descriptorMagic else { + throw POSIXError(.EPROTO) + } + guard try reader.uint16() == wireVersion else { + throw POSIXError(.EPROTONOSUPPORT) + } + _ = try reader.uint16() + generation = try reader.uint64() + socketPath = try reader.string(limit: 4096) + token = try reader.bytes(limit: 256) + try reader.finish() + guard generation != 0, !socketPath.isEmpty, token.count >= 16 else { + throw POSIXError(.EINVAL) + } + } +} + +struct WireEntry { + let path: String + let name: String + let nodeID: UInt64 + let parentID: UInt64 + let type: WireEntryType + let mode: UInt32 + let uid: UInt32 + let gid: UInt32 + let size: UInt64 + let allocSize: UInt64 + let modifyTime: timespec + let changeTime: timespec + let accessTime: timespec + let namespaceID: UInt64 + + init(reader: inout WireReader) throws { + path = try reader.string(limit: 1 << 20) + name = try reader.string(limit: 4096) + nodeID = try reader.uint64() + parentID = try reader.uint64() + guard let type = WireEntryType(rawValue: try reader.uint8()) else { + throw POSIXError(.EPROTO) + } + self.type = type + mode = try reader.uint32() + uid = try reader.uint32() + gid = try reader.uint32() + size = try reader.uint64() + allocSize = try reader.uint64() + modifyTime = try reader.time() + changeTime = try reader.time() + accessTime = try reader.time() + namespaceID = try reader.uint64() + } +} + +struct WireStatFS { + let blockSize: UInt32 + let ioSize: UInt32 + let totalBytes: UInt64 + let availableBytes: UInt64 + let freeBytes: UInt64 + let usedBytes: UInt64 + let totalFiles: UInt64 + let freeFiles: UInt64 + + init(reader: inout WireReader) throws { + blockSize = try reader.uint32() + ioSize = try reader.uint32() + totalBytes = try reader.uint64() + availableBytes = try reader.uint64() + freeBytes = try reader.uint64() + usedBytes = try reader.uint64() + totalFiles = try reader.uint64() + freeFiles = try reader.uint64() + } +} + +struct WireWriter { + private(set) var data = Data() + + mutating func raw(_ value: Data) { + data.append(value) + } + + mutating func uint8(_ value: UInt8) { + data.append(value) + } + + mutating func uint16(_ value: UInt16) { + appendFixed(value) + } + + mutating func uint32(_ value: UInt32) { + appendFixed(value) + } + + mutating func uint64(_ value: UInt64) { + appendFixed(value) + } + + mutating func int64(_ value: Int64) { + appendFixed(UInt64(bitPattern: value)) + } + + mutating func bytes(_ value: Data) { + uint32(UInt32(value.count)) + raw(value) + } + + mutating func string(_ value: String) { + bytes(Data(value.utf8)) + } + + mutating func time(_ value: timespec) { + int64(Int64(value.tv_sec)) + uint32(UInt32(value.tv_nsec)) + } + + private mutating func appendFixed(_ value: T) { + var littleEndian = value.littleEndian + withUnsafeBytes(of: &littleEndian) { data.append(contentsOf: $0) } + } +} + +struct WireReader { + private let data: Data + private var offset = 0 + + init(_ data: Data) { + self.data = data + } + + mutating func raw(count: Int) throws -> Data { + guard count >= 0, offset <= data.count, data.count - offset >= count else { + throw POSIXError(.EPROTO) + } + let result = data.subdata(in: offset..<(offset + count)) + offset += count + return result + } + + mutating func uint8() throws -> UInt8 { + let value = try raw(count: 1) + return value[value.startIndex] + } + + mutating func uint16() throws -> UInt16 { + try readFixed(UInt16.self) + } + + mutating func uint32() throws -> UInt32 { + try readFixed(UInt32.self) + } + + mutating func uint64() throws -> UInt64 { + try readFixed(UInt64.self) + } + + mutating func int64() throws -> Int64 { + Int64(bitPattern: try uint64()) + } + + mutating func bytes(limit: Int) throws -> Data { + let count = Int(try uint32()) + guard count <= limit else { + throw POSIXError(.E2BIG) + } + return try raw(count: count) + } + + mutating func string(limit: Int) throws -> String { + guard let value = String(data: try bytes(limit: limit), encoding: .utf8) else { + throw POSIXError(.EILSEQ) + } + return value + } + + mutating func time() throws -> timespec { + let seconds = try int64() + let nanoseconds = try uint32() + guard nanoseconds < 1_000_000_000 else { + throw POSIXError(.EPROTO) + } + return timespec(tv_sec: Int(seconds), tv_nsec: Int(nanoseconds)) + } + + mutating func finish() throws { + guard offset == data.count else { + throw POSIXError(.EPROTO) + } + } + + private mutating func readFixed(_ type: T.Type) throws -> T { + let value = try raw(count: MemoryLayout.size) + return value.withUnsafeBytes { bytes in + T(littleEndian: bytes.loadUnaligned(as: T.self)) + } + } +} + +private struct WireFrame { + let operation: WireOperation + let requestID: UInt64 + let generation: UInt64 + let status: Int32 + let payload: Data +} + +final class WireConnection { + private let lock = NSLock() + private var descriptor: WireDescriptor + private var socket: Int32 = -1 + private var requestID: UInt64 = 1 + private var maxPayload = defaultMaxPayload + + init(descriptor: WireDescriptor) throws { + self.descriptor = descriptor + try connect() + var hello = WireWriter() + hello.bytes(descriptor.token) + var reader = WireReader(try requestLocked(operation: .hello, generation: 0, payload: hello.data)) + maxPayload = Int(try reader.uint32()) + _ = try reader.uint64() + try reader.finish() + guard maxPayload >= 4096 else { + throw POSIXError(.EPROTO) + } + } + + deinit { + closeSocket() + } + + func request(_ operation: WireOperation, payload: Data = Data()) throws -> Data { + lock.lock() + defer { lock.unlock() } + return try requestLocked(operation: operation, generation: descriptor.generation, payload: payload) + } + + func close() { + lock.lock() + closeSocket() + lock.unlock() + } + + private func connect() throws { + let descriptor = Darwin.socket(AF_UNIX, SOCK_STREAM, 0) + guard descriptor >= 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + var noSigPipe: Int32 = 1 + _ = setsockopt(descriptor, SOL_SOCKET, SO_NOSIGPIPE, &noSigPipe, socklen_t(MemoryLayout.size)) + var address = sockaddr_un() + address.sun_family = sa_family_t(AF_UNIX) + let pathBytes = Array(self.descriptor.socketPath.utf8CString) + guard pathBytes.count <= MemoryLayout.size(ofValue: address.sun_path) else { + Darwin.close(descriptor) + throw POSIXError(.ENAMETOOLONG) + } + withUnsafeMutableBytes(of: &address.sun_path) { destination in + destination.initializeMemory(as: UInt8.self, repeating: 0) + for (index, value) in pathBytes.enumerated() { + destination[index] = UInt8(bitPattern: value) + } + } + let result = withUnsafePointer(to: &address) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { socketAddress in + Darwin.connect(descriptor, socketAddress, socklen_t(MemoryLayout.size)) + } + } + guard result == 0 else { + let code = errno + Darwin.close(descriptor) + throw POSIXError(POSIXErrorCode(rawValue: code) ?? .EIO) + } + socket = descriptor + } + + private func requestLocked(operation: WireOperation, generation: UInt64, payload: Data) throws -> Data { + guard socket >= 0 else { + throw POSIXError(.ENOTCONN) + } + guard payload.count <= maxPayload else { + throw POSIXError(.E2BIG) + } + let currentID = requestID + requestID &+= 1 + var header = WireWriter() + header.raw(wireMagic) + header.uint16(wireVersion) + header.uint8(1) + header.uint8(operation.rawValue) + header.uint32(0) + header.uint64(currentID) + header.uint64(generation) + header.uint32(0) + header.uint32(UInt32(payload.count)) + header.uint32(0) + try writeAll(header.data) + try writeAll(payload) + + var responseReader = WireReader(try readExact(count: wireHeaderSize)) + guard try responseReader.raw(count: 4) == wireMagic, + try responseReader.uint16() == wireVersion, + try responseReader.uint8() == 2, + try responseReader.uint8() == operation.rawValue else { + throw POSIXError(.EPROTO) + } + _ = try responseReader.uint32() + guard try responseReader.uint64() == currentID, + try responseReader.uint64() == descriptor.generation else { + throw POSIXError(.EPROTO) + } + let status = Int32(bitPattern: try responseReader.uint32()) + let payloadLength = Int(try responseReader.uint32()) + _ = try responseReader.uint32() + try responseReader.finish() + guard payloadLength <= maxPayload else { + throw POSIXError(.E2BIG) + } + let responsePayload = try readExact(count: payloadLength) + if status != 0 { + throw POSIXError(POSIXErrorCode(rawValue: status) ?? .EIO) + } + return responsePayload + } + + private func readExact(count: Int) throws -> Data { + var result = Data(count: count) + var completed = 0 + while completed < count { + let amount = result.withUnsafeMutableBytes { bytes in + Darwin.read(socket, bytes.baseAddress!.advanced(by: completed), count - completed) + } + if amount == 0 { + throw POSIXError(.ECONNRESET) + } + if amount < 0 { + if errno == EINTR { continue } + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + completed += amount + } + return result + } + + private func writeAll(_ data: Data) throws { + var completed = 0 + while completed < data.count { + let amount = data.withUnsafeBytes { bytes in + Darwin.write(socket, bytes.baseAddress!.advanced(by: completed), data.count - completed) + } + if amount < 0 { + if errno == EINTR { continue } + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + guard amount > 0 else { + throw POSIXError(.EIO) + } + completed += amount + } + } + + private func closeSocket() { + if socket >= 0 { + Darwin.close(socket) + socket = -1 + } + } +} + +final class DaemonClient { + let descriptor: WireDescriptor + private let control: WireConnection + + init(descriptor: WireDescriptor) throws { + self.descriptor = descriptor + control = try WireConnection(descriptor: descriptor) + } + + func newConnection() throws -> WireConnection { + try WireConnection(descriptor: descriptor) + } + + func ping() throws { + _ = try control.request(.ping) + } + + func getattr(_ path: String) throws -> WireEntry { + var writer = WireWriter() + writer.string(path) + var reader = WireReader(try control.request(.getattr, payload: writer.data)) + let entry = try WireEntry(reader: &reader) + try reader.finish() + return entry + } + + func readDir(_ path: String) throws -> [WireEntry] { + var writer = WireWriter() + writer.string(path) + var reader = WireReader(try control.request(.readDir, payload: writer.data)) + let count = Int(try reader.uint32()) + var entries: [WireEntry] = [] + entries.reserveCapacity(count) + for _ in 0.. UInt64 { + var writer = WireWriter() + writer.string(path) + writer.uint32(UInt32(bitPattern: flags)) + var reader = WireReader(try connection.request(.open, payload: writer.data)) + let handle = try reader.uint64() + try reader.finish() + return handle + } + + func create(_ path: String, flags: Int32) throws -> (WireConnection, UInt64, WireEntry) { + let connection = try newConnection() + var writer = WireWriter() + writer.string(path) + writer.uint32(UInt32(bitPattern: flags)) + var reader = WireReader(try connection.request(.create, payload: writer.data)) + let handle = try reader.uint64() + let entry = try WireEntry(reader: &reader) + try reader.finish() + return (connection, handle, entry) + } + + func read(handle: UInt64, offset: Int64, length: Int, connection: WireConnection) throws -> Data { + var writer = WireWriter() + writer.uint64(handle) + writer.int64(offset) + writer.uint32(UInt32(length)) + var reader = WireReader(try connection.request(.read, payload: writer.data)) + let data = try reader.bytes(limit: defaultMaxPayload) + try reader.finish() + return data + } + + func write(handle: UInt64, offset: Int64, data: Data, connection: WireConnection) throws -> Int { + var writer = WireWriter() + writer.uint64(handle) + writer.int64(offset) + writer.bytes(data) + var reader = WireReader(try connection.request(.write, payload: writer.data)) + let count = Int(try reader.uint32()) + try reader.finish() + return count + } + + func handleOperation(_ operation: WireOperation, handle: UInt64, connection: WireConnection) throws { + var writer = WireWriter() + writer.uint64(handle) + _ = try connection.request(operation, payload: writer.data) + } + + func truncate(_ path: String, size: UInt64) throws { + guard size <= UInt64(Int64.max) else { throw POSIXError(.EFBIG) } + var writer = WireWriter() + writer.string(path) + writer.int64(Int64(size)) + _ = try control.request(.truncate, payload: writer.data) + } + + func mkdir(_ path: String, mode: UInt32) throws { + var writer = WireWriter() + writer.string(path) + writer.uint32(mode) + _ = try control.request(.mkdir, payload: writer.data) + } + + func rename(_ oldPath: String, _ newPath: String) throws { + var writer = WireWriter() + writer.string(oldPath) + writer.string(newPath) + _ = try control.request(.rename, payload: writer.data) + } + + func remove(_ path: String, directory: Bool) throws { + var writer = WireWriter() + writer.string(path) + _ = try control.request(directory ? .rmdir : .unlink, payload: writer.data) + } + + func statfs() throws -> WireStatFS { + var reader = WireReader(try control.request(.statfs)) + let result = try WireStatFS(reader: &reader) + try reader.finish() + return result + } + + func sync() throws { + _ = try control.request(.sync) + } + + func namespaceVersion() throws -> UInt64 { + var reader = WireReader(try control.request(.namespaceVersion)) + let version = try reader.uint64() + try reader.finish() + return version + } + + func setAttributes( + _ path: String, + valid: UInt32, + mode: UInt32, + uid: UInt32, + gid: UInt32, + accessTime: timespec, + modifyTime: timespec + ) throws { + var writer = WireWriter() + writer.string(path) + writer.uint32(valid) + writer.uint32(mode) + writer.uint32(uid) + writer.uint32(gid) + writer.time(accessTime) + writer.time(modifyTime) + _ = try control.request(.setattr, payload: writer.data) + } + + func getXattr(_ path: String, name: String) throws -> Data { + var writer = WireWriter() + writer.string(path) + writer.string(name) + var reader = WireReader(try control.request(.getXattr, payload: writer.data)) + let value = try reader.bytes(limit: defaultMaxPayload) + try reader.finish() + return value + } + + func setXattr(_ path: String, name: String, value: Data, policy: UInt32) throws { + var writer = WireWriter() + writer.string(path) + writer.string(name) + writer.uint32(policy) + writer.bytes(value) + _ = try control.request(.setXattr, payload: writer.data) + } + + func listXattrs(_ path: String) throws -> [String] { + var writer = WireWriter() + writer.string(path) + var reader = WireReader(try control.request(.listXattrs, payload: writer.data)) + let count = Int(try reader.uint32()) + guard count <= defaultMaxPayload / 4 else { + throw POSIXError(.E2BIG) + } + var attributes: [String] = [] + attributes.reserveCapacity(count) + for _ in 0.. 1 { + do { + exit(try runCommand(Array(CommandLine.arguments.dropFirst()))) + } catch { + fputs("CodexFoldFSKit: \(error)\n", stderr) + exit(1) + } + } + NSApplication.shared.setActivationPolicy(.accessory) + NSApplication.shared.terminate(nil) + } + + private static func runCommand(_ arguments: [String]) throws -> Int32 { + guard let root = FileManager.default.containerURL( + forSecurityApplicationGroupIdentifier: appGroupIdentifier + ) else { + throw POSIXError(.ENOENT) + } + switch arguments.first { + case "--app-group-path": + print(root.path) + return 0 + case "--app-group-write-probe": + let probe = root.appendingPathComponent("host-write-probe", isDirectory: false) + try Data("ok\n".utf8).write(to: probe, options: .atomic) + try FileManager.default.removeItem(at: probe) + return 0 + case "--run-helper": + guard arguments.count >= 2 else { + throw POSIXError(.EINVAL) + } + return try runHelper(executable: arguments[1], arguments: Array(arguments.dropFirst(2))) + default: + throw POSIXError(.EINVAL) + } + } + + private static func runHelper(executable: String, arguments: [String]) throws -> Int32 { + guard executable.hasPrefix("/") else { + throw POSIXError(.EINVAL) + } + let process = Process() + process.executableURL = URL(fileURLWithPath: executable) + process.arguments = arguments + process.standardInput = FileHandle.standardInput + process.standardOutput = FileHandle.standardOutput + process.standardError = FileHandle.standardError + + var environment: [String: String] = [:] + let parentEnvironment = ProcessInfo.processInfo.environment + for key in inheritedEnvironmentKeys { + if let value = parentEnvironment[key] { + environment[key] = value + } + } + environment["CODEXFOLD_LAUNCHER_PARENT_PID"] = String(Darwin.getpid()) + process.environment = environment + + var signalSources: [DispatchSourceSignal] = [] + for signalNumber in [SIGTERM, SIGINT, SIGHUP] { + Darwin.signal(signalNumber, SIG_IGN) + let source = DispatchSource.makeSignalSource(signal: signalNumber, queue: .global()) + source.setEventHandler { + if process.isRunning { + _ = Darwin.kill(process.processIdentifier, signalNumber) + } + } + source.resume() + signalSources.append(source) + } + + try process.run() + process.waitUntilExit() + signalSources.forEach { $0.cancel() } + if process.terminationReason == .uncaughtSignal { + return 128 + process.terminationStatus + } + return process.terminationStatus + } +} diff --git a/platform/darwin/fskit/Host/Info.plist b/platform/darwin/fskit/Host/Info.plist new file mode 100644 index 0000000..0fed7d2 --- /dev/null +++ b/platform/darwin/fskit/Host/Info.plist @@ -0,0 +1,26 @@ + + + + + CFBundleDisplayName + CodexFold FSKit + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + APPL + CFBundleShortVersionString + 0.1.0 + CFBundleVersion + 1 + LSMinimumSystemVersion + $(MACOSX_DEPLOYMENT_TARGET) + LSUIElement + + + diff --git a/platform/darwin/fskit/project.yml b/platform/darwin/fskit/project.yml new file mode 100644 index 0000000..1b0e5ef --- /dev/null +++ b/platform/darwin/fskit/project.yml @@ -0,0 +1,39 @@ +name: CodexFoldFSKit +options: + bundleIdPrefix: vip.jstar.codexfold +settings: + base: + MACOSX_DEPLOYMENT_TARGET: "27.0" + DEVELOPMENT_TEAM: Y987FUR837 + CODE_SIGN_STYLE: Automatic + SWIFT_VERSION: "5.0" +targets: + CodexFoldFSKit: + type: application + platform: macOS + sources: + - path: Host + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: vip.jstar.codexfold.fskitprofileprobe + INFOPLIST_FILE: Host/Info.plist + CODE_SIGN_ENTITLEMENTS: CodexFoldFSKit.entitlements + REGISTER_APP_GROUPS: YES + dependencies: + - target: CodexFoldFSKitModule + embed: true + CodexFoldFSKitModule: + type: extensionkit-extension + platform: macOS + sources: + - path: Extension + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: vip.jstar.codexfold.fskitprofileprobe.module + INFOPLIST_FILE: Extension/Info.plist + CODE_SIGN_ENTITLEMENTS: Extension/CodexFoldFSKitModule.entitlements + REGISTER_APP_GROUPS: YES + ENABLE_APP_SANDBOX: YES + SKIP_INSTALL: YES + APPLICATION_EXTENSION_API_ONLY: YES + LD_RUNPATH_SEARCH_PATHS: "$(inherited) @executable_path/../Frameworks @executable_path/../../../../Frameworks" From fc62a49ed0877c478134f7d1f6c5cb2a409e1a53 Mon Sep 17 00:00:00 2001 From: King Star Date: Sat, 18 Jul 2026 19:26:11 +0800 Subject: [PATCH 32/33] chore: establish repository collaboration standards --- .github/CODEOWNERS | 8 ++ .github/ISSUE_TEMPLATE/bug_report.yml | 58 ++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 5 ++ .github/ISSUE_TEMPLATE/feature_request.yml | 29 +++++++ .github/pull_request_template.md | 21 +++++ .github/workflows/ci.yml | 60 +++++++++++++- .gitignore | 1 + CODE_OF_CONDUCT.md | 22 ++++++ CONTRIBUTING.md | 63 +++++++++++++++ README.md | 7 +- SECURITY.md | 14 +++- docs/maintainer-guide.md | 79 +++++++++++++++++++ ...arent-session-filesystem-implementation.md | 4 +- ...1-transparent-session-filesystem-design.md | 12 +-- ...ion-filesystem-implementation-alignment.md | 18 ++--- docs/validation-macos-canary.md | 10 ++- 16 files changed, 389 insertions(+), 22 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/pull_request_template.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 docs/maintainer-guide.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..bef3f1f --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,8 @@ +* @jstar0 + +/docs/superpowers/specs/ @jstar0 +/internal/fold/ @jstar0 +/internal/pack/ @jstar0 +/internal/storage/ @jstar0 +/internal/vfs/ @jstar0 +/platform/darwin/fskit/ @jstar0 diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..2794e4d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,58 @@ +name: Bug report +description: Report a reproducible CodexFold defect using synthetic or redacted data. +title: "bug: " +labels: [bug] +body: + - type: markdown + attributes: + value: Do not attach real Codex rollouts, databases, credentials, private prompts, or unredacted logs. Use GitHub Security Advisories for vulnerabilities. + - type: input + id: version + attributes: + label: Version or commit + placeholder: v0.2.1 or commit SHA + validations: + required: true + - type: dropdown + id: platform + attributes: + label: Platform + options: + - macOS + - Linux + - Windows + - Other + validations: + required: true + - type: textarea + id: behavior + attributes: + label: Observed behavior + description: Include the command, sanitized error, and whether storage-only or filesystem-preview behavior was involved. + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected behavior + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: Synthetic reproduction + description: Provide minimal steps using generated or redacted fixtures. + validations: + required: true + - type: textarea + id: verification + attributes: + label: Verification already attempted + description: List doctor, tests, hashes, restart checks, or rollback attempts without private data. + - type: checkboxes + id: hygiene + attributes: + label: Data hygiene + options: + - label: I removed real session content, credentials, private prompts, local paths, and unredacted logs. + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..9d58c48 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Security vulnerability + url: https://github.com/jstar0/codexfold/security/advisories/new + about: Report vulnerabilities and sensitive findings privately. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..ad1ec8a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,29 @@ +name: Feature request +description: Propose a product behavior or engineering improvement. +title: "feat: " +labels: [enhancement] +body: + - type: textarea + id: problem + attributes: + label: Problem + description: Describe the concrete workflow or limitation, not only the proposed implementation. + validations: + required: true + - type: textarea + id: outcome + attributes: + label: Required outcome + description: State observable acceptance criteria. + validations: + required: true + - type: textarea + id: constraints + attributes: + label: Safety and compatibility constraints + description: Note byte identity, rollback, privacy, platform, performance, or release-gate implications. + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Include simpler options and why they are insufficient. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..25924dd --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,21 @@ +## Outcome + +Describe the user-visible or maintainer-visible result. + +## Safety Impact + +Describe effects on session bytes, routing, storage, filesystem behavior, service lifecycle, compatibility, and rollback. Write `None` only when none apply. + +## Verification + +List the exact commands and real environments used. Distinguish unit, synthetic, mounted-adapter, real-client, restart, and production evidence. + +## Checklist + +- [ ] The change matches the product contract and does not weaken a release gate. +- [ ] Tests cover the behavior or regression. +- [ ] `go test ./...`, race tests, vet, formatting, and required cross-builds pass. +- [ ] Platform-specific validation was run when platform code changed. +- [ ] Documentation and readiness language match the available evidence. +- [ ] No real rollout, credential, private prompt, local path, database, log, build artifact, or Xcode user state is included. +- [ ] Production Codex data and production service definitions were not used for development validation. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13fc7fa..2cef764 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,13 +4,43 @@ on: push: branches: [main] pull_request: + workflow_dispatch: permissions: contents: read +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: + quality: + name: quality + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - name: Check formatting + shell: bash + run: | + files="$(gofmt -l .)" + if [ -n "$files" ]; then + printf '%s\n' "$files" + exit 1 + fi + - name: Check module files + run: | + go mod tidy + git diff --exit-code -- go.mod go.sum + - run: go vet ./... + test: + name: test (${{ matrix.os }}) strategy: + fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} @@ -18,7 +48,35 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: "1.26.x" + go-version-file: go.mod cache: true - run: go test ./... -count=1 - run: go build ./cmd/codexfold + + race: + name: race + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - run: go test -race ./... -count=1 + + cross-build: + name: cross-build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - name: Build Linux and Windows artifacts + shell: bash + run: | + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o "$RUNNER_TEMP/codexfold-linux-amd64" ./cmd/codexfold + CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o "$RUNNER_TEMP/codexfold-windows-amd64.exe" ./cmd/codexfold + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test -c -o "$RUNNER_TEMP/codexfold-testfs-linux.test" ./internal/testfs + CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go test -c -o "$RUNNER_TEMP/codexfold-testfs-windows.test.exe" ./internal/testfs diff --git a/.gitignore b/.gitignore index 6059122..05d796c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /codexfold +/codexfold.exe /*.sqlite /*.sqlite-shm /*.sqlite-wal diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..7cec415 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,22 @@ +# Code of Conduct + +CodexFold contributors are expected to collaborate professionally, focus review on technical behavior and evidence, and respect the privacy and safety constraints of a project that handles local conversation data. + +## Expected Behavior + +- Be direct, respectful, and specific about technical concerns. +- Critique code, design, tests, and claims rather than people. +- Disclose uncertainty and distinguish observed evidence from inference. +- Protect private data and report sensitive findings through a private channel. +- Accept maintainer decisions on safety gates, even when they delay a feature or release. + +## Unacceptable Behavior + +- Harassment, personal attacks, discrimination, threats, or sustained disruption. +- Publishing private session content, credentials, personal information, or security details without authorization. +- Misrepresenting test coverage, readiness, provenance, or compatibility evidence. +- Pressuring contributors to bypass review, rollback, privacy, or production-safety requirements. + +## Enforcement + +Maintainers may edit or remove contributions, comments, or access that violate this policy. Report sensitive conduct issues privately to the repository owner through GitHub; use a private Security Advisory when the report also involves confidential data or a vulnerability. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..f520e01 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,63 @@ +# Contributing to CodexFold + +CodexFold handles private local session data and implements storage and filesystem behavior where a silent mismatch is unacceptable. Contributions should be small enough to review, explicit about safety boundaries, and backed by evidence that matches the claim. + +## Before You Start + +- Read the [product contract](docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md), the [implementation alignment](docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md), and the [maintainer guide](docs/maintainer-guide.md). +- Open an issue for product behavior, format, compatibility, or architecture changes before implementing a large change. +- Never use a real rollout, state database, credential, signing export, or private prompt as a fixture. +- Do not weaken a release gate to make an implementation pass. + +## Development Setup + +Requirements: + +- Go version declared in `go.mod`. +- Git. +- Xcode 27 and XcodeGen for native macOS FSKit work. +- Platform prerequisites only when testing the corresponding preview adapter. + +Run the common quality gate: + +```bash +./scripts/test-cross-platform.sh +git diff --check +``` + +For a focused change, run the smallest relevant package tests first, then the common gate before requesting review. + +## Branches and Commits + +- Create a branch from current `main`; use a descriptive prefix such as `feat/`, `fix/`, `docs/`, or `test/`. +- Use conventional, imperative commit subjects such as `fix: reject stale native writer state`. +- Keep generated files, binaries, user-specific Xcode state, local databases, and test artifacts out of commits. +- Update `platform/darwin/fskit/project.yml` first and regenerate the Xcode project; do not hand-maintain personal Xcode state. + +## Pull Requests + +- Explain the user-visible outcome, safety impact, and exact evidence. +- Distinguish unit, synthetic, mounted-adapter, real-client, host-restart, and production evidence. One category never substitutes for another. +- Add or update tests before changing a status or readiness claim. +- Keep production Codex homes and production service definitions untouched during development validation. +- Resolve review conversations and keep the branch current with `main` before merge. + +## Filesystem Changes + +Native filesystem work must remain isolated until every relevant gate passes. Use disposable homes, stores, native roots, mount points, and service labels. A test may not route a production SQLite record or remove a production JSONL source. + +The macOS terminal architecture is: + +```text +Codex CLI / Desktop + -> Apple-native Swift FSKit extension + -> versioned binary UDS IPC + -> Go CodexFold daemon + -> packfile + index + manifest + append delta + COW backing +``` + +NFS and FUSE-T are development fallback or historical evidence only. They are not acceptable replacements for native FSKit release gates. + +## Security + +Report vulnerabilities through GitHub Security Advisories. Public issues and pull requests must contain only synthetic or fully redacted data; see [SECURITY.md](SECURITY.md). diff --git a/README.md b/README.md index d33f8c4..58fc13e 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,11 @@ The requirements and release gates for normal JSONL paths backed transparently b The unreleased transparent-filesystem branch remains `fs-engine-preview`: -- macOS uses the explicitly selected synchronous FUSE-T NFS backend and has real Codex CLI/Desktop canary evidence. FUSE-T's FSKit backend was tested and rejected after a deterministic same-offset JSONL byte-loss failure. +- macOS now targets an Apple-native Swift FSKit extension connected over a versioned Unix-domain-socket protocol to the Go CodexFold daemon. Isolated mounted behavior, service crash recovery, and atomic app/binary rollback gates pass; read-ahead performance/coherency and full real Codex CLI/Desktop product acceptance remain open. +- The earlier synchronous FUSE-T NFS route remains historical validation evidence and a development fallback only. FUSE-T's third-party FSKit backend remains rejected after deterministic byte-loss and cache-invalidation failures; it is not the Apple-native FSKit implementation in this repository. - Linux FUSE3 has real unprivileged read, append, copy-on-write, truncate, archive rename, crash recovery, remount, performance, and `systemd --user` lifecycle evidence. - Windows has a WinFsp adapter and native Windows Service host that cross-compile, but no real Windows/WinFsp host has validated them yet. -- Retention, actual in-flight power loss, and the remaining platform-specific client and upgrade gates still block promotion. +- The production service and production Codex home remain disabled. Native FSKit performance/coherency, real CLI/Desktop acceptance, retention, actual in-flight power loss, and the remaining platform-specific client and upgrade gates still block promotion. See [the Linux FUSE3 validation](docs/validation-linux-fuse3.md) and [the macOS canary validation](docs/validation-macos-canary.md) for the evidence boundary. The default build remains storage-only; platform mounts require explicit build tags and installed host prerequisites. @@ -148,4 +149,4 @@ go vet ./... go build ./cmd/codexfold ``` -See [the architecture](docs/design.md), [Fold V1 format](docs/fold-v1.md), [v0.2 validation](docs/validation-v0.2.md), and [the transparent filesystem product contract](docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md). +See [the architecture](docs/design.md), [Fold V1 format](docs/fold-v1.md), [v0.2 validation](docs/validation-v0.2.md), [the transparent filesystem product contract](docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md), and [the maintainer guide](docs/maintainer-guide.md). diff --git a/SECURITY.md b/SECURITY.md index 415bb02..696b35b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,4 +2,16 @@ CodexFold processes local conversation rollouts that may contain secrets or private data. The project does not transmit scan inputs or report field contents. -Please report vulnerabilities privately through GitHub Security Advisories. Do not include real session files, credentials, or private prompts in public issues. +## Supported Code + +Security fixes target the latest release and the current `main` branch. Preview filesystem branches may change quickly and must not be treated as production-safe unless the repository explicitly publishes a platform readiness claim. + +## Reporting + +Please report vulnerabilities privately through GitHub Security Advisories. Do not include real session files, credentials, private prompts, local filesystem paths, service tokens, or unredacted logs in public issues. + +Include the affected version or commit, operating system, impact, and a minimal synthetic reproducer when possible. Maintainers will acknowledge the report, assess whether private coordination is required, and publish remediation details after a safe fix is available. + +## Data Handling + +Tests and bug reports must use generated or redacted fixtures. A contributor must never commit a real Codex rollout, Codex state database, credential, signing identity export, provisioning profile, or production service definition. diff --git a/docs/maintainer-guide.md b/docs/maintainer-guide.md new file mode 100644 index 0000000..1169e8e --- /dev/null +++ b/docs/maintainer-guide.md @@ -0,0 +1,79 @@ +# Maintainer Guide + +## Source of Truth + +Use this order when documents or code appear to disagree: + +1. The transparent filesystem product contract. +2. The implementation alignment document. +3. Current platform validation reports. +4. Current code and tests. +5. Historical plans and validation evidence. + +Changing architecture, safety guarantees, or readiness language requires updating the contract and alignment in the same pull request. + +## Current Architecture and Status + +The storage engine is released separately from the transparent filesystem preview. The macOS terminal candidate is Apple-native Swift FSKit -> versioned UDS -> Go daemon. Linux uses FUSE3 and Windows targets WinFsp. Production Codex routing remains disabled until the named platform gates pass. + +FUSE-T NFS evidence is retained to preserve regression knowledge. FUSE-T's own FSKit backend is rejected and must not be confused with the native Swift extension in `platform/darwin/fskit`. + +## Required Pull Request Gates + +Every pull request must pass: + +```bash +gofmt -l . +go mod tidy +git diff --exit-code -- go.mod go.sum +go test ./... -count=1 +go test -race ./... -count=1 +go vet ./... +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /tmp/codexfold-linux-amd64 ./cmd/codexfold +CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o /tmp/codexfold-windows-amd64.exe ./cmd/codexfold +git diff --check +``` + +Native macOS FSKit changes additionally require an XcodeGen consistency check and Release build with the Xcode version declared by the project. Commit `project.yml` and the regenerated project together. + +## Isolated Native FSKit Validation + +Never point development tests at `~/.codex`. Create a disposable Codex home, store, native root, mount point, service label, and service definition. Production `com.codexfold.fs` must remain disabled during preview work. + +Mounted behavior tests use explicit paths: + +```bash +CODEXFOLD_NATIVE_FSKIT_MOUNT=/absolute/disposable/mount \ +CODEXFOLD_NATIVE_FSKIT_NATIVE_ROOT=/absolute/disposable/native \ +go test ./internal/mountfs -run '^TestNativeFSKitMounted' -count=1 -v +``` + +An app or binary update must use the transactional service command. The updater must stop both launchd jobs, wait for daemon and supervisor process locks to release, install staged definitions/app/binary, verify Host-child ancestry, mount health, and running build SHA, and restore the previous generation on failure. Do not replace an active extension bundle manually. + +## Evidence Levels + +- Unit or fixture tests prove only the code path they exercise. +- Mounted tests prove adapter behavior against disposable data. +- Real CLI/Desktop tests prove current-client behavior only for the exact tested versions. +- Restart and crash matrices prove process recovery, not power-loss durability. +- A successful canary does not satisfy retention or production readiness. + +Readiness claims must use only the capability names defined in the product contract. + +## Data and Artifact Hygiene + +- Keep all test rollouts synthetic and valid JSONL when they use a `.jsonl` suffix. +- Use `.bin` for arbitrary filesystem mutation fixtures so native writer preflight cannot mistake them for Codex rollouts. +- Clean disposable mount and native-backing paths even when the mount disappears during a test. +- Do not commit DerivedData, built apps, binaries, databases, logs, `xcuserdata`, or provisioning profiles. +- Do not print inherited launchd environments or credentials in public logs. + +## Merge and Release Procedure + +1. Obtain an approving review and green required checks. +2. Squash merge into `main` unless preserving separate audited commits is materially useful. +3. Re-run platform-specific signed builds and isolated real-adapter gates for release candidates. +4. Verify release notes distinguish implemented, tested, preview, canary, and production-ready behavior. +5. Never delete retained native sources or enable bulk enrollment before the contract permits it. + +If a release or service update fails, preserve the failing evidence, restore the last verified app/binary/definition generation, verify exact bytes and build identity, and keep automatic enrollment disabled until the incident is understood. diff --git a/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md b/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md index c3c4f05..fc0fcff 100644 --- a/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md +++ b/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md @@ -6,7 +6,9 @@ **Architecture:** Extend Fold V1 with a block-addressable packed resolver, then place a platform-neutral exact-byte session engine above it. The engine composes an immutable manifest base with an append delta or verified writable backing; platform adapters only translate native file operations. Migration, compatibility quarantine, fallback, and promotion remain explicit journaled transactions, with real Codex routing disabled until shadow and platform gates pass. -**Tech Stack:** Go 1.26, zstd, Cobra, modernc SQLite, `cgofuse` v1.6.0 behind platform/build tags, FUSE-T 1.2.7 on macOS, FUSE3 on Linux, and WinFsp plus Windows SCM on Windows. +**Tech Stack:** Go 1.26, zstd, Cobra, modernc SQLite, an Apple-native Swift FSKit extension with versioned UDS IPC on macOS, FUSE3 on Linux, and WinFsp plus Windows SCM on Windows. FUSE-T remains historical validation evidence and a development fallback, not the terminal macOS architecture. + +> Architecture update, 2026-07-18: the terminal macOS route is the Apple-native Swift FSKit extension -> versioned binary UDS -> Go daemon. Earlier FUSE-T task evidence remains useful regression history but does not authorize reverting the product architecture or claiming native FSKit readiness. ## Alignment Snapshot diff --git a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md index e25a8da..89f2639 100644 --- a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md +++ b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md @@ -270,13 +270,13 @@ The trace suite covers listing, opening, scrolling old history, resume, sending ### macOS -- Selected production adapter: FUSE-T `1.2.7` with the NFS backend explicitly requested as `backend=nfs`; relying on FUSE-T's default backend is not allowed. -- Service: user launch service with keep-alive and mount health monitoring. +- Selected production candidate: an Apple-native Swift FSKit extension using versioned binary Unix-domain-socket IPC to the Go CodexFold daemon. +- Service: two user launch services, one for the Host/Go daemon chain and one for mount supervision, with child-process lock ownership, build identity, mount health, and atomic app/binary rollback checks. - Required tests: APFS native baseline, Apple Silicon, Codex Desktop, Codex CLI, canonical `sessions` and `archived_sessions` namespace moves, sleep/wake, network changes, user logout/login, daemon kill, mount restart, and Codex upgrade. -- FUSE-T is the validated userspace host for this project; macFUSE is not a prerequisite for the current macOS route. -- The FUSE-T NFS mount must use synchronous write requests before its health identity becomes readable. This is verified through the live `MNT_SYNCHRONOUS` mount flag and a real same-offset JSONL write regression; libfuse `direct_io` or disabled attribute caching alone is not accepted as evidence. +- The Swift extension exposes regular JSONL paths and filesystem metadata while the Go daemon owns packed reads, append delta, copy-on-write backing, generation recovery, and canonical routing. The production implementation may not depend on NFS or a third-party FUSE compatibility layer. +- FUSE-T `1.2.7` with synchronous NFS remains historical canary evidence and a development-only fallback. It is not the terminal macOS architecture and cannot satisfy native FSKit production readiness on its own. - FUSE-T `1.2.7`'s FSKit backend is rejected for production. An isolated real mount lost the first of two complete JSONL records written at the same stale EOF, and managed-to-native route changes remained cached past the five-second correctness gate. Basic read/write, `F_FULLFSYNC`, truncate, remount, and throughput results do not override a byte-loss failure. FUSE-T also documents that notifications are unavailable for its FSKit backend. -- Native FSKit remains a research adapter rather than a fallback selected at runtime. The earlier probes did not provide a complete canonical namespace and did not recover automatically from every extension-process failure; no native FSKit route may replace the selected NFS backend without passing the complete platform contract independently. +- The Apple-native FSKit implementation is distinct from FUSE-T's rejected FSKit backend. It remains `fs-engine-preview` until its complete performance, cache coherency, real-client, crash, upgrade, rollback, retention, and power-loss gates pass independently. - Platform readiness requires a directory-level canonical namespace or an equivalent mechanism that keeps Codex archive and unarchive moves native-compatible. ### Linux @@ -301,7 +301,7 @@ Platform readiness is independent. Passing macOS gates does not imply Linux or W The platform-neutral core defines byte layout and transaction behavior, not a lowest-common-denominator filesystem API. Each adapter must implement the strongest native semantics Codex uses on that platform; macOS behavior may not be weakened to match Windows or Linux limitations. -FUSE-T, FUSE3, and WinFsp are current candidates rather than product promises. If native-operation traces or platform gates disqualify a candidate, it must be replaced without weakening `TF-001` through `TF-022`. +Apple-native FSKit, FUSE3, and WinFsp are current candidates rather than product promises. If native-operation traces or platform gates disqualify a candidate, it must be replaced without weakening `TF-001` through `TF-022`. ## Migration And Rollback diff --git a/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md b/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md index be96ee1..610c756 100644 --- a/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md +++ b/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md @@ -6,7 +6,7 @@ This document aligns the original product commitments, the canonical transparent It does not redesign CodexFold, authorize real-session enrollment, promote the capability above `fs-engine-preview`, or treat fixtures as production evidence. CodexFold remains a standalone public product. Native launchd, `systemd --user`, and Windows SCM supervision are part of the standalone runtime; private or external control-plane coupling remains outside it. -Baseline refreshed against the current Task 12 through Task 15 implementation and validation evidence on 2026-07-16. +Baseline refreshed against the native FSKit development checkpoint and current Task 12 through Task 15 implementation on 2026-07-18. ## Original Commitment To Requirement Mapping @@ -36,22 +36,22 @@ Baseline refreshed against the current Task 12 through Task 15 implementation an | Requirement | Current implementation | Tests or evidence | Status | | --- | --- | --- | --- | | `TF-001` | `internal/cli/fs.go`, `internal/mountfs`, canonical migration, automatic enrollment, and routing | Isolated CLI/Desktop direct-open, resume, and automatic-enrollment canaries | Partial only at release level: implemented and verified on macOS canaries; real-home automatic apply remains preview-gated | -| `TF-002` | `internal/mountfs`, `internal/sessionns`, `internal/mountid` | Real FUSE-T operation tests and isolated unmodified clients | Implemented for the validated macOS client versions | +| `TF-002` | `internal/mountfs`, `internal/fskitproto`, the Swift FSKit extension, canonical routing, and mount identity | Native FSKit mounted behavior tests plus historical FUSE-T unmodified-client evidence | Partial: native FSKit behavior coverage passes in isolation; full real Codex CLI/Desktop acceptance on the native route remains open | | `TF-003` | Neutral operation layer plus exact compatibility contracts in `internal/compat` | Real macOS traces and adapter canaries plus real Linux FUSE3 operations | Partial: current installed macOS clients and Linux adapter operations are covered; real Linux Codex clients and Windows are not validated | | `TF-004` | `internal/scan`, `internal/cdc`, `internal/fold`, `internal/pack` | Repeated field, record, CDC, fork, and non-prefix corpus tests | Implemented | | `TF-005` | `internal/vfs` append delta and writer leases | Append-without-hydration tests and real CLI/Desktop append evidence | Implemented | -| `TF-006` | `internal/vfs` copy-on-write backing and neutral write operations | Random-write, truncate, interruption, and real FUSE-T mutation tests | Implemented | +| `TF-006` | `internal/vfs` copy-on-write backing and neutral write operations | Random-write, truncate, interruption, native FSKit mounted mutation, and historical FUSE-T tests | Implemented | | `TF-007` | Immutable packs, in-memory index, bounded cache, random-read resolver | Pack round-trip/corruption tests and 758 MiB packed-read benchmark | Implemented | -| `TF-008` | `internal/fsctl` benchmark and `internal/testfs` stress harness | `docs/validation-fs-preview.md`, synchronous FUSE-T measurements, and Linux FUSE3 race performance | Partial: shared-core, measured macOS, and Linux adapter safety floors pass; cold/full-distribution and Windows metrics remain open | +| `TF-008` | Packed-read benchmark, mounted performance tests, and `internal/testfs` stress harness | 758 MiB core benchmark, native FSKit cold/warm measurements, historical synchronous FUSE-T measurements, and Linux FUSE3 race performance | Partial: the shared core passes; native FSKit read-ahead performance/coherency and Windows metrics remain open | | `TF-009` | Journal recovery, generation recovery, service keep-alive, restart-safe retirement | Recovery tests, daemon restart canaries, managed Deep Idle sleep/wake, and actual retained-source host reboot | Partial: no actual power loss during an in-flight transaction | | `TF-010` | Shadow compare, optimistic routes, retained snapshots, current-byte fallback | 90,000 real random-range comparisons, rollback and failure-containment canaries, and one bounded retained-source user-home canary | Implemented for macOS canaries; retention remains open | | `TF-011` | `internal/enroll`, `fs enroll`, and the bounded standalone-service loop discover existing, new, and forked sessions, persist stability observations, take a fail-closed native-writer snapshot, and reuse fold/pack/migrate transactions | Policy tests plus a real writable-descriptor probe and isolated canonical FUSE enrollment, daemon restart, real CLI append, quarantine, and failed-cutover evidence | Implemented; real-home automatic apply remains disabled until platform promotion | -| `TF-012` | Shared Go core, explicitly selected macOS FUSE-T NFS, Linux FUSE3, and Windows WinFsp adapters | Real macOS and Linux adapter tests; rejected FUSE-T FSKit correctness canary; default and WinFsp Windows cross-compiles | Partial: Windows has implementation and compile evidence only | +| `TF-012` | Shared Go core, Apple-native Swift FSKit on macOS, Linux FUSE3, and Windows WinFsp adapters | Native FSKit mounted behavior/crash/rollback tests, historical FUSE-T evidence, real Linux adapter tests, and Windows cross-compiles | Partial: native FSKit real-client acceptance and all real Windows gates remain open | | `TF-013` | Canonical capability type in `internal/fsctl/status.go` | Status rejection tests and CLI status tests | Implemented; current status is `fs-engine-preview` | | `TF-014` | Snapshot retention and destructive-action guards | Migration, rollback, and quarantine tests | Implemented as a safety rule; retention promotion gates remain open | | `TF-015` | Exact-version compatibility and update preflight quarantine | Unknown-version fallback and isolated canary tests | Implemented; the currently installed macOS CLI and Desktop are covered | -| `TF-016` | Tagged adapter prerequisite errors and non-elevating native launchd, `systemd --user`, and Windows SCM lifecycle | Stub, authorization-gated FUSE-T, real Linux service, and Windows cross-compile evidence | Implemented; Windows runtime execution remains unverified | -| `TF-017` | Canonical namespace, write-sealed backing, mount identity, platform mount policy, route normalization, process lock | Neutral, real FUSE-T, real Linux FUSE3, launchd/systemd restart, stale-offset write, crash recovery, and rollback tests | Implemented for macOS canaries and Linux adapter gates; Windows remains unverified | +| `TF-016` | Native FSKit Host/extension packaging, non-elevating launchd, `systemd --user`, and Windows SCM lifecycle | Signed native FSKit app installation/rollback, launchd crash matrix, real Linux service, and Windows cross-compile evidence | Implemented; native release packaging and Windows runtime execution remain unverified | +| `TF-017` | Canonical namespace, write-sealed backing, mount identity, route normalization, daemon/supervisor locks, and build identity | Native FSKit mounted namespace tests, Host/child crash matrix, app/binary rollback, real Linux FUSE3, and historical FUSE-T evidence | Partial: isolated macOS and Linux gates pass; native real-client, retention, and Windows gates remain open | | `TF-018` | `internal/codex` spawn edges, `internal/family` graph/content evidence, `internal/archive` guarded transactions, and public `fork-family` plus `archive` commands | Diverse relationship fixtures, repeated-record performance regression, source-change rejection, official archive trace, native apply/recovery, and isolated managed FUSE-T archive/unarchive plus daemon restart | Implemented | | `TF-019` | `internal/contain` and `internal/prune`; public `contains` and `remove-contained` commands | Exact containment, archived-only apply, transaction rollback, and recovery-manifest tests | Implemented | | `TF-020` | Exact fold/migrate paths are byte-preserving; `repair-rollout` and `reconcile-rollout` write separate explicit outputs; a static production-import boundary prevents other workflows from invoking reconciliation | `internal/reconcile`, CLI behavior, and AST boundary tests | Implemented | @@ -72,7 +72,7 @@ Baseline refreshed against the current Task 12 through Task 15 implementation an | Task 8: standalone CLI and automatic enrollment | Complete | Command surface, guarded lifecycle, bounded planner/apply loop, native service arguments, and isolated real FUSE enrollment evidence | Production enablement remains outside Task 8 | | Task 9: service lifecycle and update guard | Complete | Commit `4589ffa`; launchd, real `systemd --user`, Windows SCM compile, and preflight tests pass | Windows runtime and stronger automatic update claims remain release-gated | | Task 10: synthetic, crash, performance, and compile gates | Complete for the shared engine | Commit `a1ac76e`; preview validation report | It cannot satisfy real-adapter or retention gates | -| Task 11: real macOS trace, adapter, shadow, and canary | Partial | Sanitized real CLI/Desktop/FUSE-T evidence, managed sleep/wake, retained-source host reboot, current-client contracts, canonical user-home activation, and synchronous isolated plus bounded user-home real CLI canaries are public | Dedicated user-home canary retention, actual in-flight power loss, and seven-day retention remain | +| Task 11: real macOS trace, adapter, shadow, and canary | Partial | Historical real CLI/Desktop/FUSE-T evidence plus native FSKit mounted behavior, Host/child crash recovery, and atomic update rollback | Native FSKit read-ahead coherency/performance, complete real CLI/Desktop resume/fork/archive/restart acceptance, actual in-flight power loss, and retention remain | | Task 12: bounded automatic discovery and enrollment | Complete | Planner/apply/service tests plus isolated canonical FUSE enrollment, daemon restart, real CLI append, quarantine, and failed-cutover evidence | Real-home automatic apply remains platform-gated | | Task 13: conservative branch lifecycle and content-change boundary | Complete | Spawn-edge family reports, exact relationship comparison, official-compatible guarded archive and recovery, separate exact-contained deletion, and static content-change boundaries pass unit, race, native, and managed FUSE-T validation | None in this task | | Task 14: hard storage budgets, retention, cleanup, and accounting | Complete | Platform-neutral inventory, hard preflight, lease-aware bounded GC, truthful accounting, low-space and repeated-GC tests | Destructive retention remains platform-gated | @@ -82,7 +82,7 @@ Baseline refreshed against the current Task 12 through Task 15 implementation an | Missing behavior | Exact implementation work | Required verification | | --- | --- | --- | -| Remaining macOS disruptive and retention gates | Keep the dedicated retained-source user-home canary bounded to one explicitly selected session; perform an actual in-flight power-loss test only in a disposable host or VM; then complete the incident-free retention window | Clean doctor, exact SHA after restart and recovery cases, rollback, no route loss, and seven incident-free days | +| Remaining macOS native FSKit gates | Complete read-ahead coherency/performance, run isolated real CLI/Desktop resume/fork/archive/restart acceptance, then perform power-loss and retention canaries only on disposable or explicitly approved data | Exact full-file and random-range SHA, bounded latency/RSS, clean crash recovery, exact rollback, no route loss, and the required incident-free window | | Linux remaining readiness | Keep the implemented FUSE3 adapter and systemd lifecycle behind platform gates | Real Linux Codex traces, client upgrade quarantine, rollback, and retention | | Windows readiness | Execute the implemented WinFsp adapter and SCM host without moving shared behavior out of the core | Native operations, crash/restart, performance, real Codex traces, upgrade quarantine, rollback, and retention on a real Windows host | diff --git a/docs/validation-macos-canary.md b/docs/validation-macos-canary.md index 11e51e9..1280124 100644 --- a/docs/validation-macos-canary.md +++ b/docs/validation-macos-canary.md @@ -1,6 +1,14 @@ # macOS Adapter And Canary Validation -## Current Status +## Native FSKit Development Checkpoint + +As of 2026-07-18, the terminal macOS candidate is the Apple-native Swift FSKit extension connected over versioned binary UDS IPC to the Go daemon. Isolated mounted metadata, xattr, append, random-write, truncate, namespace, archive, mmap, open-unlink, and external namespace tests pass. Host/Go-child and supervisor crash recovery, build-identity checks, environment sanitization, and atomic app/definition/binary rollback also pass. A 758 MiB packed-core benchmark remains well above the filesystem target with bounded RSS. + +The current open gate is the native adapter path: cold reads exposed excessive 8 KiB FSKit/UDS round trips, and a bounded 1 MiB read-ahead implementation has been compiled but still requires installed-candidate performance and cache-coherency acceptance. Full isolated real Codex CLI/Desktop resume, fork, archive, restart, and exact JSONL verification must follow. Production activation and real-home migration remain disabled. + +The FUSE-T evidence below is retained as historical compatibility and regression evidence. It is not the terminal architecture and must not be used to claim Apple-native FSKit readiness. + +## Historical FUSE-T Status The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have passed for read, append, resume, fork, child-session enrollment, canonical archive/unarchive moves, launchd restart, rollback, namespace deactivation, and unknown-version quarantine. A retained-source CLI canary survived an actual host reboot while managed, then resumed through the recovered mount and rolled back to an exact native JSONL. The currently installed CLI and Desktop versions also passed exact compatibility and isolated retained-source canaries. Process-level interruption recovery now covers append, compaction, migration, and rollback, and a managed session passed an actual Deep Idle sleep/wake cycle followed by a real model turn. The user Codex home now uses the canonical namespace with ordinary sessions remaining native passthrough and one explicitly selected retained-source canary managed for observation. The project remains at `fs-engine-preview` because that canary has not completed retention, in-flight transaction evidence does not claim an actual power-loss test, and the seven-day incident-free gate has not completed. From 1120bf21fcee99b0615372f1e39da36c7a90a8cc Mon Sep 17 00:00:00 2001 From: King Star Date: Sat, 18 Jul 2026 19:33:20 +0800 Subject: [PATCH 33/33] ci: align platform gates with validated support --- .github/workflows/ci.yml | 15 +++++++++- CONTRIBUTING.md | 2 ++ docs/maintainer-guide.md | 2 ++ internal/mountfs/native_fskit_server_test.go | 29 ++++++++++++-------- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cef764..62e8670 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -53,6 +53,19 @@ jobs: - run: go test ./... -count=1 - run: go build ./cmd/codexfold + windows-compile: + name: windows-compile + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - name: Compile all test packages without claiming runtime validation + run: go test ./... -run '^$' -count=1 + - run: go build ./cmd/codexfold + race: name: race runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f520e01..1b17113 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,6 +27,8 @@ git diff --check For a focused change, run the smallest relevant package tests first, then the common gate before requesting review. +Windows currently has compile and cross-build gates only. Do not describe those checks as real WinFsp or Windows Service runtime validation. + ## Branches and Commits - Create a branch from current `main`; use a descriptive prefix such as `feat/`, `fix/`, `docs/`, or `test/`. diff --git a/docs/maintainer-guide.md b/docs/maintainer-guide.md index 1169e8e..cf898d3 100644 --- a/docs/maintainer-guide.md +++ b/docs/maintainer-guide.md @@ -36,6 +36,8 @@ git diff --check Native macOS FSKit changes additionally require an XcodeGen consistency check and Release build with the Xcode version declared by the project. Commit `project.yml` and the regenerated project together. +Windows CI compiles every package and test binary but does not execute the full runtime suite. That is intentional until a real Windows/WinFsp host validates directory durability, locking, service, mount, and file-sharing semantics. A green Windows compile check is not Windows readiness evidence. + ## Isolated Native FSKit Validation Never point development tests at `~/.codex`. Create a disposable Codex home, store, native root, mount point, service label, and service definition. Production `com.codexfold.fs` must remain disabled during preview work. diff --git a/internal/mountfs/native_fskit_server_test.go b/internal/mountfs/native_fskit_server_test.go index 2630ad2..5466eae 100644 --- a/internal/mountfs/native_fskit_server_test.go +++ b/internal/mountfs/native_fskit_server_test.go @@ -6,6 +6,7 @@ import ( "errors" "os" "path/filepath" + "runtime" "strings" "syscall" "testing" @@ -209,11 +210,7 @@ func TestNativeFSKitServerExposesReadOnlyMountIdentity(t *testing.T) { } func TestNativeFSKitServerPublishesDirectoryResourceWithScopedSocket(t *testing.T) { - root, err := os.MkdirTemp("/private/tmp", "cfs-r-") - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.RemoveAll(root) }) + root := shortNativeFSKitTestDir(t, "cfs-r-") resource := filepath.Join(root, "native-fskit") filesystem := NewCanonical() filesystem.SetNativeRoot(filepath.Join(root, "native")) @@ -237,7 +234,7 @@ func TestNativeFSKitServerPublishesDirectoryResourceWithScopedSocket(t *testing. }() deadline := time.Now().Add(5 * time.Second) var client *fskitproto.Client - err = nil + var err error for time.Now().Before(deadline) { client, err = fskitproto.DialResource(resource, 100*time.Millisecond) if err == nil { @@ -346,10 +343,7 @@ func TestNativeFSKitServerNormalizesFSKitWholeFileSnapshotsIntoJSONLAppends(t *t func startNativeFSKitTestServer(t *testing.T, filesystem *Filesystem, root string) (*fskitproto.Client, func()) { t.Helper() - socketRoot, err := os.MkdirTemp("/private/tmp", "cfs-") - if err != nil { - t.Fatal(err) - } + socketRoot := shortNativeFSKitTestDir(t, "cfs-") ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) options := NativeFSKitServerOptions{ @@ -379,7 +373,6 @@ func startNativeFSKitTestServer(t *testing.T, filesystem *Filesystem, root strin } stop := func() { cancel() - defer os.RemoveAll(socketRoot) select { case err := <-done: if err != nil && !errors.Is(err, context.Canceled) { @@ -392,6 +385,20 @@ func startNativeFSKitTestServer(t *testing.T, filesystem *Filesystem, root strin return client, stop } +func shortNativeFSKitTestDir(t *testing.T, pattern string) string { + t.Helper() + base := os.TempDir() + if runtime.GOOS == "darwin" || runtime.GOOS == "linux" { + base = "/tmp" + } + root, err := os.MkdirTemp(base, pattern) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(root) }) + return root +} + func writeNativeFSKitTestPayload(t *testing.T, client *fskitproto.Client, handle uint64, offset int64, data []byte) { t.Helper() encoder := fskitproto.NewEncoder(20 + len(data))