From 8412c4d131bf023dcab177251f29a5324a26797b Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:23:59 -0500 Subject: [PATCH 01/34] fix(daemon/remote): publish bundle extractions by swap, not destroy-then-rename extractBundle removed the live extraction and only then renamed the new clone into place, so anything that went wrong in between left the link holding neither tree: the deferred staging cleanup deleted the replacement on the way out. The doc comment claimed the opposite, that staging plus rename kept dest intact on error, which is true only of a clone failure. Two reachable ways to hit it, both reproduced. A removal that fails partway (a subdirectory the daemon cannot delete) reports an error with the prior tree already gutted. And every upload is handled in its own goroutine, so two uploads of one link id interleaved their removal and rename, failing with "directory not empty" and letting one call's removal wipe a tree another had just published. Move the live tree aside into staging instead of deleting it, rename the clone into place, and put the old tree back if that fails. If the restore fails too, keep staging so the only remaining copy survives and name it in the error. A refcounted per-destination lock serializes extracts. Swapping a directory is two renames and cannot be made atomic, so the comment now says what the code actually guarantees: on every error return dest holds one of the two trees, but a crash between the renames leaves it in staging with nothing to reap it on restart. The clone's deadline now starts once the lock is held, and bundle verify gets its own. Sharing one gitTimeout meant an upload queued behind a slow clone spent its budget waiting and then failed on the clone. A staging cleanup that fails is logged rather than dropped, since staging now holds a whole copy of the prior tree. --- internal/daemon/remote/bundle.go | 114 ++++++++++++-- internal/daemon/remote/bundle_test.go | 208 ++++++++++++++++++++++++++ 2 files changed, 309 insertions(+), 13 deletions(-) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index 398c1bcb2..c1f82fb08 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -13,6 +13,7 @@ import ( "os/exec" "path/filepath" "strings" + "sync" "time" "github.com/Gitlawb/zero/internal/daemon" @@ -129,16 +130,18 @@ func (b *Bridge) receiveBundle(conn net.Conn) bundleResult { return bundleResult{Message: "stage bundle: " + err.Error()} } - ctx, cancel := context.WithTimeout(context.Background(), gitTimeout) - defer cancel() - if err := gitBundleVerify(ctx, tmpName); err != nil { + verifyCtx, cancelVerify := context.WithTimeout(context.Background(), gitTimeout) + defer cancelVerify() + if err := gitBundleVerify(verifyCtx, tmpName); err != nil { return bundleResult{Message: "bundle verify: " + err.Error()} } dest := filepath.Join(b.bundleDir, id) if !withinDir(b.bundleDir, dest) { return bundleResult{Message: "invalid link id"} } - if err := extractBundle(ctx, tmpName, dest); err != nil { + // extractBundle starts the clone's own gitTimeout once it holds the lock for + // dest, so an upload queued behind another does not spend that budget waiting. + if err := extractBundle(context.Background(), tmpName, dest, b.logf); err != nil { return bundleResult{Message: "extract bundle: " + err.Error()} } return bundleResult{OK: true, Path: dest} @@ -168,27 +171,112 @@ func streamFramesToFile(r io.Reader, w io.Writer, size int64) error { return nil } -// extractBundle clones bundleFile into a staging dir, then atomically renames it -// over dest (replacing any prior extraction for this link id). git clone needs a -// non-existent target, so the staging+rename keeps the live dest intact on error. -func extractBundle(ctx context.Context, bundleFile, dest string) error { +// stagingPrefix names the per-extract staging directories created beside dest. +const stagingPrefix = ".staging-" + +// renameDir moves a directory into its published location. It is a var so tests +// can force a failure at the steps whose errors would otherwise be unrecoverable. +var renameDir = os.Rename + +// extractLocks serializes extracts per destination. Each bundle upload runs in +// its own connection goroutine, so two uploads of one link id would otherwise +// interleave their swap steps and clobber each other. +var extractLocks = struct { + mu sync.Mutex + locks map[string]*extractLock +}{locks: map[string]*extractLock{}} + +type extractLock struct { + mu sync.Mutex + refs int +} + +// lockExtract blocks until dest is free and returns its release func. Entries +// are refcounted so the map cannot grow with every link id ever uploaded. +func lockExtract(dest string) func() { + extractLocks.mu.Lock() + entry := extractLocks.locks[dest] + if entry == nil { + entry = &extractLock{} + extractLocks.locks[dest] = entry + } + entry.refs++ + extractLocks.mu.Unlock() + + entry.mu.Lock() + return func() { + entry.mu.Unlock() + extractLocks.mu.Lock() + entry.refs-- + if entry.refs == 0 { + delete(extractLocks.locks, dest) + } + extractLocks.mu.Unlock() + } +} + +// extractBundle clones bundleFile into a staging dir beside dest, then swaps the +// clone into place (replacing any prior extraction for this link id). git clone +// needs a non-existent target, hence the staging dir. The live tree is moved +// aside rather than deleted and is put back if the publish fails, so on every +// error return dest holds either the prior extraction or the new one, never +// neither. Swapping a directory is two renames and cannot be made atomic, so a +// crash between them leaves dest absent with the prior tree in staging/backup; +// nothing reaps that on restart. logf may be nil. +func extractBundle(ctx context.Context, bundleFile, dest string, logf func(string, ...any)) error { + if logf == nil { + logf = func(string, ...any) {} + } parent := filepath.Dir(dest) if err := os.MkdirAll(parent, 0o700); err != nil { return err } - staging, err := os.MkdirTemp(parent, ".staging-*") + unlock := lockExtract(dest) + defer unlock() + + staging, err := os.MkdirTemp(parent, stagingPrefix+"*") if err != nil { return err } - defer func() { _ = os.RemoveAll(staging) }() + // Staging also holds the prior tree while the swap is in flight, so it is + // only cleaned up once dest is known to hold one of the two trees. + cleanupStaging := true + defer func() { + if !cleanupStaging { + return + } + // A failure here strands a whole copy of the prior tree under a + // dot-prefixed dir nothing else enumerates, so say so rather than + // leaking it silently. + if err := os.RemoveAll(staging); err != nil { + logf("remote: could not remove bundle staging dir %s: %v", staging, err) + } + }() + cloneCtx, cancelClone := context.WithTimeout(ctx, gitTimeout) + defer cancelClone() cloneDest := filepath.Join(staging, "repo") - if err := gitClone(ctx, bundleFile, cloneDest); err != nil { + if err := gitClone(cloneCtx, bundleFile, cloneDest); err != nil { return err } - if err := os.RemoveAll(dest); err != nil { + + // Every rename stays inside parent, so none of them crosses a filesystem. + backup := filepath.Join(staging, "backup") + restore := func() error { return nil } + if err := os.Rename(dest, backup); err == nil { + restore = func() error { return renameDir(backup, dest) } + } else if !os.IsNotExist(err) { return err } - return os.Rename(cloneDest, dest) + if err := renameDir(cloneDest, dest); err != nil { + if restoreErr := restore(); restoreErr != nil { + // dest is empty and the only copy of the prior tree is the backup, + // so keep staging rather than deleting the tree on the way out. + cleanupStaging = false + return fmt.Errorf("publish extraction: %w (prior tree left in %s: %v)", err, backup, restoreErr) + } + return err + } + return nil } // ---- client side ----------------------------------------------------------- diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index 7f711a3bd..f462c7ac7 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -2,11 +2,17 @@ package remote import ( "context" + "errors" + "fmt" "os" "os/exec" "path/filepath" "runtime" + "slices" + "strings" + "sync" "testing" + "time" ) // initTestRepo creates a temp git work tree with one committed file and returns @@ -191,3 +197,205 @@ func TestBridgeBundleRejectsBadToken(t *testing.T) { t.Fatal("bundle upload with a bad token must be refused") } } + +// testBundle commits content to a throwaway repo and returns a bundle of it. +func testBundle(t *testing.T, file, content string) string { + t.Helper() + repo := initTestRepo(t, file, content) + out := filepath.Join(t.TempDir(), "b.bundle") + ctx, cancel := context.WithTimeout(context.Background(), gitTimeout) + defer cancel() + if err := gitBundleCreate(ctx, repo, out); err != nil { + t.Fatalf("bundle create: %v", err) + } + return out +} + +// An extract that cannot clear the live tree must leave that tree alone. The old +// code removed dest before it had anything to publish, so a partial removal -- +// here a subdirectory the daemon cannot delete -- destroyed the prior extraction +// and the deferred staging cleanup then deleted the replacement. +func TestExtractBundleKeepsPriorTreeWhenLiveTreeCannotBeCleared(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root ignores the directory permissions this test relies on") + } + if runtime.GOOS == "windows" { + t.Skip("POSIX directory permissions") + } + ctx := context.Background() + dest := filepath.Join(t.TempDir(), "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + locked := filepath.Join(dest, "locked") + if err := os.MkdirAll(locked, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(locked, "keep.txt"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chmod(locked, 0o500); err != nil { + t.Fatal(err) + } + // The publish moves the old tree (locked subdir included) into staging, so + // restore write permission wherever it ended up or TempDir cleanup fails. + t.Cleanup(func() { + _ = filepath.WalkDir(filepath.Dir(dest), func(path string, d os.DirEntry, err error) error { + if err == nil && d.IsDir() { + _ = os.Chmod(path, 0o700) + } + return nil + }) + }) + + var logged []string + logf := func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) } + if err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, logf); err != nil { + t.Fatalf("extract over an undeletable subtree: %v", err) + } + // The prior tree moved into staging, so the cleanup cannot delete it either. + // That strands a whole copy of the repo and must not pass silently. + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "staging dir") }) { + t.Errorf("a staging dir that could not be removed was not reported: %v", logged) + } + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil { + t.Fatalf("dest holds neither the old tree nor the new one: %v", err) + } + if string(got) != "v1" { + t.Fatalf("a.txt = %q, want the newly published %q", got, "v1") + } +} + +// Every bundle upload is handled in its own goroutine, so two uploads of one +// link id can extract at the same time. The old code let them delete and rename +// over each other: extracts failed with "directory not empty" and one call's +// removal could wipe a tree another had already published. +func TestExtractBundleConcurrentSameDestAlwaysLeavesATree(t *testing.T) { + ctx := context.Background() + dest := filepath.Join(t.TempDir(), "proj-1") + first := testBundle(t, "a.txt", "v0") + second := testBundle(t, "a.txt", "v1") + if err := extractBundle(ctx, first, dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + // The unsafe window is the two back-to-back renames after the clone. Without + // this delay it is narrow enough that the test still passes a good fraction + // of the time with the lock removed, which would make it a guard in name only. + real := renameDir + renameDir = func(from, to string) error { + time.Sleep(2 * time.Millisecond) + return real(from, to) + } + t.Cleanup(func() { renameDir = real }) + + var wg sync.WaitGroup + var mu sync.Mutex + var failures []error + for round := 0; round < 15; round++ { + for _, src := range []string{first, second} { + wg.Add(1) + go func(src string) { + defer wg.Done() + if err := extractBundle(ctx, src, dest, nil); err != nil { + mu.Lock() + failures = append(failures, err) + mu.Unlock() + } + }(src) + } + wg.Wait() + } + + if len(failures) > 0 { + t.Errorf("concurrent extracts of one link id failed: %v", failures) + } + if _, err := os.Stat(filepath.Join(dest, ".git")); err != nil { + t.Errorf("dest is not a work tree after concurrent extracts: %v", err) + } + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil { + t.Fatalf("dest holds no extraction: %v", err) + } + if string(got) != "v0" && string(got) != "v1" { + t.Fatalf("a.txt = %q, want one of the uploaded trees", got) + } +} + +// The publish rename is the one step whose failure the old code could not come +// back from: dest was already deleted. It must now put the prior tree back. +func TestExtractBundleRestoresPriorTreeWhenPublishFails(t *testing.T) { + ctx := context.Background() + dest := filepath.Join(t.TempDir(), "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + // Fail only the publish, so the restore rename that follows it still runs. + real := renameDir + calls := 0 + renameDir = func(from, to string) error { + calls++ + if calls == 1 { + return errors.New("injected publish failure") + } + return real(from, to) + } + t.Cleanup(func() { renameDir = real }) + + if err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, nil); err == nil { + t.Fatal("a failed publish must be reported") + } + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil { + t.Fatalf("prior extraction was not restored: %v", err) + } + if string(got) != "v0" { + t.Fatalf("a.txt = %q, want the prior %q", got, "v0") + } +} + +// If the publish fails and the prior tree cannot be put back either, dest is +// empty and the backup is the only copy left. It must survive the cleanup so an +// operator can recover it by hand. +func TestExtractBundleKeepsBackupWhenRestoreAlsoFails(t *testing.T) { + ctx := context.Background() + bundleDir := t.TempDir() + dest := filepath.Join(bundleDir, "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + real := renameDir + renameDir = func(string, string) error { return errors.New("injected rename failure") } + t.Cleanup(func() { renameDir = real }) + + err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, nil) + if err == nil { + t.Fatal("a failed publish must be reported") + } + if !strings.Contains(err.Error(), "prior tree left in") { + t.Errorf("error should point at the retained backup, got: %v", err) + } + + entries, readErr := os.ReadDir(bundleDir) + if readErr != nil { + t.Fatal(readErr) + } + found := "" + for _, e := range entries { + if strings.HasPrefix(e.Name(), stagingPrefix) { + if _, statErr := os.Stat(filepath.Join(bundleDir, e.Name(), "backup", "a.txt")); statErr == nil { + found = e.Name() + } + } + } + if found == "" { + t.Fatal("the prior tree was deleted along with the staging dir") + } + got, readErr := os.ReadFile(filepath.Join(bundleDir, found, "backup", "a.txt")) + if readErr != nil || string(got) != "v0" { + t.Fatalf("retained backup = %q, err %v, want %q", got, readErr, "v0") + } +} From efa6adc63ade565aeead3eebfe23661a39457d8e Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:24:03 -0500 Subject: [PATCH 02/34] fix(daemon/remote): refuse link ids that can name a staging directory Extracts stage into .staging-* directories created beside dest, in the bundle dir itself, but sanitizeLinkID accepted .staging-123, .git and ..foo. Link ids come from the client's --id flag and travel over the wire, so an id could name another extract's in-flight staging dir, whose removal then deletes that clone mid-flight. Refuse a leading '.' outright rather than only the two traversal names. That keeps the staging namespace out of reach by construction and drops the hidden-directory ids along with it. The check runs on the upload path too, so a bad id fails before the client dials. This rejects ids that used to be accepted. Nothing documents the charset and a dot-prefixed id was never useful, but an existing link named that way stops working and needs renaming. --- internal/daemon/remote/bundle.go | 10 ++++++---- internal/daemon/remote/bundle_test.go | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index c1f82fb08..0be8a5d8a 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -172,6 +172,7 @@ func streamFramesToFile(r io.Reader, w io.Writer, size int64) error { } // stagingPrefix names the per-extract staging directories created beside dest. +// sanitizeLinkID refuses every dot-prefixed id so a link can never name one. const stagingPrefix = ".staging-" // renameDir moves a directory into its published location. It is a var so tests @@ -411,8 +412,9 @@ func runGit(ctx context.Context, dir string, args ...string) error { } // sanitizeLinkID validates a link id used as a single path component under the -// bundle dir. It allows letters, digits, '-', '_', '.', forbids the traversal -// names, and caps the length — so it can never escape the bundle dir. +// bundle dir. It allows letters, digits, '-', '_', '.', forbids a leading '.', +// and caps the length, so an id can never escape the bundle dir and can never +// name one of the stagingPrefix directories an extract creates beside it. func sanitizeLinkID(id string) (string, error) { id = strings.TrimSpace(id) if id == "" { @@ -421,8 +423,8 @@ func sanitizeLinkID(id string) (string, error) { if len(id) > 128 { return "", errors.New("remote: link id too long (max 128)") } - if id == "." || id == ".." { - return "", errors.New("remote: invalid link id") + if strings.HasPrefix(id, ".") { + return "", errors.New("remote: link id may not start with '.'") } for _, r := range id { switch { diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index f462c7ac7..2679464b5 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -323,6 +323,21 @@ func TestExtractBundleConcurrentSameDestAlwaysLeavesATree(t *testing.T) { } } +// Staging directories live beside dest under a reserved dot prefix, so a link id +// must never be able to name one. +func TestSanitizeLinkIDRejectsDotPrefixedIDs(t *testing.T) { + for _, id := range []string{".", "..", ".staging-1", ".staging-abc", ".git", "..foo", ".hidden"} { + if _, err := sanitizeLinkID(id); err == nil { + t.Errorf("sanitizeLinkID(%q) was accepted; it can collide with a staging dir", id) + } + } + for _, id := range []string{"proj-1", "a.b", "x_1", "A1", "repo.git"} { + if _, err := sanitizeLinkID(id); err != nil { + t.Errorf("sanitizeLinkID(%q) = %v, want accepted", id, err) + } + } +} + // The publish rename is the one step whose failure the old code could not come // back from: dest was already deleted. It must now put the prior tree back. func TestExtractBundleRestoresPriorTreeWhenPublishFails(t *testing.T) { From cce928e0f84cab9cb0e01053cccfce9e85abb3c0 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:50:13 -0500 Subject: [PATCH 03/34] fix(daemon/remote): survive an interrupted extract and a second daemon Two holes were left after the swap fix. Swapping a directory is two renames, so a crash between them leaves the link's only tree sitting in a staging dir with nothing to put it back. And the lock that serializes extracts is in-process, so a second daemon pointed at the same --bundle-dir does not see it. Take a per-link advisory file lock (lockutil, the same kernel-held locks cron and swarm use) alongside the in-process one, under the lock dir .extract-locks, which link ids cannot name. The wait is bounded and respects the caller's context. Record the link id in the staging dir before moving its tree, then have NewBridge repair the dir once before it serves. A backup whose link has no live tree is put back, one whose link already has a tree is dropped, and a staging dir with no backup is only reaped once it is older than any clone could be, so a running extract is never swept out from under itself. A marker that does not name a valid link inside the bundle dir is refused and the tree left where it is, so a corrupt marker cannot steer a rename. Verified end to end against a real bridge over TLS: a daemon started on a bundle dir left mid-swap restores the link and logs it, where the previous build leaves it gone for good. --- internal/daemon/remote/bridge.go | 12 +- internal/daemon/remote/bundle.go | 126 ++++++++++++ internal/daemon/remote/bundle_test.go | 283 ++++++++++++++++++++++++++ 3 files changed, 420 insertions(+), 1 deletion(-) diff --git a/internal/daemon/remote/bridge.go b/internal/daemon/remote/bridge.go index ef4057acc..5066813f1 100644 --- a/internal/daemon/remote/bridge.go +++ b/internal/daemon/remote/bridge.go @@ -100,6 +100,16 @@ func NewBridge(opts BridgeOptions) (*Bridge, error) { if maxBundleBytes <= 0 { maxBundleBytes = defaultMaxBundleBytes } + bundleDir := strings.TrimSpace(opts.BundleDir) + if bundleDir != "" { + // A previous run may have died mid-swap, leaving a link's only tree in a + // staging dir. Nothing serves yet, so repair before the first upload. + recoverBundleDir(bundleDir, func(format string, args ...any) { + if opts.Log != nil { + opts.Log(fmt.Sprintf(format, args...)) + } + }) + } return &Bridge{ server: opts.Server, auth: opts.Authenticator, @@ -107,7 +117,7 @@ func NewBridge(opts BridgeOptions) (*Bridge, error) { minVersion: minVersion, handshakeTimeout: handshakeTimeout, authFailDelay: authFailDelay, - bundleDir: strings.TrimSpace(opts.BundleDir), + bundleDir: bundleDir, maxBundleBytes: maxBundleBytes, log: opts.Log, sem: make(chan struct{}, maxConns), diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index 0be8a5d8a..6104242a6 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -17,6 +17,7 @@ import ( "time" "github.com/Gitlawb/zero/internal/daemon" + "github.com/Gitlawb/zero/internal/lockutil" ) // gitTimeout bounds a single git invocation (bundle create/verify, clone) so a @@ -175,6 +176,17 @@ func streamFramesToFile(r io.Reader, w io.Writer, size int64) error { // sanitizeLinkID refuses every dot-prefixed id so a link can never name one. const stagingPrefix = ".staging-" +// lockDirName holds the per-link advisory lock files that serialize extracts +// across processes. Dot-prefixed for the same reason stagingPrefix is. +const lockDirName = ".extract-locks" + +// stagingLinkFile records, inside a staging dir, which link the backup beside it +// belongs to. Without it a crash leaves an orphan nothing can attribute. +const stagingLinkFile = "link" + +// extractLockPoll is how often a cross-process extract lock is retried. +const extractLockPoll = 50 * time.Millisecond + // renameDir moves a directory into its published location. It is a var so tests // can force a failure at the steps whose errors would otherwise be unrecoverable. var renameDir = os.Rename @@ -216,6 +228,110 @@ func lockExtract(dest string) func() { } } +// lockExtractFile takes the cross-process advisory lock for dest, waiting until +// ctx is done or the wait budget runs out. The in-process lock already excludes +// this daemon's own goroutines; this excludes a second daemon sharing the dir. +func lockExtractFile(ctx context.Context, bundleDir, dest string) (func(), error) { + lockDir := filepath.Join(bundleDir, lockDirName) + if err := os.MkdirAll(lockDir, 0o700); err != nil { + return nil, err + } + path := filepath.Join(lockDir, filepath.Base(dest)+".lock") + deadline := time.NewTimer(gitTimeout) + defer deadline.Stop() + for { + lock, err := lockutil.TryAcquireFileLockAt(bundleDir, path) + if err == nil { + return func() { _ = lock.Release() }, nil + } + if !errors.Is(err, lockutil.ErrLockHeld) { + return nil, err + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-deadline.C: + return nil, fmt.Errorf("remote: timed out waiting for the extract lock on %s", dest) + case <-time.After(extractLockPoll): + } + } +} + +// recoverBundleDir repairs what a crash left behind in dir. A staging dir whose +// backup belongs to a link with no live tree is put back; one that no extract +// can still own is removed. It is called once at bridge construction, before any +// upload is served, and never removes a staging dir a live extract may hold. +func recoverBundleDir(dir string, logf func(string, ...any)) { + if logf == nil { + logf = func(string, ...any) {} + } + entries, err := os.ReadDir(dir) + if err != nil { + if !os.IsNotExist(err) { + logf("remote: could not scan bundle dir %s: %v", dir, err) + } + return + } + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), stagingPrefix) { + continue + } + staging := filepath.Join(dir, entry.Name()) + if restoreStagedBackup(dir, staging, logf) { + continue + } + // No backup to attribute. Only reap once no clone can still be running: + // gitTimeout bounds a clone, so anything older than that is abandoned. + info, err := entry.Info() + if err != nil || time.Since(info.ModTime()) < 2*gitTimeout { + continue + } + if err := os.RemoveAll(staging); err != nil { + logf("remote: could not remove abandoned staging dir %s: %v", staging, err) + } + } +} + +// restoreStagedBackup puts a staged backup back if its link has no live tree. +// It reports whether staging was dealt with and needs no further handling. +func restoreStagedBackup(dir, staging string, logf func(string, ...any)) bool { + backup := filepath.Join(staging, "backup") + if _, err := os.Stat(backup); err != nil { + return false + } + raw, err := os.ReadFile(filepath.Join(staging, stagingLinkFile)) + if err != nil { + logf("remote: staged tree in %s has no link marker; leaving it in place", staging) + return true + } + id, err := sanitizeLinkID(string(raw)) + if err != nil { + logf("remote: staged tree in %s names an invalid link (%v); leaving it in place", staging, err) + return true + } + dest := filepath.Join(dir, id) + if !withinDir(dir, dest) { + logf("remote: staged tree in %s names a link outside the bundle dir; leaving it in place", staging) + return true + } + if _, err := os.Stat(dest); err == nil { + // The link already has a tree, so the backup is a stale copy. + if err := os.RemoveAll(staging); err != nil { + logf("remote: could not remove superseded staging dir %s: %v", staging, err) + } + return true + } + if err := os.Rename(backup, dest); err != nil { + logf("remote: could not restore the staged tree for %s from %s: %v", id, staging, err) + return true + } + logf("remote: restored the work tree for %s from %s after an interrupted extract", id, staging) + if err := os.RemoveAll(staging); err != nil { + logf("remote: could not remove staging dir %s after restoring %s: %v", staging, id, err) + } + return true +} + // extractBundle clones bundleFile into a staging dir beside dest, then swaps the // clone into place (replacing any prior extraction for this link id). git clone // needs a non-existent target, hence the staging dir. The live tree is moved @@ -234,6 +350,11 @@ func extractBundle(ctx context.Context, bundleFile, dest string, logf func(strin } unlock := lockExtract(dest) defer unlock() + unlockFile, err := lockExtractFile(ctx, parent, dest) + if err != nil { + return err + } + defer unlockFile() staging, err := os.MkdirTemp(parent, stagingPrefix+"*") if err != nil { @@ -262,6 +383,11 @@ func extractBundle(ctx context.Context, bundleFile, dest string, logf func(strin // Every rename stays inside parent, so none of them crosses a filesystem. backup := filepath.Join(staging, "backup") + // Record the link before moving its tree, so a crash in the swap window + // leaves something recoverBundleDir can attribute and put back. + if err := os.WriteFile(filepath.Join(staging, stagingLinkFile), []byte(filepath.Base(dest)), 0o600); err != nil { + return err + } restore := func() error { return nil } if err := os.Rename(dest, backup); err == nil { restore = func() error { return renameDir(backup, dest) } diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index 2679464b5..391f3741a 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -13,6 +13,8 @@ import ( "sync" "testing" "time" + + "github.com/Gitlawb/zero/internal/lockutil" ) // initTestRepo creates a temp git work tree with one committed file and returns @@ -413,4 +415,285 @@ func TestExtractBundleKeepsBackupWhenRestoreAlsoFails(t *testing.T) { if readErr != nil || string(got) != "v0" { t.Fatalf("retained backup = %q, err %v, want %q", got, readErr, "v0") } + + // The retained tree carries its link marker, so the next bridge start puts + // it back rather than leaving the link empty forever. + renameDir = real + recoverBundleDir(bundleDir, nil) + got, readErr = os.ReadFile(filepath.Join(dest, "a.txt")) + if readErr != nil { + t.Fatalf("recovery did not restore the retained tree: %v", readErr) + } + if string(got) != "v0" { + t.Fatalf("recovered a.txt = %q, want %q", got, "v0") + } +} + +// The link marker is read off disk, so a hostile or corrupt one must not steer a +// rename anywhere outside the bundle dir. +func TestRecoverBundleDirRefusesAMarkerThatEscapesTheBundleDir(t *testing.T) { + for _, marker := range []string{"../evil", "/etc/evil", "..", "a/b", ".hidden"} { + dir := t.TempDir() + staging := plantInterruptedExtract(t, dir, "proj-1", "v0") + if err := os.WriteFile(filepath.Join(staging, stagingLinkFile), []byte(marker), 0o600); err != nil { + t.Fatal(err) + } + outside := filepath.Join(filepath.Dir(dir), "evil") + + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }) + + if _, err := os.Stat(outside); !os.IsNotExist(err) { + t.Errorf("marker %q created %s: %v", marker, outside, err) + } + if _, err := os.Stat(filepath.Join(staging, "backup", "a.txt")); err != nil { + t.Errorf("marker %q: the tree should be left in place, not moved: %v", marker, err) + } + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "leaving it in place") }) { + t.Errorf("marker %q should be reported, got %v", marker, logged) + } + } +} + +// plantInterruptedExtract builds the on-disk state a crash between the two swap +// renames leaves behind: the link's only tree sitting in a staging dir. +func plantInterruptedExtract(t *testing.T, bundleDir, linkID, content string) string { + t.Helper() + staging := filepath.Join(bundleDir, stagingPrefix+"crashed") + backup := filepath.Join(staging, "backup") + if err := os.MkdirAll(filepath.Join(backup, ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(backup, "a.txt"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(staging, stagingLinkFile), []byte(linkID), 0o600); err != nil { + t.Fatal(err) + } + return staging +} + +func TestRecoverBundleDirRestoresInterruptedExtract(t *testing.T) { + dir := t.TempDir() + staging := plantInterruptedExtract(t, dir, "proj-1", "v0") + + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }) + + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil { + t.Fatalf("the interrupted extract was not restored: %v", err) + } + if string(got) != "v0" { + t.Fatalf("restored a.txt = %q, want %q", got, "v0") + } + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Errorf("staging should be gone after a successful restore, got %v", err) + } + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "restored the work tree") }) { + t.Errorf("a restore should be reported, got %v", logged) + } +} + +func TestRecoverBundleDirDropsBackupWhenTheLinkAlreadyHasATree(t *testing.T) { + dir := t.TempDir() + staging := plantInterruptedExtract(t, dir, "proj-1", "stale") + live := filepath.Join(dir, "proj-1") + if err := os.MkdirAll(live, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(live, "a.txt"), []byte("live"), 0o644); err != nil { + t.Fatal(err) + } + + recoverBundleDir(dir, nil) + + got, err := os.ReadFile(filepath.Join(live, "a.txt")) + if err != nil || string(got) != "live" { + t.Fatalf("the live tree must win: got %q err %v", got, err) + } + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Errorf("a superseded backup should be removed, got %v", err) + } +} + +func TestRecoverBundleDirKeepsABackupItCannotAttribute(t *testing.T) { + dir := t.TempDir() + staging := plantInterruptedExtract(t, dir, "proj-1", "v0") + if err := os.Remove(filepath.Join(staging, stagingLinkFile)); err != nil { + t.Fatal(err) + } + + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }) + + if _, err := os.Stat(filepath.Join(staging, "backup", "a.txt")); err != nil { + t.Errorf("an unattributable tree must be left alone, not deleted: %v", err) + } + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "no link marker") }) { + t.Errorf("an unattributable tree should be reported, got %v", logged) + } +} + +func TestRecoverBundleDirLeavesAStagingDirAnExtractCouldStillOwn(t *testing.T) { + dir := t.TempDir() + staging := filepath.Join(dir, stagingPrefix+"live") + if err := os.MkdirAll(filepath.Join(staging, "repo"), 0o700); err != nil { + t.Fatal(err) + } + + recoverBundleDir(dir, nil) + + if _, err := os.Stat(staging); err != nil { + t.Errorf("a fresh staging dir may belong to a running clone: %v", err) + } +} + +func TestRecoverBundleDirReapsAbandonedStaging(t *testing.T) { + dir := t.TempDir() + staging := filepath.Join(dir, stagingPrefix+"old") + if err := os.MkdirAll(filepath.Join(staging, "repo"), 0o700); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-3 * gitTimeout) + if err := os.Chtimes(staging, old, old); err != nil { + t.Fatal(err) + } + + recoverBundleDir(dir, nil) + + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Errorf("a staging dir older than any clone should be reaped, got %v", err) + } +} + +// A second daemon sharing the bundle dir must not extract the same link at the +// same time; the in-process lock cannot see it, so an advisory file lock does. +func TestExtractBundleWaitsForACrossProcessLock(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + b := testBundle(t, "a.txt", "v0") + + lockDir := filepath.Join(dir, lockDirName) + if err := os.MkdirAll(lockDir, 0o700); err != nil { + t.Fatal(err) + } + held, err := lockutil.TryAcquireFileLockAt(dir, filepath.Join(lockDir, "proj-1.lock")) + if err != nil { + t.Fatalf("take the lock as the other daemon would: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + if err := extractBundle(ctx, b, dest, nil); err == nil { + t.Fatal("an extract must not proceed while another process holds the link") + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Errorf("a blocked extract must not touch dest, got %v", err) + } + + // Once the other holder is gone the same extract goes through. + if err := held.Release(); err != nil { + t.Fatal(err) + } + if err := extractBundle(context.Background(), b, dest, nil); err != nil { + t.Fatalf("extract after the lock was released: %v", err) + } + if _, err := os.Stat(filepath.Join(dest, "a.txt")); err != nil { + t.Errorf("extract did not publish: %v", err) + } +} + +// End to end over TLS: two clients uploading the same link id at once must both +// succeed and leave one valid work tree. +func TestBridgeConcurrentUploadsOfOneLinkID(t *testing.T) { + srv := newBridgeServer(t, staticLauncher()) + auth, _ := NewTokenAuthenticator("tok") + bundleRoot := t.TempDir() + addr, ca := startBridge(t, srv, BridgeOptions{Authenticator: auth, BundleDir: bundleRoot}) + + first := initTestRepo(t, "a.txt", "v0") + second := initTestRepo(t, "a.txt", "v1") + cfg := RemoteConfig{Address: addr, Token: "tok", CACertFile: ca} + + var wg sync.WaitGroup + var mu sync.Mutex + var failures []error + for _, repo := range []string{first, second, first, second} { + wg.Add(1) + go func(repo string) { + defer wg.Done() + if _, err := UploadRepoBundle(cfg, repo, "proj-1"); err != nil { + mu.Lock() + failures = append(failures, err) + mu.Unlock() + } + }(repo) + } + wg.Wait() + + if len(failures) > 0 { + t.Errorf("concurrent uploads of one link id failed: %v", failures) + } + dest := filepath.Join(bundleRoot, "proj-1") + if _, err := os.Stat(filepath.Join(dest, ".git")); err != nil { + t.Fatalf("the link has no work tree after concurrent uploads: %v", err) + } + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil { + t.Fatalf("extracted tree is missing its file: %v", err) + } + if string(got) != "v0" && string(got) != "v1" { + t.Fatalf("a.txt = %q, want one of the uploaded trees", got) + } +} + +// End to end: a link id that could name a staging dir is refused, and refused +// before the client dials. +func TestBridgeRejectsDotPrefixedLinkID(t *testing.T) { + srv := newBridgeServer(t, staticLauncher()) + auth, _ := NewTokenAuthenticator("tok") + bundleRoot := t.TempDir() + addr, ca := startBridge(t, srv, BridgeOptions{Authenticator: auth, BundleDir: bundleRoot}) + + repo := initTestRepo(t, "a.txt", "v0") + for _, id := range []string{".staging-1", ".git", ".hidden"} { + if _, err := UploadRepoBundle(RemoteConfig{Address: addr, Token: "tok", CACertFile: ca}, repo, id); err == nil { + t.Errorf("upload with link id %q should be refused", id) + } + if _, err := os.Stat(filepath.Join(bundleRoot, id)); !os.IsNotExist(err) { + t.Errorf("link id %q must not create %s: %v", id, filepath.Join(bundleRoot, id), err) + } + } +} + +// End to end: a bridge started over a bundle dir a crash left mid-swap puts the +// tree back before it serves anything, and the link is usable again. +func TestBridgeRecoversInterruptedExtractOnStart(t *testing.T) { + srv := newBridgeServer(t, staticLauncher()) + auth, _ := NewTokenAuthenticator("tok") + bundleRoot := t.TempDir() + staging := plantInterruptedExtract(t, bundleRoot, "proj-1", "recovered") + + addr, ca := startBridge(t, srv, BridgeOptions{Authenticator: auth, BundleDir: bundleRoot}) + + got, err := os.ReadFile(filepath.Join(bundleRoot, "proj-1", "a.txt")) + if err != nil { + t.Fatalf("the bridge did not restore the interrupted extract: %v", err) + } + if string(got) != "recovered" { + t.Fatalf("restored a.txt = %q, want %q", got, "recovered") + } + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Errorf("staging should be cleared after recovery, got %v", err) + } + // The recovered link still accepts a fresh upload. + repo := initTestRepo(t, "a.txt", "v2") + if _, err := UploadRepoBundle(RemoteConfig{Address: addr, Token: "tok", CACertFile: ca}, repo, "proj-1"); err != nil { + t.Fatalf("upload to a recovered link: %v", err) + } + got, err = os.ReadFile(filepath.Join(bundleRoot, "proj-1", "a.txt")) + if err != nil || string(got) != "v2" { + t.Fatalf("after re-upload a.txt = %q, err %v, want %q", got, err, "v2") + } } From 6936d998274ec6635875cd2608254e12636834da Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:50:13 -0500 Subject: [PATCH 04/34] fix(dictation): keep the previous engine install if promotion fails downloadVerifyExtract removed destDir and only then renamed the freshly extracted stage over it, the same shape just fixed in the remote bundle extractor. A rename that fails after the removal (or a crash in that window) leaves the user with no engine at all, and the deferred stage cleanup takes the replacement with it. Pull the promotion into promoteStagedDir, which sets the previous install aside instead of deleting it and puts it back if the rename fails. If that restore fails too, the set-aside copy is kept rather than cleaned up, and the error names it. --- internal/dictation/download.go | 52 +++++++++++-- internal/dictation/download_test.go | 117 ++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 7 deletions(-) diff --git a/internal/dictation/download.go b/internal/dictation/download.go index b1343327e..85692c826 100644 --- a/internal/dictation/download.go +++ b/internal/dictation/download.go @@ -695,13 +695,8 @@ func downloadVerifyExtract(ctx context.Context, client *http.Client, asset resol if err := extractTarBz2(tmpPath, stageDir, "Extracting "+label, progress); err != nil { return err } - // Remove any pre-existing destDir so Rename is atomic on the same - // filesystem (os.Rename refuses to overwrite a non-empty dir). - if err := os.RemoveAll(destDir); err != nil { - return fmt.Errorf("clearing previous %s install: %w", label, err) - } - if err := os.Rename(stageDir, destDir); err != nil { - return fmt.Errorf("promoting staged %s: %w", label, err) + if err := promoteStagedDir(stageDir, destDir, label); err != nil { + return err } cleanupStage = false return nil @@ -757,6 +752,49 @@ func resolveEnginePaths(engineDir string, targetWindows bool) (bin, server strin return bin, server } +// promoteStagedDir moves stageDir into place at destDir. os.Rename refuses to +// overwrite a non-empty directory, so any previous install has to move out of +// the way first; it is set aside rather than deleted, and put back if the +// promotion fails, so destDir is never left holding no install at all. If the +// restore fails too, the set-aside copy is kept and named in the error. +func promoteStagedDir(stageDir, destDir, label string) error { + holder := "" + cleanupHolder := true + defer func() { + if holder != "" && cleanupHolder { + _ = os.RemoveAll(holder) + } + }() + + restore := func() error { return nil } + if _, err := os.Lstat(destDir); err == nil { + holder, err = os.MkdirTemp(filepath.Dir(destDir), filepath.Base(destDir)+".previous-*") + if err != nil { + return fmt.Errorf("setting aside previous %s install: %w", label, err) + } + previous := filepath.Join(holder, "install") + if err := os.Rename(destDir, previous); err != nil { + return fmt.Errorf("setting aside previous %s install: %w", label, err) + } + restore = func() error { return renameStagedDir(previous, destDir) } + } else if !os.IsNotExist(err) { + return fmt.Errorf("checking previous %s install: %w", label, err) + } + + if err := renameStagedDir(stageDir, destDir); err != nil { + if restoreErr := restore(); restoreErr != nil { + cleanupHolder = false + return fmt.Errorf("promoting staged %s: %w (previous install left in %s: %v)", label, err, holder, restoreErr) + } + return fmt.Errorf("promoting staged %s: %w", label, err) + } + return nil +} + +// renameStagedDir moves a directory into its published location. It is a var so +// tests can force the failures the restore path above exists for. +var renameStagedDir = os.Rename + // extractTarBz2 unpacks a bzip2-compressed tar into destDir, guarding against // path-traversal entries. It reports extraction progress by how much of the // (compressed) archive it has consumed — the same MB scale as the download, so diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index da267e35a..364dac2bd 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -243,3 +243,120 @@ func TestUnsupportedPlatformSetupError(t *testing.T) { t.Fatalf("want *SetupError for unsupported platform, got %v", err) } } + +func stagedTree(t *testing.T, dir, content string) string { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "engine"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return dir +} + +func TestPromoteStagedDirReplacesAPreviousInstall(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + if err := promoteStagedDir(stage, dest, "engine"); err != nil { + t.Fatalf("promote: %v", err) + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "new" { + t.Fatalf("engine = %q, err %v, want %q", got, err, "new") + } + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if strings.Contains(e.Name(), ".previous-") { + t.Errorf("the set-aside install should be cleaned up, found %s", e.Name()) + } + } +} + +func TestPromoteStagedDirWorksWithNoPreviousInstall(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + if err := promoteStagedDir(stage, dest, "engine"); err != nil { + t.Fatalf("promote: %v", err) + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "new" { + t.Fatalf("engine = %q, err %v, want %q", got, err, "new") + } +} + +// The old code deleted the previous install before it had anything to put in +// its place, so a failed promotion left the user with no engine at all. +func TestPromoteStagedDirKeepsThePreviousInstallWhenPromotionFails(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + real := renameStagedDir + calls := 0 + renameStagedDir = func(from, to string) error { + calls++ + if calls == 1 { + return errors.New("injected promotion failure") + } + return real(from, to) + } + t.Cleanup(func() { renameStagedDir = real }) + + if err := promoteStagedDir(stage, dest, "engine"); err == nil { + t.Fatal("a failed promotion must be reported") + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil { + t.Fatalf("the previous install was not restored: %v", err) + } + if string(got) != "old" { + t.Fatalf("engine = %q, want the previous %q", got, "old") + } +} + +// If the restore fails too, the only remaining copy must survive the cleanup. +func TestPromoteStagedDirKeepsTheSetAsideCopyWhenRestoreAlsoFails(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + real := renameStagedDir + renameStagedDir = func(string, string) error { return errors.New("injected rename failure") } + t.Cleanup(func() { renameStagedDir = real }) + + err := promoteStagedDir(stage, dest, "engine") + if err == nil { + t.Fatal("a failed promotion must be reported") + } + if !strings.Contains(err.Error(), "previous install left in") { + t.Errorf("the error should name the retained copy, got: %v", err) + } + entries, readErr := os.ReadDir(root) + if readErr != nil { + t.Fatal(readErr) + } + found := false + for _, e := range entries { + if !strings.Contains(e.Name(), ".previous-") { + continue + } + got, readErr := os.ReadFile(filepath.Join(root, e.Name(), "install", "engine")) + if readErr == nil && string(got) == "old" { + found = true + } + } + if !found { + t.Error("the previous install was deleted along with the holder dir") + } +} From f0fce95535627839a244b13f83280b0776fbd8bf Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:04:20 -0500 Subject: [PATCH 05/34] fix(daemon/remote): do not let startup recovery take a live extract's tree A crashed extract and a running one leave the same thing on disk: the backup set aside in staging and dest briefly absent between the two renames. Recovery could not tell them apart, so a second daemon starting in that window restored the backup out from under the running extract. The upload then failed with a rename error and an apology naming a backup path that no longer existed. Only the per-link lock separates the two cases. Recovery now tries that lock without waiting and skips any link something still owns, which is also the right answer for a link another daemon is actively serving. --- internal/daemon/remote/bundle.go | 45 +++++++++++++++++++------ internal/daemon/remote/bundle_test.go | 48 +++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 10 deletions(-) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index 6104242a6..51f1a832f 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -232,21 +232,16 @@ func lockExtract(dest string) func() { // ctx is done or the wait budget runs out. The in-process lock already excludes // this daemon's own goroutines; this excludes a second daemon sharing the dir. func lockExtractFile(ctx context.Context, bundleDir, dest string) (func(), error) { - lockDir := filepath.Join(bundleDir, lockDirName) - if err := os.MkdirAll(lockDir, 0o700); err != nil { - return nil, err - } - path := filepath.Join(lockDir, filepath.Base(dest)+".lock") deadline := time.NewTimer(gitTimeout) defer deadline.Stop() for { - lock, err := lockutil.TryAcquireFileLockAt(bundleDir, path) - if err == nil { - return func() { _ = lock.Release() }, nil - } - if !errors.Is(err, lockutil.ErrLockHeld) { + release, held, err := tryLockExtractFile(bundleDir, dest) + if err != nil { return nil, err } + if !held { + return release, nil + } select { case <-ctx.Done(): return nil, ctx.Err() @@ -257,6 +252,24 @@ func lockExtractFile(ctx context.Context, bundleDir, dest string) (func(), error } } +// tryLockExtractFile takes the per-link advisory lock without waiting. It +// reports held when a live extract owns the link, which is never an error: the +// caller either waits or leaves that link alone. +func tryLockExtractFile(bundleDir, dest string) (release func(), held bool, err error) { + lockDir := filepath.Join(bundleDir, lockDirName) + if err := os.MkdirAll(lockDir, 0o700); err != nil { + return nil, false, err + } + lock, err := lockutil.TryAcquireFileLockAt(bundleDir, filepath.Join(lockDir, filepath.Base(dest)+".lock")) + if err != nil { + if errors.Is(err, lockutil.ErrLockHeld) { + return nil, true, nil + } + return nil, false, err + } + return func() { _ = lock.Release() }, false, nil +} + // recoverBundleDir repairs what a crash left behind in dir. A staging dir whose // backup belongs to a link with no live tree is put back; one that no extract // can still own is removed. It is called once at bridge construction, before any @@ -314,6 +327,18 @@ func restoreStagedBackup(dir, staging string, logf func(string, ...any)) bool { logf("remote: staged tree in %s names a link outside the bundle dir; leaving it in place", staging) return true } + // A live extract mid-swap looks exactly like a crashed one: its backup is + // aside and dest is briefly absent. Only the lock tells them apart, so skip + // any link something still owns rather than taking its tree. + release, held, err := tryLockExtractFile(dir, dest) + if err != nil { + logf("remote: could not lock %s while recovering %s: %v", id, staging, err) + return true + } + if held { + return true + } + defer release() if _, err := os.Stat(dest); err == nil { // The link already has a tree, so the backup is a stale copy. if err := os.RemoveAll(staging); err != nil { diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index 391f3741a..d5ade1567 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -697,3 +697,51 @@ func TestBridgeRecoversInterruptedExtractOnStart(t *testing.T) { t.Fatalf("after re-upload a.txt = %q, err %v, want %q", got, err, "v2") } } + +// A crashed extract and a live one look identical on disk: the backup is aside +// and dest is briefly absent. A second daemon starting in that window must not +// take the running extract's tree. +func TestRecoverBundleDirLeavesALiveExtractAlone(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + inPublish := make(chan struct{}) + real := renameDir + var once sync.Once + renameDir = func(from, to string) error { + // Stall only the publish (clone into dest), not the restore. + if filepath.Base(from) == "repo" { + once.Do(func() { close(inPublish) }) + time.Sleep(300 * time.Millisecond) + } + return real(from, to) + } + t.Cleanup(func() { renameDir = real }) + + var wg sync.WaitGroup + var extractErr error + wg.Add(1) + go func() { + defer wg.Done() + extractErr = extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil) + }() + + <-inPublish + time.Sleep(20 * time.Millisecond) // the second daemon starts mid-swap + recoverBundleDir(dir, nil) + wg.Wait() + + if extractErr != nil { + t.Errorf("recovery interfered with a live extract: %v", extractErr) + } + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil { + t.Fatalf("dest holds no tree after the live extract: %v", err) + } + if string(got) != "v1" { + t.Fatalf("a.txt = %q, want the published %q", got, "v1") + } +} From af745d0c61401e8ccd3253410b724e923dc62b59 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:45:07 -0500 Subject: [PATCH 06/34] fix(daemon/remote): stop recovery from destroying the tree it should restore Two ways the new startup recovery lost data, both found by an adversarial review on a second model and both reproduced before fixing. A link can have more than one staged backup: a staging cleanup that could not finish leaves one behind, and a later crash adds another. Recovery walked them in directory order, so an older leftover could be restored first and the newer backup then deleted as superseded. Order staged dirs newest backup first, and only drop a backup that is provably older than the live tree; a backup that is not may be the newer copy no restart has published yet. Link ids starting with '.' were legal until the previous commit, so a work tree may already be published under a name that now matches the staging prefix. The age reaper treated it as an abandoned extract and deleted it on the first start after upgrading. A directory with a .git at its root is not a staged extract, so leave it alone and say so. --- internal/daemon/remote/bundle.go | 44 +++++++++++-- internal/daemon/remote/bundle_test.go | 94 +++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 6 deletions(-) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index 51f1a832f..e7acaff3f 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -12,6 +12,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strings" "sync" "time" @@ -285,17 +286,41 @@ func recoverBundleDir(dir string, logf func(string, ...any)) { } return } + // One link can have several staged backups: a cleanup that could not finish + // leaves one behind, and a later crash adds another. Newest first, so the + // tree that comes back is the most recent one rather than whichever the + // directory happened to list first. + staged := make([]string, 0, len(entries)) + backupTime := map[string]time.Time{} for _, entry := range entries { if !entry.IsDir() || !strings.HasPrefix(entry.Name(), stagingPrefix) { continue } - staging := filepath.Join(dir, entry.Name()) + path := filepath.Join(dir, entry.Name()) + staged = append(staged, path) + if info, err := os.Stat(filepath.Join(path, "backup")); err == nil { + backupTime[path] = info.ModTime() + } + } + slices.SortFunc(staged, func(a, b string) int { + return backupTime[b].Compare(backupTime[a]) + }) + + for _, staging := range staged { if restoreStagedBackup(dir, staging, logf) { continue } - // No backup to attribute. Only reap once no clone can still be running: - // gitTimeout bounds a clone, so anything older than that is abandoned. - info, err := entry.Info() + // No backup to attribute. A dir with a .git at its root is not staging at + // all: link ids starting with '.' used to be accepted, so this may be a + // work tree someone published under a name that now looks reserved. + // Never reap that. + if _, err := os.Stat(filepath.Join(staging, ".git")); err == nil { + logf("remote: %s holds a work tree, not a staged extract; leaving it in place", staging) + continue + } + // Only reap once no clone can still be running: gitTimeout bounds a + // clone, so anything older than that is abandoned. + info, err := os.Stat(staging) if err != nil || time.Since(info.ModTime()) < 2*gitTimeout { continue } @@ -339,8 +364,15 @@ func restoreStagedBackup(dir, staging string, logf func(string, ...any)) bool { return true } defer release() - if _, err := os.Stat(dest); err == nil { - // The link already has a tree, so the backup is a stale copy. + if destInfo, err := os.Stat(dest); err == nil { + // The link already has a tree. Only drop the backup when it is provably + // the older copy; otherwise it may be the newer one a restart has not + // published yet, and deleting it would lose that work. + backupInfo, statErr := os.Stat(backup) + if statErr != nil || !backupInfo.ModTime().Before(destInfo.ModTime()) { + logf("remote: staged tree in %s is not older than the live tree for %s; leaving it in place", staging, id) + return true + } if err := os.RemoveAll(staging); err != nil { logf("remote: could not remove superseded staging dir %s: %v", staging, err) } diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index d5ade1567..c8e36bfd4 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -745,3 +745,97 @@ func TestRecoverBundleDirLeavesALiveExtractAlone(t *testing.T) { t.Fatalf("a.txt = %q, want the published %q", got, "v1") } } + +// stageBackup plants a staging dir holding a backup tree for linkID. +func stageBackup(t *testing.T, dir, name, linkID, content string, age time.Duration) string { + t.Helper() + staging := filepath.Join(dir, stagingPrefix+name) + backup := filepath.Join(staging, "backup") + if err := os.MkdirAll(filepath.Join(backup, ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(backup, "a.txt"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(staging, stagingLinkFile), []byte(linkID), 0o600); err != nil { + t.Fatal(err) + } + if age > 0 { + when := time.Now().Add(-age) + if err := os.Chtimes(backup, when, when); err != nil { + t.Fatal(err) + } + } + return staging +} + +// One link can end up with more than one staged backup: a staging cleanup that +// could not finish leaves one behind, and a later crash adds another. Recovery +// must bring back the newest, and must not delete the newer one as superseded +// just because directory order put the older one first. +func TestRecoverBundleDirRestoresTheNewestOfSeveralBackups(t *testing.T) { + dir := t.TempDir() + stageBackup(t, dir, "aaa-old", "proj-1", "v0", time.Hour) + newer := stageBackup(t, dir, "zzz-new", "proj-1", "v1", 0) + + recoverBundleDir(dir, nil) + + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil { + t.Fatalf("nothing was restored: %v", err) + } + if string(got) != "v1" { + t.Errorf("restored a.txt = %q, want the newest tree %q", got, "v1") + if _, err := os.Stat(filepath.Join(newer, "backup", "a.txt")); err != nil { + t.Errorf("and the newest tree was deleted as superseded: %v", err) + } + } +} + +// A backup that is not provably older than the live tree may be the newer copy +// a restart has not published yet, so it is kept rather than dropped. +func TestRecoverBundleDirKeepsABackupNewerThanTheLiveTree(t *testing.T) { + dir := t.TempDir() + live := filepath.Join(dir, "proj-1") + if err := os.MkdirAll(live, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(live, "a.txt"), []byte("live"), 0o644); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-time.Hour) + if err := os.Chtimes(live, old, old); err != nil { + t.Fatal(err) + } + staging := stageBackup(t, dir, "newer", "proj-1", "v2", 0) + + recoverBundleDir(dir, nil) + + if _, err := os.Stat(filepath.Join(staging, "backup", "a.txt")); err != nil { + t.Errorf("a backup newer than the live tree must be kept: %v", err) + } +} + +// Link ids starting with '.' used to be accepted, so a work tree may already be +// published under a name that now looks like a staging dir. The reaper must not +// mistake it for an abandoned extract and delete it on the first start. +func TestRecoverBundleDirKeepsAWorkTreePublishedUnderAReservedName(t *testing.T) { + dir := t.TempDir() + published := filepath.Join(dir, stagingPrefix+"foo") + if err := os.MkdirAll(filepath.Join(published, ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(published, "a.txt"), []byte("someones repo"), 0o644); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-3 * gitTimeout) + if err := os.Chtimes(published, old, old); err != nil { + t.Fatal(err) + } + + recoverBundleDir(dir, nil) + + if _, err := os.Stat(filepath.Join(published, "a.txt")); err != nil { + t.Errorf("a published work tree was reaped on upgrade: %v", err) + } +} From db658d77ca3494ea5d344cd6371af83ab2fcead5 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:11:11 -0500 Subject: [PATCH 07/34] fix(dictation): restore an install left aside by an interrupted promotion promoteStagedDir sets the previous install aside before renaming the new one into place. A process stop between those two renames leaves destDir absent and the only usable install inside the .previous-* holder, and nothing looked at that holder: EnsureLocalEngine gates on fileExists and would download a fresh engine instead, so a host that cannot reach the network stayed without dictation while holding a working copy. Put the holder back before the idempotency check. Anything already at destDir wins, and that check is explicit rather than leaning on os.Rename refusing an existing directory. --- internal/dictation/download.go | 31 ++++++++++++ internal/dictation/download_test.go | 75 +++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/internal/dictation/download.go b/internal/dictation/download.go index 85692c826..f414a8550 100644 --- a/internal/dictation/download.go +++ b/internal/dictation/download.go @@ -487,6 +487,10 @@ func EnsureLocalEngine(ctx context.Context, opts DownloadOptions) (EngineCompone targetWindows := strings.HasPrefix(key, "windows-") // Resolve through the tarball's flattened subdir so an ALREADY-extracted // engine is found and not needlessly re-downloaded (the idempotency check). + // A previous run may have been stopped mid-promotion, leaving the only + // install in a holder beside engineDir. Put it back before deciding whether + // anything needs downloading. + restoreInterruptedPromotion(engineDir) binPath, serverPath := resolveEnginePaths(engineDir, targetWindows) if !fileExists(binPath) { pinned := "" @@ -752,6 +756,33 @@ func resolveEnginePaths(engineDir string, targetWindows bool) (bin, server strin return bin, server } +// restoreInterruptedPromotion puts back an install that promoteStagedDir set +// aside but never replaced, which is what a process stop between its two renames +// leaves behind: destDir absent and the only usable copy in a .previous-* holder +// nothing else looks at. Anything already at destDir wins, and the check for it +// is explicit rather than leaning on os.Rename refusing an existing directory. +// Best effort by design, since the caller can still download a fresh engine. +func restoreInterruptedPromotion(destDir string) { + if _, err := os.Lstat(destDir); err == nil { + return + } + holders, err := filepath.Glob(destDir + ".previous-*") + if err != nil { + return + } + for _, holder := range holders { + install := filepath.Join(holder, "install") + if _, err := os.Stat(install); err != nil { + continue + } + if err := renameStagedDir(install, destDir); err != nil { + continue + } + _ = os.RemoveAll(holder) + return + } +} + // promoteStagedDir moves stageDir into place at destDir. os.Rename refuses to // overwrite a non-empty directory, so any previous install has to move out of // the way first; it is set aside rather than deleted, and put back if the diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index 364dac2bd..08a86e99a 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -360,3 +360,78 @@ func TestPromoteStagedDirKeepsTheSetAsideCopyWhenRestoreAlsoFails(t *testing.T) t.Error("the previous install was deleted along with the holder dir") } } + +// A process stop between the two renames in promoteStagedDir leaves destDir +// absent and the only usable install in a .previous-* holder. Nothing else knows +// about that holder, so without a repair the engine is re-downloaded and a host +// that cannot reach the network stays without dictation despite having a copy. +func TestRestoreInterruptedPromotionPutsTheInstallBack(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + holder := filepath.Join(root, filepath.Base(dest)+".previous-abc") + install := filepath.Join(holder, "install") + if err := os.MkdirAll(install, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(install, "engine"), []byte("kept"), 0o644); err != nil { + t.Fatal(err) + } + + restoreInterruptedPromotion(dest) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil { + t.Fatalf("the interrupted promotion was not restored: %v", err) + } + if string(got) != "kept" { + t.Fatalf("restored engine = %q, want %q", got, "kept") + } + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("the holder should be cleared after a restore, got %v", err) + } +} + +// A holder is a leftover, never a replacement for whatever is already at destDir, +// empty or not. os.Rename refuses an existing directory either way, so this +// pins the behavior rather than one implementation of it. +func TestRestoreInterruptedPromotionLeavesAnExistingDestAlone(t *testing.T) { + for _, tc := range []struct{ name, live string }{ + {"empty dest", ""}, + {"populated dest", "live"}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + if err := os.MkdirAll(dest, 0o755); err != nil { + t.Fatal(err) + } + if tc.live != "" { + if err := os.WriteFile(filepath.Join(dest, "engine"), []byte(tc.live), 0o644); err != nil { + t.Fatal(err) + } + } + holder := filepath.Join(root, filepath.Base(dest)+".previous-abc") + install := filepath.Join(holder, "install") + if err := os.MkdirAll(install, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(install, "engine"), []byte("stale"), 0o644); err != nil { + t.Fatal(err) + } + + restoreInterruptedPromotion(dest) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if tc.live == "" { + if err == nil { + t.Fatalf("an existing dest was replaced by a stale holder: engine = %q", got) + } + } else if err != nil || string(got) != tc.live { + t.Fatalf("engine = %q, err %v, want the live %q", got, err, tc.live) + } + if _, err := os.Stat(install); err != nil { + t.Errorf("the holder must be left intact when dest exists: %v", err) + } + }) + } +} From 23da5893d0f239d09ccf8feed1e169a6886c209e Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:38:57 -0500 Subject: [PATCH 08/34] fix(dictation): restore an interrupted model promotion, and the newest one promoteStagedDir is shared by the engine and the model, but only the engine path called restoreInterruptedPromotion. A stop after the model directory was renamed into its holder left modelDir absent, so the next start went straight to dirHasModel, missed the verified model sitting in the holder, and fell into resolveAsset. Offline that is not a slow path, it is no dictation at all. Recovery also took the first Glob match, whose lexical order says nothing about which install is more recent. A cleanup that could not finish leaves an older holder behind, and a later interrupted promotion then gives recovery two valid installs to choose between. The promotion now records its creation time in the holder name and recovery restores the newest, leaving any holder it cannot order intact rather than deleting it on a guess. --- internal/dictation/download.go | 50 +++++++++++++++- internal/dictation/download_test.go | 89 +++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 2 deletions(-) diff --git a/internal/dictation/download.go b/internal/dictation/download.go index f414a8550..19032afcd 100644 --- a/internal/dictation/download.go +++ b/internal/dictation/download.go @@ -2,6 +2,7 @@ package dictation import ( "archive/tar" + "cmp" "compress/bzip2" "context" "crypto/sha256" @@ -13,7 +14,10 @@ import ( "os" "path/filepath" "runtime" + "slices" + "strconv" "strings" + "time" ) // Auto-download of the local engine + a default model (opt-in, behind a confirm @@ -519,6 +523,10 @@ func EnsureLocalEngine(ctx context.Context, opts DownloadOptions) (EngineCompone modelDirName = "model-moonshine-tiny-en-int8" } modelDir := filepath.Join(opts.DestRoot, modelDirName) + // promoteStagedDir is shared with the model, so a stop mid-promotion leaves + // the model in a holder too. Put it back before deciding anything is + // missing: without this an offline user has no download to fall back on. + restoreInterruptedPromotion(modelDir) if !dirHasModel(modelDir) { asset, err := resolveAsset(ctx, client, apiBase, modelReleaseTag, modelName, "") if err != nil { @@ -756,6 +764,28 @@ func resolveEnginePaths(engineDir string, targetWindows bool) (bin, server strin return bin, server } +// holderSuffix is what promoteStagedDir appends to an install's own name for the +// holder it sets that install aside in. The creation time goes in the name +// because recovery has to pick the NEWEST holder when a failed cleanup left an +// older one behind, and nothing else records that order: Glob sorts lexically +// and a directory mtime tracks the install's contents, not its promotion. +const holderSuffix = ".previous-" + +// holderStamp reads back the creation time in a holder name, reporting false for +// a name it cannot order (one this package did not write). +func holderStamp(destDir, holder string) (int64, bool) { + rest := strings.TrimPrefix(filepath.Base(holder), filepath.Base(destDir)+holderSuffix) + digits, _, found := strings.Cut(rest, "-") + if !found { + return 0, false + } + stamp, err := strconv.ParseInt(digits, 10, 64) + if err != nil { + return 0, false + } + return stamp, true +} + // restoreInterruptedPromotion puts back an install that promoteStagedDir set // aside but never replaced, which is what a process stop between its two renames // leaves behind: destDir absent and the only usable copy in a .previous-* holder @@ -766,10 +796,23 @@ func restoreInterruptedPromotion(destDir string) { if _, err := os.Lstat(destDir); err == nil { return } - holders, err := filepath.Glob(destDir + ".previous-*") + holders, err := filepath.Glob(destDir + holderSuffix + "*") if err != nil { return } + // Newest first: an unstamped holder is the least recent thing we can claim + // to know about, so it is only reached once every stamped one has failed. + slices.SortStableFunc(holders, func(a, b string) int { + sa, oka := holderStamp(destDir, a) + sb, okb := holderStamp(destDir, b) + if oka != okb { + if oka { + return -1 + } + return 1 + } + return cmp.Compare(sb, sa) + }) for _, holder := range holders { install := filepath.Join(holder, "install") if _, err := os.Stat(install); err != nil { @@ -778,6 +821,8 @@ func restoreInterruptedPromotion(destDir string) { if err := renameStagedDir(install, destDir); err != nil { continue } + // Only the holder this install came out of is removed; an older one is + // left for a human, never deleted on a guess about which is current. _ = os.RemoveAll(holder) return } @@ -799,7 +844,8 @@ func promoteStagedDir(stageDir, destDir, label string) error { restore := func() error { return nil } if _, err := os.Lstat(destDir); err == nil { - holder, err = os.MkdirTemp(filepath.Dir(destDir), filepath.Base(destDir)+".previous-*") + holder, err = os.MkdirTemp(filepath.Dir(destDir), + fmt.Sprintf("%s%s%020d-*", filepath.Base(destDir), holderSuffix, time.Now().UnixNano())) if err != nil { return fmt.Errorf("setting aside previous %s install: %w", label, err) } diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index 08a86e99a..395a43bc4 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -435,3 +435,92 @@ func TestRestoreInterruptedPromotionLeavesAnExistingDestAlone(t *testing.T) { }) } } + +// An interrupted promotion of the MODEL leaves it in a holder exactly as it does +// for the engine, and the model side is what an offline user loses if nothing +// puts it back: with no network there is no download to fall back to. +func TestEnsureLocalEngineRestoresAnInterruptedModelPromotion(t *testing.T) { + srv := fakeReleaseServer(t, engineSHA, modelSHA) + dest := t.TempDir() + opts := DownloadOptions{ + DestRoot: dest, EngineVersion: "test", APIBase: srv.URL, platformKey: "linux-amd64", skipPinned: true, + } + comp, err := EnsureLocalEngine(context.Background(), opts) + if err != nil { + t.Fatalf("seeding the install: %v", err) + } + + // Stop the world where promoteStagedDir has moved the model aside but has + // not yet published the staged copy. + modelDir := filepath.Dir(comp.ModelPath) + holder := modelDir + ".previous-abc" + if err := os.MkdirAll(holder, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Rename(modelDir, filepath.Join(holder, "install")); err != nil { + t.Fatal(err) + } + + offline := offlineAPIBase(t) + got, err := EnsureLocalEngine(context.Background(), DownloadOptions{ + DestRoot: dest, EngineVersion: "test", APIBase: offline, platformKey: "linux-amd64", skipPinned: true, + }) + if err != nil { + t.Fatalf("the model set aside by an interrupted promotion was not restored: %v", err) + } + if !fileExists(filepath.Join(got.ModelPath, "tokens.txt")) { + t.Errorf("model tokens.txt missing under %q", got.ModelPath) + } + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("the holder should be cleared after a restore, got %v", err) + } +} + +// offlineAPIBase returns an API base nothing is listening on, so any attempt to +// resolve a release asset fails instead of silently downloading. +func offlineAPIBase(t *testing.T) string { + t.Helper() + srv := httptest.NewServer(http.NewServeMux()) + url := srv.URL + srv.Close() + return url +} + +// plantHolder writes an install into a holder named the way promoteStagedDir +// names one, so recovery sees the same shape it does in production. +func plantHolder(t *testing.T, destDir string, stamp int64, content string) string { + t.Helper() + holder := fmt.Sprintf("%s%s%020d-%d", destDir, holderSuffix, stamp, stamp) + install := filepath.Join(holder, "install") + if err := os.MkdirAll(install, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(install, "engine"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return holder +} + +// A cleanup that could not finish leaves an old holder behind; a later +// interrupted promotion adds a second one. Recovery has to put back the newer +// install, and Glob's lexical order is no evidence of which that is. +func TestRestoreInterruptedPromotionPrefersTheNewestHolder(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stale := plantHolder(t, dest, 100, "stale") + current := plantHolder(t, dest, 200, "current") + + restoreInterruptedPromotion(dest) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "current" { + t.Fatalf("restored engine = %q (err %v), want the newest holder's %q", got, err, "current") + } + if _, err := os.Stat(current); !os.IsNotExist(err) { + t.Errorf("the restored holder should be cleared, got %v", err) + } + // The loser is left for a human rather than deleted on a guess. + if _, err := os.Stat(filepath.Join(stale, "install", "engine")); err != nil { + t.Errorf("the older holder must be left intact: %v", err) + } +} From 1179915ff22278a05f56fee64689037a23de3cc7 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:38:57 -0500 Subject: [PATCH 09/34] fix(daemon/remote): order bundle recovery by the extract, not by mtimes restoreStagedBackup proved a backup stale by comparing directory mtimes, which two directories can tie on: a filesystem with coarse timestamps gives the backup and the tree published over it the same value, and recovery then keeps a superseded work tree forever. The proof does not need a timestamp. A backup is filled by renaming dest aside, so it only ever holds the tree that was live before dest, and dest holding anything at all means a later extract published over it. That leaves one case where the ordering is genuinely unknown: a tree recovery itself just put back was not published over anything. Extracts now stamp their staging name with a creation time, so recovery restores the newest backup and drops the older ones it can order against it. A backup carrying no comparable order is kept and reported. The two tests that pinned the mtime contract move to this one. A backup newer than the live tree was reachable only by setting mtimes by hand, never by the transaction, so that case is replaced by the fail-safe that does hold. --- internal/daemon/remote/bundle.go | 84 +++++++++++++++------ internal/daemon/remote/bundle_test.go | 103 ++++++++++++++++++++------ 2 files changed, 142 insertions(+), 45 deletions(-) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index e7acaff3f..8e7b6acb6 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -1,6 +1,7 @@ package remote import ( + "cmp" "context" "crypto/sha256" "encoding/hex" @@ -13,6 +14,7 @@ import ( "os/exec" "path/filepath" "slices" + "strconv" "strings" "sync" "time" @@ -289,25 +291,33 @@ func recoverBundleDir(dir string, logf func(string, ...any)) { // One link can have several staged backups: a cleanup that could not finish // leaves one behind, and a later crash adds another. Newest first, so the // tree that comes back is the most recent one rather than whichever the - // directory happened to list first. - staged := make([]string, 0, len(entries)) - backupTime := map[string]time.Time{} + // directory happened to list first. The order comes from the stamp the + // extract wrote into the staging name, not from a directory mtime: mtimes + // track a tree's contents, and a coarse filesystem gives two of them the + // same value anyway. + staged := make([]stagedExtract, 0, len(entries)) for _, entry := range entries { if !entry.IsDir() || !strings.HasPrefix(entry.Name(), stagingPrefix) { continue } - path := filepath.Join(dir, entry.Name()) - staged = append(staged, path) - if info, err := os.Stat(filepath.Join(path, "backup")); err == nil { - backupTime[path] = info.ModTime() - } + stamp, stamped := stagingStamp(entry.Name()) + staged = append(staged, stagedExtract{path: filepath.Join(dir, entry.Name()), stamp: stamp, stamped: stamped}) } - slices.SortFunc(staged, func(a, b string) int { - return backupTime[b].Compare(backupTime[a]) + slices.SortStableFunc(staged, func(a, b stagedExtract) int { + if a.stamped != b.stamped { + if a.stamped { + return -1 + } + return 1 + } + return cmp.Compare(b.stamp, a.stamp) }) - for _, staging := range staged { - if restoreStagedBackup(dir, staging, logf) { + // Which links this pass put a tree back on, and the staging it came from. + restored := map[string]stagedExtract{} + for _, s := range staged { + staging := s.path + if restoreStagedBackup(dir, s, restored, logf) { continue } // No backup to attribute. A dir with a .git at its root is not staging at @@ -330,9 +340,33 @@ func recoverBundleDir(dir string, logf func(string, ...any)) { } } +// stagedExtract is a staging dir plus the creation order recorded in its name. +// stamped is false for a name this package did not write, which is an ordering +// it must not claim to know. +type stagedExtract struct { + path string + stamp int64 + stamped bool +} + +// stagingStamp reads back the creation time extractBundle put in a staging name. +func stagingStamp(name string) (int64, bool) { + digits, _, found := strings.Cut(strings.TrimPrefix(name, stagingPrefix), "-") + if !found { + return 0, false + } + stamp, err := strconv.ParseInt(digits, 10, 64) + if err != nil { + return 0, false + } + return stamp, true +} + // restoreStagedBackup puts a staged backup back if its link has no live tree. // It reports whether staging was dealt with and needs no further handling. -func restoreStagedBackup(dir, staging string, logf func(string, ...any)) bool { +// restored carries the links this recovery pass has already put a tree back on. +func restoreStagedBackup(dir string, s stagedExtract, restored map[string]stagedExtract, logf func(string, ...any)) bool { + staging := s.path backup := filepath.Join(staging, "backup") if _, err := os.Stat(backup); err != nil { return false @@ -364,13 +398,18 @@ func restoreStagedBackup(dir, staging string, logf func(string, ...any)) bool { return true } defer release() - if destInfo, err := os.Stat(dest); err == nil { - // The link already has a tree. Only drop the backup when it is provably - // the older copy; otherwise it may be the newer one a restart has not - // published yet, and deleting it would lose that work. - backupInfo, statErr := os.Stat(backup) - if statErr != nil || !backupInfo.ModTime().Before(destInfo.ModTime()) { - logf("remote: staged tree in %s is not older than the live tree for %s; leaving it in place", staging, id) + if _, err := os.Stat(dest); err == nil { + // The link already has a tree, and a backup only ever holds the tree that + // was live BEFORE it: backup is filled by renaming dest aside, so dest + // holding anything at all means a later extract published over it. That + // is what makes the backup superseded -- not a timestamp comparison, + // which two directories can tie on and which tracks a tree's contents + // rather than when it was promoted. + if from, ours := restored[dest]; ours && !(s.stamped && from.stamped && s.stamp < from.stamp) { + // dest is a tree THIS pass put back, so nothing published over this + // backup and the reasoning above does not apply. Without an order + // both names agree on, which of the two is current is unknown. + logf("remote: staged tree in %s cannot be ordered against the tree just restored for %s; leaving it in place", staging, id) return true } if err := os.RemoveAll(staging); err != nil { @@ -382,6 +421,7 @@ func restoreStagedBackup(dir, staging string, logf func(string, ...any)) bool { logf("remote: could not restore the staged tree for %s from %s: %v", id, staging, err) return true } + restored[dest] = s logf("remote: restored the work tree for %s from %s after an interrupted extract", id, staging) if err := os.RemoveAll(staging); err != nil { logf("remote: could not remove staging dir %s after restoring %s: %v", staging, id, err) @@ -413,7 +453,9 @@ func extractBundle(ctx context.Context, bundleFile, dest string, logf func(strin } defer unlockFile() - staging, err := os.MkdirTemp(parent, stagingPrefix+"*") + // The stamp records this extract's place in the order, which is what lets + // recovery tell an older leftover staging dir from a newer one. + staging, err := os.MkdirTemp(parent, fmt.Sprintf("%s%020d-*", stagingPrefix, time.Now().UnixNano())) if err != nil { return err } diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index c8e36bfd4..c4d9975aa 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -746,9 +746,14 @@ func TestRecoverBundleDirLeavesALiveExtractAlone(t *testing.T) { } } -// stageBackup plants a staging dir holding a backup tree for linkID. -func stageBackup(t *testing.T, dir, name, linkID, content string, age time.Duration) string { +// stageBackup plants a staging dir holding a backup tree for linkID, named the +// way extractBundle names one so recovery can order it. A stamp of 0 plants an +// unstamped name, which is the shape recovery must refuse to order. +func stageBackup(t *testing.T, dir, name, linkID, content string, stamp int64) string { t.Helper() + if stamp > 0 { + name = fmt.Sprintf("%020d-%s", stamp, name) + } staging := filepath.Join(dir, stagingPrefix+name) backup := filepath.Join(staging, "backup") if err := os.MkdirAll(filepath.Join(backup, ".git"), 0o700); err != nil { @@ -760,12 +765,6 @@ func stageBackup(t *testing.T, dir, name, linkID, content string, age time.Durat if err := os.WriteFile(filepath.Join(staging, stagingLinkFile), []byte(linkID), 0o600); err != nil { t.Fatal(err) } - if age > 0 { - when := time.Now().Add(-age) - if err := os.Chtimes(backup, when, when); err != nil { - t.Fatal(err) - } - } return staging } @@ -775,8 +774,9 @@ func stageBackup(t *testing.T, dir, name, linkID, content string, age time.Durat // just because directory order put the older one first. func TestRecoverBundleDirRestoresTheNewestOfSeveralBackups(t *testing.T) { dir := t.TempDir() - stageBackup(t, dir, "aaa-old", "proj-1", "v0", time.Hour) - newer := stageBackup(t, dir, "zzz-new", "proj-1", "v1", 0) + // Lexically first, so directory order alone would pick the older tree. + stageBackup(t, dir, "aaa-old", "proj-1", "v0", 100) + newer := stageBackup(t, dir, "zzz-new", "proj-1", "v1", 200) recoverBundleDir(dir, nil) @@ -792,27 +792,48 @@ func TestRecoverBundleDirRestoresTheNewestOfSeveralBackups(t *testing.T) { } } -// A backup that is not provably older than the live tree may be the newer copy -// a restart has not published yet, so it is kept rather than dropped. -func TestRecoverBundleDirKeepsABackupNewerThanTheLiveTree(t *testing.T) { +// Dropping a backup rests on the live tree having been published over it, which +// is not true of a tree recovery itself just put back. A second backup that +// carries no order recovery can compare against that restore may be the newer +// copy, so it is kept rather than dropped. +func TestRecoverBundleDirKeepsAnUnorderableBackupAgainstATreeItRestored(t *testing.T) { dir := t.TempDir() - live := filepath.Join(dir, "proj-1") - if err := os.MkdirAll(live, 0o700); err != nil { - t.Fatal(err) + stamped := stageBackup(t, dir, "current", "proj-1", "v1", 200) + unordered := stageBackup(t, dir, "leftover", "proj-1", "v2", 0) + + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }) + + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("the stamped backup should be restored: got %q err %v", got, err) } - if err := os.WriteFile(filepath.Join(live, "a.txt"), []byte("live"), 0o644); err != nil { - t.Fatal(err) + if _, err := os.Stat(filepath.Join(stamped, "backup")); !os.IsNotExist(err) { + t.Errorf("the restored staging dir should be cleared, got %v", err) } - old := time.Now().Add(-time.Hour) - if err := os.Chtimes(live, old, old); err != nil { - t.Fatal(err) + if _, err := os.Stat(filepath.Join(unordered, "backup", "a.txt")); err != nil { + t.Errorf("a backup that cannot be ordered against the restore must be kept: %v", err) } - staging := stageBackup(t, dir, "newer", "proj-1", "v2", 0) + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "cannot be ordered") }) { + t.Errorf("keeping an unorderable backup should be reported, got %v", logged) + } +} + +// An older backup IS dropped once the tree restored over it is provably newer, +// so the fail-safe above does not turn into a leak of every leftover. +func TestRecoverBundleDirDropsAnOlderBackupAfterRestoringANewerOne(t *testing.T) { + dir := t.TempDir() + stale := stageBackup(t, dir, "stale", "proj-1", "v0", 100) + stageBackup(t, dir, "current", "proj-1", "v1", 200) recoverBundleDir(dir, nil) - if _, err := os.Stat(filepath.Join(staging, "backup", "a.txt")); err != nil { - t.Errorf("a backup newer than the live tree must be kept: %v", err) + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("the newest backup should be restored: got %q err %v", got, err) + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Errorf("the superseded backup should be removed, got %v", err) } } @@ -839,3 +860,37 @@ func TestRecoverBundleDirKeepsAWorkTreePublishedUnderAReservedName(t *testing.T) t.Errorf("a published work tree was reaped on upgrade: %v", err) } } + +// A coarse-resolution filesystem can stamp the backup and the live tree with the +// same directory mtime, which is why recovery must not decide staleness from +// timestamps at all: a backup only ever holds the tree that was live BEFORE the +// one at dest, so a link that has a tree supersedes it either way. +func TestRecoverBundleDirDropsBackupWhenMtimesAreEqual(t *testing.T) { + dir := t.TempDir() + staging := plantInterruptedExtract(t, dir, "proj-1", "stale") + live := filepath.Join(dir, "proj-1") + if err := os.MkdirAll(live, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(live, "a.txt"), []byte("live"), 0o644); err != nil { + t.Fatal(err) + } + // What a one-second-granularity filesystem produces for a backup and a + // publish that happen in the same second. + tie := time.Unix(1700000000, 0) + for _, path := range []string{filepath.Join(staging, "backup"), live} { + if err := os.Chtimes(path, tie, tie); err != nil { + t.Fatal(err) + } + } + + recoverBundleDir(dir, nil) + + got, err := os.ReadFile(filepath.Join(live, "a.txt")) + if err != nil || string(got) != "live" { + t.Fatalf("the live tree must win: got %q err %v", got, err) + } + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Errorf("a superseded backup should be removed, got %v", err) + } +} From 46d12d8a46c863aeb0c10a91c2aeaed665496b39 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:20:45 -0500 Subject: [PATCH 10/34] fix(dictation): find holders by prefix rather than by glob pattern destDir is a path, not a pattern, and filepath.Glob reads it as one. A '[' anywhere in the install root opens a character class, the pattern then matches nothing, and recovery quietly leaves the interrupted install stranded: the same outcome as having no recovery at all, for a user whose config directory happens to contain a bracket. Scanning the parent for the name prefix has no such reading, and is what the bundle side already does. The end-to-end test covers both consumers by driving the real promotion into the state a killed process leaves, then recovering it with no network to fall back on. It also asserts the holder name promoteStagedDir wrote is one holderStamp can read: restoring a lone holder works either way, so nothing else would notice the two halves drifting apart until a second holder appeared. --- internal/dictation/download.go | 13 ++- internal/dictation/download_test.go | 158 ++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 1 deletion(-) diff --git a/internal/dictation/download.go b/internal/dictation/download.go index 19032afcd..f369dc3de 100644 --- a/internal/dictation/download.go +++ b/internal/dictation/download.go @@ -796,10 +796,21 @@ func restoreInterruptedPromotion(destDir string) { if _, err := os.Lstat(destDir); err == nil { return } - holders, err := filepath.Glob(destDir + holderSuffix + "*") + // ReadDir and a prefix rather than filepath.Glob: destDir is a real path, + // and a '[' anywhere in it opens a character class to Glob, which then + // matches nothing and strands the install this exists to put back. + parent := filepath.Dir(destDir) + entries, err := os.ReadDir(parent) if err != nil { return } + prefix := filepath.Base(destDir) + holderSuffix + holders := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() && strings.HasPrefix(entry.Name(), prefix) { + holders = append(holders, filepath.Join(parent, entry.Name())) + } + } // Newest first: an unstamped holder is the least recent thing we can claim // to know about, so it is only reached once every stamped one has failed. slices.SortStableFunc(holders, func(a, b string) int { diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index 395a43bc4..c1931994c 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -524,3 +524,161 @@ func TestRestoreInterruptedPromotionPrefersTheNewestHolder(t *testing.T) { t.Errorf("the older holder must be left intact: %v", err) } } + +// interruptPromotion drives the REAL promotion into the state a process killed +// between its two renames leaves: destDir absent, the only install in a holder +// promoteStagedDir named. Both renames fail, so nothing is put back in process +// and the holder is retained rather than cleaned up. +func interruptPromotion(t *testing.T, destDir, label string) { + t.Helper() + stage := destDir + ".incoming" + if err := os.MkdirAll(stage, 0o755); err != nil { + t.Fatal(err) + } + real := renameStagedDir + renameStagedDir = func(string, string) error { return errors.New("injected rename failure") } + err := promoteStagedDir(stage, destDir, label) + renameStagedDir = real + if err == nil { + t.Fatalf("a promotion whose publish and restore both fail must report an error") + } + if _, statErr := os.Lstat(destDir); !os.IsNotExist(statErr) { + t.Fatalf("the interrupted promotion should leave %s absent, got %v", destDir, statErr) + } +} + +// holdersFor lists the holders promoteStagedDir left beside destDir. +func holdersFor(t *testing.T, destDir string) []string { + t.Helper() + holders, err := filepath.Glob(destDir + holderSuffix + "*") + if err != nil { + t.Fatal(err) + } + return holders +} + +// End to end over the real transaction, for BOTH consumers of it. The engine and +// the model are installed for real, one of them is interrupted mid-promotion by +// the real promoteStagedDir, and the next start has to put it back with no +// network to fall back on. Nothing plants a holder by hand, so the name recovery +// orders by is the one promoteStagedDir writes -- the half a hand-built fixture +// cannot check. +func TestEnsureLocalEngineRecoversARealInterruptedPromotionOffline(t *testing.T) { + for _, tc := range []struct { + name string + label string + // dirFor picks the install this case interrupts, given a finished setup. + dirFor func(t *testing.T, dest string, comp EngineComponents) string + }{ + { + name: "engine", + label: "Engine", + dirFor: func(t *testing.T, dest string, comp EngineComponents) string { + t.Helper() + matches, err := filepath.Glob(filepath.Join(dest, "engine-*")) + if err != nil || len(matches) != 1 { + t.Fatalf("want exactly one engine dir under %s, got %v (err %v)", dest, matches, err) + } + return matches[0] + }, + }, + { + name: "model", + label: "Model", + dirFor: func(t *testing.T, dest string, comp EngineComponents) string { + t.Helper() + return filepath.Dir(comp.ModelPath) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + srv := fakeReleaseServer(t, engineSHA, modelSHA) + dest := t.TempDir() + comp, err := EnsureLocalEngine(context.Background(), DownloadOptions{ + DestRoot: dest, EngineVersion: "test", APIBase: srv.URL, platformKey: "linux-amd64", skipPinned: true, + }) + if err != nil { + t.Fatalf("seeding the install: %v", err) + } + + target := tc.dirFor(t, dest, comp) + interruptPromotion(t, target, tc.label) + holders := holdersFor(t, target) + if len(holders) != 1 { + t.Fatalf("want exactly one holder beside %s, got %v", target, holders) + } + // The name promoteStagedDir wrote must be one recovery can order by. + // Restoring a lone holder works either way, so without this the two + // halves could drift apart and only a second holder would show it. + if _, ok := holderStamp(target, holders[0]); !ok { + t.Fatalf("recovery cannot order the name promoteStagedDir wrote: %q", holders[0]) + } + + // No network: if the holder is not found there is nothing to fall + // back on, which is the failure the offline user actually sees. + got, err := EnsureLocalEngine(context.Background(), DownloadOptions{ + DestRoot: dest, EngineVersion: "test", APIBase: offlineAPIBase(t), platformKey: "linux-amd64", skipPinned: true, + }) + if err != nil { + t.Fatalf("the interrupted %s promotion was not recovered offline: %v", tc.name, err) + } + if !fileExists(got.BinaryPath) { + t.Errorf("engine binary missing after recovery: %q", got.BinaryPath) + } + if !fileExists(filepath.Join(got.ModelPath, "tokens.txt")) { + t.Errorf("model tokens.txt missing after recovery under %q", got.ModelPath) + } + if holders := holdersFor(t, target); len(holders) != 0 { + t.Errorf("the holder should be cleared after recovery, got %v", holders) + } + }) + } +} + +// A path is not a pattern. An install root containing a glob metacharacter -- a +// '[' is the one that silently matches nothing -- must not cost a user the +// install recovery exists to put back. +func TestRestoreInterruptedPromotionFindsHoldersUnderAnAwkwardPath(t *testing.T) { + for _, dirName := range []string{"plain", "wei[rd", "sta*r", "que?ry", "br]ack"} { + t.Run(dirName, func(t *testing.T) { + root := filepath.Join(t.TempDir(), dirName) + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + install := filepath.Join(plantHolder(t, dest, 100, "kept"), "install") + if _, err := os.Stat(install); err != nil { + t.Fatal(err) + } + + restoreInterruptedPromotion(dest) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "kept" { + t.Fatalf("the install was not restored under %q: got %q err %v", dirName, got, err) + } + }) + } +} + +// A holder can be there without an install in it: the promotion creates the +// holder first, so a stop before the rename leaves an empty one. Recovery must +// step over it and keep looking rather than treating it as the newest word on +// what to restore. +func TestRestoreInterruptedPromotionSkipsAHolderWithNoInstall(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + usable := plantHolder(t, dest, 100, "kept") + // Newer, so ordering reaches it first, but it holds nothing. + empty := fmt.Sprintf("%s%s%020d-x", dest, holderSuffix, 200) + if err := os.MkdirAll(empty, 0o755); err != nil { + t.Fatal(err) + } + + restoreInterruptedPromotion(dest) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "kept" { + t.Fatalf("the usable holder should be restored: got %q err %v", got, err) + } + if _, err := os.Stat(usable); !os.IsNotExist(err) { + t.Errorf("the restored holder should be cleared, got %v", err) + } +} From 572dc1fbbca53581a95666aa154b225b52357560 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:20:45 -0500 Subject: [PATCH 11/34] test(daemon/remote): cover bundle recovery end to end The recovery tests all planted their fixtures by hand, so none of them ran a name extractBundle actually writes. The new end-to-end test uploads over a real bridge, interrupts the swap the way a killed daemon interrupts it, and starts a fresh bridge over the same directory, asserting on the way through that the staging name recovery has to order by is the one the extract wrote. Also covers what recovery must not do: decide one link by another link's outcome, change anything on a second pass, or tell two backups stamped in the same instant apart. The reap path now runs against both name shapes. --- internal/daemon/remote/bundle_test.go | 194 ++++++++++++++++++++++++-- 1 file changed, 182 insertions(+), 12 deletions(-) diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index c4d9975aa..49ee17f46 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -550,20 +550,26 @@ func TestRecoverBundleDirLeavesAStagingDirAnExtractCouldStillOwn(t *testing.T) { } func TestRecoverBundleDirReapsAbandonedStaging(t *testing.T) { - dir := t.TempDir() - staging := filepath.Join(dir, stagingPrefix+"old") - if err := os.MkdirAll(filepath.Join(staging, "repo"), 0o700); err != nil { - t.Fatal(err) - } - old := time.Now().Add(-3 * gitTimeout) - if err := os.Chtimes(staging, old, old); err != nil { - t.Fatal(err) - } + // Both name shapes: the stamped one is what extractBundle writes now, the + // bare one is any staging dir whose name the reaper cannot read. + for _, name := range []string{"old", "00000000000000000100-old"} { + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + staging := filepath.Join(dir, stagingPrefix+name) + if err := os.MkdirAll(filepath.Join(staging, "repo"), 0o700); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-3 * gitTimeout) + if err := os.Chtimes(staging, old, old); err != nil { + t.Fatal(err) + } - recoverBundleDir(dir, nil) + recoverBundleDir(dir, nil) - if _, err := os.Stat(staging); !os.IsNotExist(err) { - t.Errorf("a staging dir older than any clone should be reaped, got %v", err) + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Errorf("a staging dir older than any clone should be reaped, got %v", err) + } + }) } } @@ -894,3 +900,167 @@ func TestRecoverBundleDirDropsBackupWhenMtimesAreEqual(t *testing.T) { t.Errorf("a superseded backup should be removed, got %v", err) } } + +// End to end over the real transaction. A real upload publishes a tree, a second +// upload is interrupted exactly the way a killed daemon interrupts it (the +// publish rename and the restore behind it both fail, so the prior tree is left +// in staging), and a fresh bridge on the same directory puts it back. Nothing +// here plants a fixture: the staging name recovery has to order by is the one +// extractBundle writes, which is the half a hand-built fixture cannot check. +func TestBridgeRecoversARealInterruptedExtractOnStart(t *testing.T) { + srv := newBridgeServer(t, staticLauncher()) + auth, _ := NewTokenAuthenticator("tok") + bundleRoot := t.TempDir() + addr, ca := startBridge(t, srv, BridgeOptions{Authenticator: auth, BundleDir: bundleRoot}) + cfg := RemoteConfig{Address: addr, Token: "tok", CACertFile: ca} + dest := filepath.Join(bundleRoot, "proj-1") + + if _, err := UploadRepoBundle(cfg, initTestRepo(t, "a.txt", "v1"), "proj-1"); err != nil { + t.Fatalf("first upload: %v", err) + } + if got, err := os.ReadFile(filepath.Join(dest, "a.txt")); err != nil || string(got) != "v1" { + t.Fatalf("first upload did not publish: got %q err %v", got, err) + } + + // Both renames fail, which is the one path that leaves the tree in staging + // with dest absent -- what a process killed between the two renames leaves. + real := renameDir + renameDir = func(string, string) error { return errors.New("injected rename failure") } + _, err := UploadRepoBundle(cfg, initTestRepo(t, "a.txt", "v2"), "proj-1") + renameDir = real + if err == nil { + t.Fatal("an upload whose publish and restore both fail must report an error") + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatalf("the interrupted swap should leave dest absent, got %v", err) + } + + // The name extractBundle wrote must be one recovery can order by. If the + // writer and the reader ever disagree on the format, this is where it shows. + entries, err := os.ReadDir(bundleRoot) + if err != nil { + t.Fatal(err) + } + stagedNames := []string{} + for _, e := range entries { + if strings.HasPrefix(e.Name(), stagingPrefix) { + stagedNames = append(stagedNames, e.Name()) + } + } + if len(stagedNames) != 1 { + t.Fatalf("want exactly one staging dir left behind, got %v", stagedNames) + } + if _, ok := stagingStamp(stagedNames[0]); !ok { + t.Fatalf("recovery cannot order the name extractBundle wrote: %q", stagedNames[0]) + } + + // A fresh daemon over the same directory repairs it before serving. + addr2, ca2 := startBridge(t, newBridgeServer(t, staticLauncher()), BridgeOptions{Authenticator: auth, BundleDir: bundleRoot}) + + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("the interrupted extract was not restored: got %q err %v", got, err) + } + for _, name := range stagedNames { + if _, err := os.Stat(filepath.Join(bundleRoot, name)); !os.IsNotExist(err) { + t.Errorf("staging %s should be cleared after recovery, got %v", name, err) + } + } + // And the repaired link still serves. + if _, err := UploadRepoBundle(RemoteConfig{Address: addr2, Token: "tok", CACertFile: ca2}, initTestRepo(t, "a.txt", "v3"), "proj-1"); err != nil { + t.Fatalf("upload to a recovered link: %v", err) + } + if got, err := os.ReadFile(filepath.Join(dest, "a.txt")); err != nil || string(got) != "v3" { + t.Fatalf("after re-upload a.txt = %q, err %v, want %q", got, err, "v3") + } +} + +// Recovery decides each link on its own: restoring one link's tree must not make +// another link's superseded backup look unorderable, and vice versa. +func TestRecoverBundleDirHandlesLinksIndependently(t *testing.T) { + dir := t.TempDir() + // proj-1 has no tree, so its backup is restored. + restorable := stageBackup(t, dir, "one", "proj-1", "v1", 100) + // proj-2 has a live tree, so its backup was published over and is dropped. + superseded := stageBackup(t, dir, "two", "proj-2", "old", 200) + live := filepath.Join(dir, "proj-2") + if err := os.MkdirAll(live, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(live, "a.txt"), []byte("live"), 0o644); err != nil { + t.Fatal(err) + } + + recoverBundleDir(dir, nil) + + if got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")); err != nil || string(got) != "v1" { + t.Errorf("proj-1 should be restored: got %q err %v", got, err) + } + if _, err := os.Stat(restorable); !os.IsNotExist(err) { + t.Errorf("proj-1 staging should be cleared, got %v", err) + } + if got, err := os.ReadFile(filepath.Join(live, "a.txt")); err != nil || string(got) != "live" { + t.Errorf("proj-2's live tree must win: got %q err %v", got, err) + } + if _, err := os.Stat(superseded); !os.IsNotExist(err) { + t.Errorf("proj-2's superseded staging should be removed, got %v", err) + } +} + +// Recovery runs on every start, so a second pass over an already-repaired +// directory must be a no-op rather than treating the tree it restored last time +// as something to move again. +func TestRecoverBundleDirIsIdempotent(t *testing.T) { + dir := t.TempDir() + stageBackup(t, dir, "one", "proj-1", "v1", 100) + + recoverBundleDir(dir, nil) + first, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + recoverBundleDir(dir, nil) + second, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + if got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")); err != nil || string(got) != "v1" { + t.Fatalf("the restored tree must survive a second pass: got %q err %v", got, err) + } + names := func(es []os.DirEntry) []string { + out := []string{} + for _, e := range es { + out = append(out, e.Name()) + } + return out + } + if !slices.Equal(names(first), names(second)) { + t.Errorf("a second recovery pass changed the directory: %v then %v", names(first), names(second)) + } +} + +// Two extracts can stamp the same instant. Recovery restores one of them and +// must then keep the other rather than deleting it as superseded: equal stamps +// are not evidence that one came first. +func TestRecoverBundleDirKeepsABackupItCannotTellApartFromTheRestore(t *testing.T) { + dir := t.TempDir() + first := stageBackup(t, dir, "one", "proj-1", "v1", 100) + second := stageBackup(t, dir, "two", "proj-1", "v2", 100) + + recoverBundleDir(dir, nil) + + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil { + t.Fatalf("nothing was restored: %v", err) + } + kept := 0 + for _, staging := range []string{first, second} { + if _, err := os.Stat(filepath.Join(staging, "backup", "a.txt")); err == nil { + kept++ + } + } + if kept != 1 { + t.Fatalf("restored %q and kept %d of the two tied backups, want exactly 1 kept", got, kept) + } +} From 3974d58acb6b2439a667df0984e830fe0a5b41cf Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:59:04 -0500 Subject: [PATCH 12/34] test(dictation): keep the awkward-path names to ones Windows can hold The awkward-path case builds a directory per glob metacharacter, but '*' and '?' are illegal in a Windows filename, so the two subtests died in their own os.MkdirAll before reaching restoreInterruptedPromotion. Run those two off Windows only; the bracket names are legal everywhere and still cover '[', the metacharacter that matches nothing rather than failing. Also apply De Morgan's law to the restored-backup check that staticcheck flagged (QF1001). Same predicate, checked over all 36 input combinations. --- internal/daemon/remote/bundle.go | 2 +- internal/dictation/download_test.go | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index 8e7b6acb6..296ffe6f1 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -405,7 +405,7 @@ func restoreStagedBackup(dir string, s stagedExtract, restored map[string]staged // is what makes the backup superseded -- not a timestamp comparison, // which two directories can tie on and which tracks a tree's contents // rather than when it was promoted. - if from, ours := restored[dest]; ours && !(s.stamped && from.stamped && s.stamp < from.stamp) { + if from, ours := restored[dest]; ours && (!s.stamped || !from.stamped || s.stamp >= from.stamp) { // dest is a tree THIS pass put back, so nothing published over this // backup and the reasoning above does not apply. Without an order // both names agree on, which of the two is current is unknown. diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index c1931994c..6db10ed17 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -639,7 +639,14 @@ func TestEnsureLocalEngineRecoversARealInterruptedPromotionOffline(t *testing.T) // '[' is the one that silently matches nothing -- must not cost a user the // install recovery exists to put back. func TestRestoreInterruptedPromotionFindsHoldersUnderAnAwkwardPath(t *testing.T) { - for _, dirName := range []string{"plain", "wei[rd", "sta*r", "que?ry", "br]ack"} { + dirNames := []string{"plain", "wei[rd", "br]ack"} + if runtime.GOOS != "windows" { + // Windows refuses these two in a filename outright, so there is no such + // path to defend there. The bracket cases above still cover the + // metacharacter that matches nothing instead of failing. + dirNames = append(dirNames, "sta*r", "que?ry") + } + for _, dirName := range dirNames { t.Run(dirName, func(t *testing.T) { root := filepath.Join(t.TempDir(), dirName) dest := filepath.Join(root, "engine-1.2.3-linux-x64") From 61f5816111f0eb83088514f6abdbd01125fef4fd Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 31 Aug 2026 22:08:00 -0500 Subject: [PATCH 13/34] fix(daemon/remote): keep a retained backup across the next recovery pass Recovery keeps a staged backup it cannot order against a tree it just put back, because neither name says which of the two is current. That fail-safe only held for one pass: the map recording what this pass restored is per-call, so on the next start dest looks published-over by a later extract and the retained copy was reaped as superseded. Park a retained staging dir under a prefix the scan does not enumerate, so a later pass leaves it alone. The rename is not forced, so an occupied name keeps its occupant and the copy simply stays put. Reported by CodeRabbit on #993. --- internal/daemon/remote/bundle.go | 24 +++++- internal/daemon/remote/bundle_test.go | 101 +++++++++++++++++++++++++- 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index 296ffe6f1..adfaaaf5f 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -179,6 +179,14 @@ func streamFramesToFile(r io.Reader, w io.Writer, size int64) error { // sanitizeLinkID refuses every dot-prefixed id so a link can never name one. const stagingPrefix = ".staging-" +// keptPrefix names a staged backup that recovery decided to keep rather than +// order against a tree it had just put back. Recovery runs again on every start, +// and by then it can no longer tell that tree from one a later extract +// published, so the copy is parked under a name the scan does not enumerate. +// sanitizeLinkID refuses every dot-prefixed id, so this can never take a link's +// name. +const keptPrefix = ".kept-" + // lockDirName holds the per-link advisory lock files that serialize extracts // across processes. Dot-prefixed for the same reason stagingPrefix is. const lockDirName = ".extract-locks" @@ -409,7 +417,8 @@ func restoreStagedBackup(dir string, s stagedExtract, restored map[string]staged // dest is a tree THIS pass put back, so nothing published over this // backup and the reasoning above does not apply. Without an order // both names agree on, which of the two is current is unknown. - logf("remote: staged tree in %s cannot be ordered against the tree just restored for %s; leaving it in place", staging, id) + logf("remote: staged tree in %s cannot be ordered against the tree just restored for %s; keeping it", staging, id) + parkKeptBackup(dir, staging, id, logf) return true } if err := os.RemoveAll(staging); err != nil { @@ -429,6 +438,19 @@ func restoreStagedBackup(dir string, s stagedExtract, restored map[string]staged return true } +// parkKeptBackup moves a staging dir out of the prefix recovery scans, so the +// next pass leaves alone what this one deliberately kept instead of reading the +// restored tree at dest as a later extract publishing over it. A rename onto an +// existing directory fails rather than replacing it, so a legacy link that +// happens to carry the parked name is never clobbered; the copy just stays where +// it is, which is the same fail-safe one pass later. +func parkKeptBackup(dir, staging, id string, logf func(string, ...any)) { + parked := filepath.Join(dir, keptPrefix+strings.TrimPrefix(filepath.Base(staging), stagingPrefix)) + if err := os.Rename(staging, parked); err != nil { + logf("remote: could not park the kept backup for %s at %s: %v", id, parked, err) + } +} + // extractBundle clones bundleFile into a staging dir beside dest, then swaps the // clone into place (replacing any prior extraction for this link id). git clone // needs a non-existent target, hence the staging dir. The live tree is moved diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index 49ee17f46..df102732f 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -802,6 +802,13 @@ func TestRecoverBundleDirRestoresTheNewestOfSeveralBackups(t *testing.T) { // is not true of a tree recovery itself just put back. A second backup that // carries no order recovery can compare against that restore may be the newer // copy, so it is kept rather than dropped. +// parkedStaging is where recovery moves a staging dir it kept, so a later pass +// does not read the restored tree at dest as a later extract publishing over it. +func parkedStaging(staging string) string { + base := filepath.Base(staging) + return filepath.Join(filepath.Dir(staging), keptPrefix+strings.TrimPrefix(base, stagingPrefix)) +} + func TestRecoverBundleDirKeepsAnUnorderableBackupAgainstATreeItRestored(t *testing.T) { dir := t.TempDir() stamped := stageBackup(t, dir, "current", "proj-1", "v1", 200) @@ -817,14 +824,100 @@ func TestRecoverBundleDirKeepsAnUnorderableBackupAgainstATreeItRestored(t *testi if _, err := os.Stat(filepath.Join(stamped, "backup")); !os.IsNotExist(err) { t.Errorf("the restored staging dir should be cleared, got %v", err) } - if _, err := os.Stat(filepath.Join(unordered, "backup", "a.txt")); err != nil { + if _, err := os.Stat(filepath.Join(parkedStaging(unordered), "backup", "a.txt")); err != nil { t.Errorf("a backup that cannot be ordered against the restore must be kept: %v", err) } + if _, err := os.Stat(unordered); !os.IsNotExist(err) { + t.Errorf("the kept backup should be parked out of the scanned prefix, got %v", err) + } if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "cannot be ordered") }) { t.Errorf("keeping an unorderable backup should be reported, got %v", logged) } } +// Recovery runs on every start. A backup the pass above deliberately kept must +// still be there after the next one, where the restore is already at dest and +// nothing records that this pass, not a later extract, put it there. +func TestRecoverBundleDirKeepsAnUnorderableBackupAcrossRestarts(t *testing.T) { + dir := t.TempDir() + stageBackup(t, dir, "current", "proj-1", "v1", 200) + unordered := stageBackup(t, dir, "leftover", "proj-1", "v2", 0) + + recoverBundleDir(dir, nil) + if _, err := os.Stat(filepath.Join(parkedStaging(unordered), "backup", "a.txt")); err != nil { + t.Fatalf("the first pass should keep the unorderable backup: %v", err) + } + + // The daemon restarts: same dir, a fresh recovery pass with no memory of the + // first. + recoverBundleDir(dir, nil) + + if !keptBackupSurvives(t, dir, "v2") { + t.Error("a backup kept as unorderable must survive the next recovery pass") + } + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil || string(got) != "v1" { + t.Errorf("the restored tree must be left alone across restarts: got %q err %v", got, err) + } + + // And it must not be re-restored or re-parked on every start after that. + recoverBundleDir(dir, nil) + if !keptBackupSurvives(t, dir, "v2") { + t.Error("the kept backup must survive a third pass too") + } +} + +// keptBackupSurvives reports whether a backup holding content is still somewhere +// under dir, wherever recovery decided to keep it. +func keptBackupSurvives(t *testing.T, dir, content string) bool { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + got, err := os.ReadFile(filepath.Join(dir, entry.Name(), "backup", "a.txt")) + if err == nil && string(got) == content { + return true + } + } + return false +} + +// Parking is a rename, and a rename onto a directory that already holds +// something must fail rather than replace it. Recovery then leaves the copy +// where it is, which is still a copy, and never eats the occupant. +func TestRecoverBundleDirDoesNotClobberAnOccupiedParkedName(t *testing.T) { + dir := t.TempDir() + stageBackup(t, dir, "current", "proj-1", "v1", 200) + unordered := stageBackup(t, dir, "leftover", "proj-1", "v2", 0) + + occupied := parkedStaging(unordered) + if err := os.MkdirAll(occupied, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(occupied, "keep.txt"), []byte("not mine"), 0o600); err != nil { + t.Fatal(err) + } + + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }) + + got, err := os.ReadFile(filepath.Join(occupied, "keep.txt")) + if err != nil || string(got) != "not mine" { + t.Errorf("the occupant of the parked name must be untouched: got %q err %v", got, err) + } + if _, err := os.Stat(filepath.Join(unordered, "backup", "a.txt")); err != nil { + t.Errorf("a backup that could not be parked must stay where it is: %v", err) + } + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "could not park") }) { + t.Errorf("a failed park should be reported, got %v", logged) + } +} + // An older backup IS dropped once the tree restored over it is provably newer, // so the fail-safe above does not turn into a leak of every leftover. func TestRecoverBundleDirDropsAnOlderBackupAfterRestoringANewerOne(t *testing.T) { @@ -1056,8 +1149,10 @@ func TestRecoverBundleDirKeepsABackupItCannotTellApartFromTheRestore(t *testing. } kept := 0 for _, staging := range []string{first, second} { - if _, err := os.Stat(filepath.Join(staging, "backup", "a.txt")); err == nil { - kept++ + for _, at := range []string{staging, parkedStaging(staging)} { + if _, err := os.Stat(filepath.Join(at, "backup", "a.txt")); err == nil { + kept++ + } } } if kept != 1 { From 2aac6e4b7b68edf4fe6933ee3e2f37379e47da0e Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 31 Aug 2026 22:29:18 -0500 Subject: [PATCH 14/34] test(dictation): cover a stamped holder winning over an unstamped one The recency sort claims an unstamped holder is the least recent thing recovery can read, so it loses to any stamped one. Only the newest-of-two-stamped half of that was covered: inverting the stamped/unstamped branch left the suite green. The name sorts first lexically, so nothing but the rule under test can produce the wanted answer. --- internal/dictation/download_test.go | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index 6db10ed17..27871a9e7 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -665,6 +665,39 @@ func TestRestoreInterruptedPromotionFindsHoldersUnderAnAwkwardPath(t *testing.T) } } +// A holder this package did not name carries no ordering anyone can read, so it +// is the least recent thing recovery can claim to know about and must lose to +// any stamped holder, however old that one's stamp is. +func TestRestoreInterruptedPromotionPrefersAStampedHolderOverAnUnstampedOne(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + // The unstamped name sorts after the stamped one lexically, so a pass that + // ignored the stamp entirely would still get this right; give it a name that + // sorts FIRST, so only the stamped-wins rule can produce the wanted answer. + unstamped := dest + holderSuffix + "aaa" + if err := os.MkdirAll(filepath.Join(unstamped, "install"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(unstamped, "install", "engine"), []byte("unstamped"), 0o644); err != nil { + t.Fatal(err) + } + stamped := plantHolder(t, dest, 100, "stamped") + + restoreInterruptedPromotion(dest) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "stamped" { + t.Fatalf("a stamped holder must win over an unstamped one: got %q err %v", got, err) + } + if _, err := os.Stat(stamped); !os.IsNotExist(err) { + t.Errorf("the restored holder should be cleared, got %v", err) + } + // The one recovery did not use is left for a human, never deleted on a guess. + if _, err := os.Stat(filepath.Join(unstamped, "install", "engine")); err != nil { + t.Errorf("the unused holder must be kept: %v", err) + } +} + // A holder can be there without an install in it: the promotion creates the // holder first, so a stop before the rename leaves an empty one. Recovery must // step over it and keep looking rather than treating it as the newest word on From aa68c04db6a8dbf0a6332d4834560fef4cef7389 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 31 Aug 2026 23:35:13 -0500 Subject: [PATCH 15/34] fix(daemon/remote,dictation): order crash recovery by a sequence, not the clock Recovery ordered its leftover directories by a time.Now().UnixNano() stamp in the directory name. Wall-clock time is not monotonic across persistence, so a VM resume, an NTP correction, or a manual change can leave the earlier of two writes carrying the larger stamp. Recovery then restores the older tree and, at the bundle site, deletes the newer one as superseded, which loses the only copy of the work tree that was live last. The dictation site cannot delete a tree but restores a stale engine or model. Both writers now number a new directory one past the highest already present and claim it with an exclusive create, retrying upward when the name is taken. An extract or promotion that reads an existing entry always numbers above it, which no clock movement can invert, and exclusive creation arbitrates the racers that neither site fully locks. Seeding from the highest present is also the whole migration: a nanosecond name written by a released binary just sets a high starting point, so old and new names keep sorting correctly together with no upgrade step. The bundle allocator counts parked .kept- names as well as staging ones. parkKeptBackup derives the parked name from the staging name, so a number handed out twice makes the second park land on an occupied name; that rename refuses, the backup stays under the scanned prefix, and the next pass deletes it as superseded. Counting parked names keeps a retained backup retained. Directory mode stays 0o700, which is what os.MkdirTemp produced. os.Mkdir takes a mode where MkdirTemp did not, so replacing the call forces the choice. --- internal/daemon/remote/bundle.go | 112 +++++++++++- internal/daemon/remote/bundle_test.go | 251 ++++++++++++++++++++++++++ internal/dictation/download.go | 92 +++++++++- internal/dictation/download_test.go | 191 ++++++++++++++++++++ 4 files changed, 631 insertions(+), 15 deletions(-) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index adfaaaf5f..2d6922902 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -9,6 +9,8 @@ import ( "errors" "fmt" "io" + "io/fs" + "math" "net" "os" "os/exec" @@ -299,10 +301,12 @@ func recoverBundleDir(dir string, logf func(string, ...any)) { // One link can have several staged backups: a cleanup that could not finish // leaves one behind, and a later crash adds another. Newest first, so the // tree that comes back is the most recent one rather than whichever the - // directory happened to list first. The order comes from the stamp the - // extract wrote into the staging name, not from a directory mtime: mtimes - // track a tree's contents, and a coarse filesystem gives two of them the - // same value anyway. + // directory happened to list first. The order comes from the sequence the + // extract allocated against the entries already in the directory, not from a + // wall clock and not from a directory mtime. A clock can move backward and + // invert two stamps; an extract that read an existing entry always numbers + // above it. Mtimes stay rejected for their own reasons: they track a tree's + // contents, and a coarse filesystem gives two of them the same value anyway. staged := make([]stagedExtract, 0, len(entries)) for _, entry := range entries { if !entry.IsDir() || !strings.HasPrefix(entry.Name(), stagingPrefix) { @@ -357,7 +361,10 @@ type stagedExtract struct { stamped bool } -// stagingStamp reads back the creation time extractBundle put in a staging name. +// stagingStamp reads back the ordering stamp extractBundle put in a staging +// name. New names carry a per-directory sequence; names written by released +// versions carry wall-clock nanoseconds. Both are plain int64s and compare the +// same way, which is what lets one directory hold a mix of them. func stagingStamp(name string) (int64, bool) { digits, _, found := strings.Cut(strings.TrimPrefix(name, stagingPrefix), "-") if !found { @@ -370,6 +377,86 @@ func stagingStamp(name string) (int64, bool) { return stamp, true } +// stagingSeqSuffix closes a sequenced staging name. The parsers cut on the first +// '-' after the digits, so a name that ends at the digits reads back as +// unstamped, which is silent: an unstamped entry sorts last and is always +// retained, so the ordering key would simply stop existing with nothing failing. +// os.MkdirTemp used to supply this separator with its random suffix. +const stagingSeqSuffix = "-seq" + +// stagingSeqAttempts bounds the walk up from a taken number, in the spirit of +// the retry limit os.MkdirTemp applies to its own random names. +const stagingSeqAttempts = 10000 + +// nextStagingSeq is the number a new staging dir should claim: one past the +// highest already in the directory. That is what makes the order survive a +// clock that moves backward. An extract that reads an existing entry always +// allocates above it, and no clock correction can invert that, whereas the +// wall-clock stamp this replaces inverted whenever the clock did. +// +// Kept backups count. parkKeptBackup derives the parked name from the staging +// name, so a number handed out twice makes the second park land on an occupied +// name; that rename refuses, the backup stays under stagingPrefix, and the next +// pass deletes it as superseded. Counting kept names keeps a parked number out +// of circulation. Recovery still does not enumerate them. +func nextStagingSeq(dir string) (int64, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return 0, err + } + var high int64 + for _, entry := range entries { + if !entry.IsDir() { + continue + } + name := entry.Name() + if strings.HasPrefix(name, keptPrefix) { + name = stagingPrefix + strings.TrimPrefix(name, keptPrefix) + } + if stamp, ok := stagingStamp(name); ok && stamp > high { + high = stamp + } + } + if high == math.MaxInt64 { + // The addition below would wrap negative, and %020d of a negative + // renders a '-' the parser reads as empty digits, so the entry would + // drop out of the ordering without saying so. Refuse instead. + return 0, fmt.Errorf("remote: %s holds a staging name at the maximum sequence; remove it before extracting again", dir) + } + return high + 1, nil +} + +// createSequencedStagingDir claims the first free name from n upward. Exclusive +// creation is what arbitrates: two extracts racing in one directory cannot both +// win a name, and the loser walks up to a number strictly above the winner's. +func createSequencedStagingDir(dir string, n int64) (string, error) { + if n < 1 { + return "", fmt.Errorf("remote: refusing to allocate staging sequence %d", n) + } + for i := 0; i < stagingSeqAttempts; i++ { + name := fmt.Sprintf("%s%020d%s", stagingPrefix, n, stagingSeqSuffix) + // The writer and the reader agree on the format or nothing is written. + // Checking here rather than trusting the format string is what keeps a + // silently unstamped name from reaching disk. + if stamp, ok := stagingStamp(name); !ok || stamp != n { + return "", fmt.Errorf("remote: staging name %q does not read back as sequence %d", name, n) + } + path := filepath.Join(dir, name) + err := os.Mkdir(path, 0o700) + if err == nil { + return path, nil + } + if !errors.Is(err, fs.ErrExist) { + return "", err + } + if n == math.MaxInt64 { + return "", fmt.Errorf("remote: staging sequence exhausted in %s", dir) + } + n++ + } + return "", fmt.Errorf("remote: could not claim a staging name in %s after %d attempts", dir, stagingSeqAttempts) +} + // restoreStagedBackup puts a staged backup back if its link has no live tree. // It reports whether staging was dealt with and needs no further handling. // restored carries the links this recovery pass has already put a tree back on. @@ -475,9 +562,18 @@ func extractBundle(ctx context.Context, bundleFile, dest string, logf func(strin } defer unlockFile() - // The stamp records this extract's place in the order, which is what lets - // recovery tell an older leftover staging dir from a newer one. - staging, err := os.MkdirTemp(parent, fmt.Sprintf("%s%020d-*", stagingPrefix, time.Now().UnixNano())) + // The sequence records this extract's place in the order, which is what lets + // recovery tell an older leftover staging dir from a newer one. It is one + // past the highest number already in the directory, claimed by exclusive + // creation, so a concurrent extract for another link cannot take the same + // value and a clock that moves backward cannot invert the order. Numbers + // written by released versions are wall-clock nanoseconds; seeding from the + // highest present keeps those sorting older with no migration step. + seq, err := nextStagingSeq(parent) + if err != nil { + return err + } + staging, err := createSequencedStagingDir(parent, seq) if err != nil { return err } diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index df102732f..70b335997 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math" "os" "os/exec" "path/filepath" @@ -1068,6 +1069,256 @@ func TestBridgeRecoversARealInterruptedExtractOnStart(t *testing.T) { } } +// A backward clock leaves an OLDER backup carrying a LARGER stamp. Recovery must +// still restore the tree that was actually live last, and must not delete it as +// superseded on the way. +// +// Fixture order is load-bearing: NewBridge repairs the directory before it +// serves, so a backup planted before the bridge starts is consumed by that first +// pass and this test would pass without proving anything. Publish first, plant +// second. +func TestExtractBundleOutOrdersALeftoverStampedInTheFuture(t *testing.T) { + // A nanosecond stamp decades ahead, which is what a clock correction + // backward leaves behind on the earlier of two writes. + const farFuture = int64(4_000_000_000_000_000_000) + + srv := newBridgeServer(t, staticLauncher()) + auth, _ := NewTokenAuthenticator("tok") + bundleRoot := t.TempDir() + addr, ca := startBridge(t, srv, BridgeOptions{Authenticator: auth, BundleDir: bundleRoot}) + cfg := RemoteConfig{Address: addr, Token: "tok", CACertFile: ca} + dest := filepath.Join(bundleRoot, "proj-1") + + if _, err := UploadRepoBundle(cfg, initTestRepo(t, "a.txt", "v1"), "proj-1"); err != nil { + t.Fatalf("first upload: %v", err) + } + + // Only now, with the bridge already up and its recovery pass behind us. + stale := stageBackup(t, bundleRoot, "old", "proj-1", "v-old", farFuture) + + real := renameDir + renameDir = func(string, string) error { return errors.New("injected rename failure") } + _, err := UploadRepoBundle(cfg, initTestRepo(t, "a.txt", "v2"), "proj-1") + renameDir = real + if err == nil { + t.Fatal("an upload whose publish and restore both fail must report an error") + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatalf("the interrupted swap should leave dest absent, got %v", err) + } + + recoverBundleDir(bundleRoot, nil) + + // The interrupted extract set the LIVE tree aside, so v1 is what recovery + // owes back. The interrupted clone of v2 never reached dest. + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v1" { + t.Errorf("recovery restored %q (err %v), want the tree that was live last, %q", got, err, "v1") + // Distinguish "restored the wrong one" from "restored the wrong one AND + // destroyed the right one", which is the data loss this test exists for. + if !liveTreeSurvivesSomewhere(t, bundleRoot, "v1") { + t.Error("the tree that was live last was deleted as superseded; no copy of it is left on disk") + } + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Errorf("the genuinely older backup should be superseded once the newer one is restored, got %v", err) + } +} + +// The allocator and the parser must agree, and a number already spoken for by a +// kept backup must never come round again. +func TestStagingNamesAllocateInOrderAndParse(t *testing.T) { + t.Run("counts up and parses", func(t *testing.T) { + dir := t.TempDir() + for want := int64(1); want <= 2; want++ { + seq, err := nextStagingSeq(dir) + if err != nil { + t.Fatal(err) + } + path, err := createSequencedStagingDir(dir, seq) + if err != nil { + t.Fatal(err) + } + stamp, ok := stagingStamp(filepath.Base(path)) + if !ok { + t.Fatalf("the allocator wrote a name recovery cannot order: %q", filepath.Base(path)) + } + if stamp != want { + t.Errorf("stamp = %d, want %d", stamp, want) + } + } + }) + + // The upgrade case: a released binary wrote wall-clock nanoseconds, so + // seeding from the highest present is the whole migration. + t.Run("seeds above a legacy nanosecond name", func(t *testing.T) { + dir := t.TempDir() + const legacy = int64(1_700_000_000_000_000_000) + if err := os.Mkdir(filepath.Join(dir, fmt.Sprintf("%s%020d-x7Kq3", stagingPrefix, legacy)), 0o700); err != nil { + t.Fatal(err) + } + seq, err := nextStagingSeq(dir) + if err != nil { + t.Fatal(err) + } + if seq <= legacy { + t.Errorf("next sequence = %d, want strictly greater than the legacy stamp %d", seq, legacy) + } + }) + + // A parked backup owns its number for as long as it exists. + t.Run("skips a number already parked", func(t *testing.T) { + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, fmt.Sprintf("%s%020d%s", keptPrefix, 7, stagingSeqSuffix)), 0o700); err != nil { + t.Fatal(err) + } + seq, err := nextStagingSeq(dir) + if err != nil { + t.Fatal(err) + } + if seq <= 7 { + t.Errorf("next sequence = %d, want strictly greater than the parked 7", seq) + } + }) +} + +// The retry branch: another writer already holds the number this one started +// from, so it walks up rather than failing or reusing. +func TestCreateSequencedStagingDirSkipsAnOccupiedNumber(t *testing.T) { + dir := t.TempDir() + taken := fmt.Sprintf("%s%020d%s", stagingPrefix, 1, stagingSeqSuffix) + if err := os.Mkdir(filepath.Join(dir, taken), 0o700); err != nil { + t.Fatal(err) + } + + got, err := createSequencedStagingDir(dir, 1) + if err != nil { + t.Fatal(err) + } + want := filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, 2, stagingSeqSuffix)) + if got != want { + t.Errorf("claimed %q, want %q", got, want) + } + if _, err := os.Stat(filepath.Join(dir, taken)); err != nil { + t.Errorf("the occupied name must be left alone: %v", err) + } +} + +// A name at the maximum would make the seeding addition wrap negative, and a +// negative renders a '-' the parser reads as empty digits, so the entry would +// leave the ordering silently. Refusing is the loud version, and the refusal +// belongs where the addition is. +func TestNextStagingSeqRefusesOverflow(t *testing.T) { + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, fmt.Sprintf("%s%020d-x", stagingPrefix, int64(math.MaxInt64))), 0o700); err != nil { + t.Fatal(err) + } + if seq, err := nextStagingSeq(dir); err == nil { + t.Errorf("nextStagingSeq returned %d, want an error rather than a wrapped value", seq) + } +} + +func TestCreateSequencedStagingDirRefusesOutOfRange(t *testing.T) { + dir := t.TempDir() + for _, n := range []int64{0, -1, math.MinInt64} { + if _, err := createSequencedStagingDir(dir, n); err == nil { + t.Errorf("createSequencedStagingDir(%d) succeeded, want an error", n) + } + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Errorf("a refused allocation must create nothing, got %v", entries) + } +} + +// Exclusive creation is the whole concurrency story, so it gets executed rather +// than argued: no two allocators may come away with the same name or number. +func TestStagingSeqAllocatesDistinctValuesConcurrently(t *testing.T) { + dir := t.TempDir() + const writers = 16 + + var wg sync.WaitGroup + paths := make([]string, writers) + errs := make([]error, writers) + for i := 0; i < writers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + seq, err := nextStagingSeq(dir) + if err != nil { + errs[i] = err + return + } + paths[i], errs[i] = createSequencedStagingDir(dir, seq) + }(i) + } + wg.Wait() + + seen := map[string]bool{} + stamps := map[int64]bool{} + for i, path := range paths { + if errs[i] != nil { + t.Fatalf("writer %d: %v", i, errs[i]) + } + if seen[path] { + t.Errorf("two writers claimed the same name %q", path) + } + seen[path] = true + stamp, ok := stagingStamp(filepath.Base(path)) + if !ok { + t.Errorf("writer %d wrote an unorderable name %q", i, filepath.Base(path)) + continue + } + if stamps[stamp] { + t.Errorf("two writers claimed sequence %d", stamp) + } + stamps[stamp] = true + } +} + +// The staging dir holds the only copy of a link's tree mid-swap, so it keeps the +// owner-only mode os.MkdirTemp gave it. +func TestStagingDirKeepsOwnerOnlyPermissions(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Go permission bits do not map to Windows ACLs") + } + dir := t.TempDir() + path, err := createSequencedStagingDir(dir, 1) + if err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o700 { + t.Errorf("staging dir mode = %o, want 0700", got) + } +} + +// liveTreeSurvivesSomewhere reports whether any staging or kept dir under root +// still holds a backup with the given content. +func liveTreeSurvivesSomewhere(t *testing.T, root, content string) bool { + t.Helper() + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + got, err := os.ReadFile(filepath.Join(root, entry.Name(), "backup", "a.txt")) + if err == nil && string(got) == content { + return true + } + } + return false +} + // Recovery decides each link on its own: restoring one link's tree must not make // another link's superseded backup look unorderable, and vice versa. func TestRecoverBundleDirHandlesLinksIndependently(t *testing.T) { diff --git a/internal/dictation/download.go b/internal/dictation/download.go index f369dc3de..cfd69c18f 100644 --- a/internal/dictation/download.go +++ b/internal/dictation/download.go @@ -8,8 +8,11 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" + "io/fs" + "math" "net/http" "os" "path/filepath" @@ -17,7 +20,6 @@ import ( "slices" "strconv" "strings" - "time" ) // Auto-download of the local engine + a default model (opt-in, behind a confirm @@ -765,14 +767,18 @@ func resolveEnginePaths(engineDir string, targetWindows bool) (bin, server strin } // holderSuffix is what promoteStagedDir appends to an install's own name for the -// holder it sets that install aside in. The creation time goes in the name +// holder it sets that install aside in. An ordering number goes in the name // because recovery has to pick the NEWEST holder when a failed cleanup left an // older one behind, and nothing else records that order: Glob sorts lexically -// and a directory mtime tracks the install's contents, not its promotion. +// and a directory mtime tracks the install's contents, not its promotion. The +// number is one past the highest already beside this install, so a clock moving +// backward cannot invert it the way the wall-clock stamp it replaces could. const holderSuffix = ".previous-" -// holderStamp reads back the creation time in a holder name, reporting false for -// a name it cannot order (one this package did not write). +// holderStamp reads back the ordering number in a holder name, reporting false +// for a name it cannot order (one this package did not write). New names carry a +// per-install sequence; names written by released versions carry wall-clock +// nanoseconds. Both compare the same way. func holderStamp(destDir, holder string) (int64, bool) { rest := strings.TrimPrefix(filepath.Base(holder), filepath.Base(destDir)+holderSuffix) digits, _, found := strings.Cut(rest, "-") @@ -786,6 +792,74 @@ func holderStamp(destDir, holder string) (int64, bool) { return stamp, true } +// holderSeqSuffix closes a sequenced holder name. holderStamp cuts on the first +// '-' after the digits, so a name ending at the digits reads back as unstamped, +// which is silent: an unstamped holder sorts last and is still restorable, so +// the ordering key would stop existing with nothing failing. os.MkdirTemp used +// to supply this separator with its random suffix. +const holderSeqSuffix = "-seq" + +// holderSeqAttempts bounds the walk up from a taken number, in the spirit of the +// retry limit os.MkdirTemp applies to its own random names. +const holderSeqAttempts = 10000 + +// nextHolderSeq is the number a new holder should claim: one past the highest +// already set aside for this install. That is what makes the order survive a +// clock that moves backward, since a promotion that reads an existing holder +// always allocates above it. Holders for a different install are a separate +// sequence and are never compared against this one. +func nextHolderSeq(destDir string) (int64, error) { + entries, err := os.ReadDir(filepath.Dir(destDir)) + if err != nil { + return 0, err + } + prefix := filepath.Base(destDir) + holderSuffix + var high int64 + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), prefix) { + continue + } + if stamp, ok := holderStamp(destDir, entry.Name()); ok && stamp > high { + high = stamp + } + } + if high == math.MaxInt64 { + // The addition below would wrap negative, and %020d of a negative + // renders a '-' the parser reads as empty digits, so the holder would + // drop out of the ordering without saying so. Refuse instead. + return 0, fmt.Errorf("a previous install of %s is named at the maximum sequence; remove it before installing again", filepath.Base(destDir)) + } + return high + 1, nil +} + +// createSequencedHolder claims the first free holder name from n upward. Nothing +// locks this directory, so exclusive creation is what arbitrates: two promotions +// racing cannot both win a name, and the loser walks up above the winner. +func createSequencedHolder(destDir string, n int64) (string, error) { + if n < 1 { + return "", fmt.Errorf("refusing to allocate holder sequence %d", n) + } + for i := 0; i < holderSeqAttempts; i++ { + path := fmt.Sprintf("%s%s%020d%s", destDir, holderSuffix, n, holderSeqSuffix) + // The writer and the reader agree on the format or nothing is written. + if stamp, ok := holderStamp(destDir, path); !ok || stamp != n { + return "", fmt.Errorf("holder name %q does not read back as sequence %d", filepath.Base(path), n) + } + err := os.Mkdir(path, 0o700) + if err == nil { + return path, nil + } + if !errors.Is(err, fs.ErrExist) { + return "", err + } + if n == math.MaxInt64 { + return "", fmt.Errorf("holder sequence exhausted for %s", filepath.Base(destDir)) + } + n++ + } + return "", fmt.Errorf("could not claim a holder name for %s after %d attempts", filepath.Base(destDir), holderSeqAttempts) +} + // restoreInterruptedPromotion puts back an install that promoteStagedDir set // aside but never replaced, which is what a process stop between its two renames // leaves behind: destDir absent and the only usable copy in a .previous-* holder @@ -855,8 +929,12 @@ func promoteStagedDir(stageDir, destDir, label string) error { restore := func() error { return nil } if _, err := os.Lstat(destDir); err == nil { - holder, err = os.MkdirTemp(filepath.Dir(destDir), - fmt.Sprintf("%s%s%020d-*", filepath.Base(destDir), holderSuffix, time.Now().UnixNano())) + var seq int64 + seq, err = nextHolderSeq(destDir) + if err != nil { + return fmt.Errorf("setting aside previous %s install: %w", label, err) + } + holder, err = createSequencedHolder(destDir, seq) if err != nil { return fmt.Errorf("setting aside previous %s install: %w", label, err) } diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index 27871a9e7..961faf37d 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -7,12 +7,14 @@ import ( "errors" "fmt" "io" + "math" "net/http" "net/http/httptest" "os" "path/filepath" "runtime" "strings" + "sync" "testing" ) @@ -665,6 +667,195 @@ func TestRestoreInterruptedPromotionFindsHoldersUnderAnAwkwardPath(t *testing.T) } } +// A backward clock leaves a STALE holder carrying a LARGER stamp. Recovery must +// still put back the install that was actually live last. Nothing is deleted +// either way at this site, so the failure is a stale restore, and the negative +// half of the assertion pins that the loser is kept. +func TestRestoreInterruptedPromotionPrefersTheRealNewerInstallOverAFutureStampedHolder(t *testing.T) { + const farFuture = int64(4_000_000_000_000_000_000) + + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + if err := os.MkdirAll(dest, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dest, "engine"), []byte("new"), 0o644); err != nil { + t.Fatal(err) + } + stale := plantHolder(t, dest, farFuture, "stale") + + // The real transaction sets "new" aside and never publishes. + interruptPromotion(t, dest, "engine") + + restoreInterruptedPromotion(dest) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "new" { + t.Errorf("restored %q (err %v), want the install that was live last, %q", got, err, "new") + } + // Whichever holder won, the one that lost is left for a human. + if _, err := os.Stat(filepath.Join(stale, "install", "engine")); err != nil { + t.Errorf("a holder that lost the ordering must be kept, not deleted: %v", err) + } +} + +// The allocator and the parser must agree, and the sequence must seed above a +// name a released binary wrote. +func TestHolderNamesAllocateInOrderAndParse(t *testing.T) { + t.Run("counts up and parses", func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + for want := int64(1); want <= 2; want++ { + seq, err := nextHolderSeq(dest) + if err != nil { + t.Fatal(err) + } + path, err := createSequencedHolder(dest, seq) + if err != nil { + t.Fatal(err) + } + stamp, ok := holderStamp(dest, path) + if !ok { + t.Fatalf("the allocator wrote a name recovery cannot order: %q", filepath.Base(path)) + } + if stamp != want { + t.Errorf("stamp = %d, want %d", stamp, want) + } + } + }) + + t.Run("seeds above a legacy nanosecond name", func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + const legacy = int64(1_700_000_000_000_000_000) + // Both shapes a released binary could have left: the MkdirTemp random + // suffix, and the digits-only one the test helper plants. + for _, suffix := range []string{"x7Kq3", "12345"} { + if err := os.MkdirAll(fmt.Sprintf("%s%s%020d-%s", dest, holderSuffix, legacy, suffix), 0o700); err != nil { + t.Fatal(err) + } + } + seq, err := nextHolderSeq(dest) + if err != nil { + t.Fatal(err) + } + if seq <= legacy { + t.Errorf("next sequence = %d, want strictly greater than the legacy stamp %d", seq, legacy) + } + }) +} + +func TestCreateSequencedHolderSkipsAnOccupiedNumber(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + taken := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 1, holderSeqSuffix) + if err := os.MkdirAll(taken, 0o700); err != nil { + t.Fatal(err) + } + + got, err := createSequencedHolder(dest, 1) + if err != nil { + t.Fatal(err) + } + want := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 2, holderSeqSuffix) + if got != want { + t.Errorf("claimed %q, want %q", got, want) + } +} + +func TestNextHolderSeqRefusesOverflow(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + if err := os.MkdirAll(fmt.Sprintf("%s%s%020d-x", dest, holderSuffix, int64(math.MaxInt64)), 0o700); err != nil { + t.Fatal(err) + } + if seq, err := nextHolderSeq(dest); err == nil { + t.Errorf("nextHolderSeq returned %d, want an error rather than a wrapped value", seq) + } +} + +func TestCreateSequencedHolderRefusesOutOfRange(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + if err := os.MkdirAll(root, 0o755); err != nil { + t.Fatal(err) + } + for _, n := range []int64{0, -1, math.MinInt64} { + if _, err := createSequencedHolder(dest, n); err == nil { + t.Errorf("createSequencedHolder(%d) succeeded, want an error", n) + } + } + if holders := holdersFor(t, dest); len(holders) != 0 { + t.Errorf("a refused allocation must create nothing, got %v", holders) + } +} + +// This site holds no lock, so exclusive creation is the only thing arbitrating. +func TestHolderSeqAllocatesDistinctValuesConcurrently(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + const writers = 16 + + var wg sync.WaitGroup + paths := make([]string, writers) + errs := make([]error, writers) + for i := 0; i < writers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + seq, err := nextHolderSeq(dest) + if err != nil { + errs[i] = err + return + } + paths[i], errs[i] = createSequencedHolder(dest, seq) + }(i) + } + wg.Wait() + + seen := map[string]bool{} + stamps := map[int64]bool{} + for i, path := range paths { + if errs[i] != nil { + t.Fatalf("writer %d: %v", i, errs[i]) + } + if seen[path] { + t.Errorf("two writers claimed the same name %q", path) + } + seen[path] = true + stamp, ok := holderStamp(dest, path) + if !ok { + t.Errorf("writer %d wrote an unorderable name %q", i, filepath.Base(path)) + continue + } + if stamps[stamp] { + t.Errorf("two writers claimed sequence %d", stamp) + } + stamps[stamp] = true + } +} + +// The holder wraps a complete previous install for as long as it exists, so it +// keeps the owner-only mode os.MkdirTemp gave it. +func TestHolderKeepsOwnerOnlyPermissions(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Go permission bits do not map to Windows ACLs") + } + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + path, err := createSequencedHolder(dest, 1) + if err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o700 { + t.Errorf("holder mode = %o, want 0700", got) + } +} + // A holder this package did not name carries no ordering anyone can read, so it // is the least recent thing recovery can claim to know about and must lose to // any stamped holder, however old that one's stamp is. From 1580a599e7e4dc6f84c2f08a3e0b1136d37ce9e3 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 1 Sep 2026 00:20:30 -0500 Subject: [PATCH 16/34] fix(daemon/remote): read only the names the staging allocator owns nextStagingSeq fed every directory entry to stagingStamp, which trims its prefix with TrimPrefix, a no-op on a name that does not carry it. The sibling directories in a bundle dir are the per-link work trees, and a link id is whatever the uploading client sent, so a link named "2024-project" set the next sequence to 2025 and one named for int64's maximum made the allocator refuse to allocate at all. That refusal aborts extractBundle before it does anything else, for every link in the directory, on every later upload and across restarts, until someone removes the directory by hand. Filter the scan the way recoverBundleDir already filters its own, so only staging and kept names reach the parser. Also make two guards mean what they claim. The permission assertion could not tell a correct 0700 from a widened 0755 under a umask of 077, where both come back 0700; it now creates a control directory first and skips with a reason rather than passing on evidence it does not have. The concurrency tests raced 16 allocators, which caught a check-then-create allocator in about half of runs; at 128 it is caught in every run. --- internal/daemon/remote/bundle.go | 12 ++++++- internal/daemon/remote/bundle_test.go | 52 ++++++++++++++++++++++++++- internal/dictation/download_test.go | 24 ++++++++++++- 3 files changed, 85 insertions(+), 3 deletions(-) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index 2d6922902..8a0bfdb12 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -409,9 +409,19 @@ func nextStagingSeq(dir string) (int64, error) { if !entry.IsDir() { continue } + // Read only the names this package writes. The sibling directories here + // are the per-link work trees, and a link id is whatever the uploading + // client sent; stagingStamp trims its prefix with TrimPrefix, which is a + // no-op on a name that lacks it, so an unfiltered scan would read a link + // named "2024-project" as sequence 2024 and one named for int64's + // maximum as a permanent refusal to allocate anything. name := entry.Name() - if strings.HasPrefix(name, keptPrefix) { + switch { + case strings.HasPrefix(name, keptPrefix): name = stagingPrefix + strings.TrimPrefix(name, keptPrefix) + case strings.HasPrefix(name, stagingPrefix): + default: + continue } if stamp, ok := stagingStamp(name); ok && stamp > high { high = stamp diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index 70b335997..917206f0b 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -1182,6 +1182,33 @@ func TestStagingNamesAllocateInOrderAndParse(t *testing.T) { }) } +// A bundle dir's ordinary contents are the per-link work trees, and a link id is +// whatever the uploading client sent. The allocator must read only the names it +// owns: a link id is not a sequence, however much it looks like one. +func TestStagingSeqIgnoresNamesItDoesNotOwn(t *testing.T) { + // sanitizeLinkID permits digits, '-' and letters, so every id here is one a + // client can actually upload under. + for _, id := range []string{"12345-abc", "2024-project", "09223372036854775807-seq"} { + t.Run(id, func(t *testing.T) { + if _, err := sanitizeLinkID(id); err != nil { + t.Fatalf("fixture is not a link id a client could send: %v", err) + } + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, id), 0o700); err != nil { + t.Fatal(err) + } + + seq, err := nextStagingSeq(dir) + if err != nil { + t.Fatalf("a link work tree must not stop the allocator: %v", err) + } + if seq != 1 { + t.Errorf("next sequence = %d, want 1: a link name is not a staging name", seq) + } + }) + } +} + // The retry branch: another writer already holds the number this one started // from, so it walks up rather than failing or reusing. func TestCreateSequencedStagingDirSkipsAnOccupiedNumber(t *testing.T) { @@ -1238,7 +1265,9 @@ func TestCreateSequencedStagingDirRefusesOutOfRange(t *testing.T) { // than argued: no two allocators may come away with the same name or number. func TestStagingSeqAllocatesDistinctValuesConcurrently(t *testing.T) { dir := t.TempDir() - const writers = 16 + // 128, not a smaller number: against a check-then-create allocator this + // detects the duplicate in every run, where 16 caught it about half the time. + const writers = 128 var wg sync.WaitGroup paths := make([]string, writers) @@ -1290,6 +1319,7 @@ func TestStagingDirKeepsOwnerOnlyPermissions(t *testing.T) { if err != nil { t.Fatal(err) } + requireUmaskAllowsWiderThan0700(t, dir) info, err := os.Stat(path) if err != nil { t.Fatal(err) @@ -1299,6 +1329,26 @@ func TestStagingDirKeepsOwnerOnlyPermissions(t *testing.T) { } } +// requireUmaskAllowsWiderThan0700 skips when the process umask would strip the +// group and other bits anyway. Without it this assertion is vacuous under a +// umask of 077: a wrongly widened 0o755 comes back as 0700 and passes, so the +// one test guarding the mode would silently stop guarding it. +func requireUmaskAllowsWiderThan0700(t *testing.T, parent string) { + t.Helper() + control := filepath.Join(parent, ".umask-control") + if err := os.Mkdir(control, 0o755); err != nil { + t.Fatal(err) + } + defer func() { _ = os.RemoveAll(control) }() + info, err := os.Stat(control) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o755 { + t.Skipf("umask masks a 0755 request down to %o, so this assertion cannot tell 0700 from a widened mode", info.Mode().Perm()) + } +} + // liveTreeSurvivesSomewhere reports whether any staging or kept dir under root // still holds a backup with the given content. func liveTreeSurvivesSomewhere(t *testing.T, root, content string) bool { diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index 961faf37d..1d9e7fc56 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -794,7 +794,9 @@ func TestCreateSequencedHolderRefusesOutOfRange(t *testing.T) { func TestHolderSeqAllocatesDistinctValuesConcurrently(t *testing.T) { root := t.TempDir() dest := filepath.Join(root, "engine-1.2.3-linux-x64") - const writers = 16 + // 128, not a smaller number: against a check-then-create allocator this + // detects the duplicate in every run, where 16 caught it about half the time. + const writers = 128 var wg sync.WaitGroup paths := make([]string, writers) @@ -847,6 +849,7 @@ func TestHolderKeepsOwnerOnlyPermissions(t *testing.T) { if err != nil { t.Fatal(err) } + requireUmaskAllowsWiderThan0700(t, root) info, err := os.Stat(path) if err != nil { t.Fatal(err) @@ -856,6 +859,25 @@ func TestHolderKeepsOwnerOnlyPermissions(t *testing.T) { } } +// requireUmaskAllowsWiderThan0700 skips when the process umask would strip the +// group and other bits anyway. Without it this assertion is vacuous under a +// umask of 077: a wrongly widened 0o755 comes back as 0700 and passes. +func requireUmaskAllowsWiderThan0700(t *testing.T, parent string) { + t.Helper() + control := filepath.Join(parent, ".umask-control") + if err := os.Mkdir(control, 0o755); err != nil { + t.Fatal(err) + } + defer func() { _ = os.RemoveAll(control) }() + info, err := os.Stat(control) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o755 { + t.Skipf("umask masks a 0755 request down to %o, so this assertion cannot tell 0700 from a widened mode", info.Mode().Perm()) + } +} + // A holder this package did not name carries no ordering anyone can read, so it // is the least recent thing recovery can claim to know about and must lose to // any stamped holder, however old that one's stamp is. From e17255ea40810ae3b9641fa3133a1eee9953a6aa Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 1 Sep 2026 00:50:37 -0500 Subject: [PATCH 17/34] fix(dictation,daemon/remote): stop a failed cleanup from stranding an install promoteStagedDir renames the previous install into a holder, publishes the new one, then removes the holder. When that removal failed the holder survived with a complete copy of the old engine or model inside it, and nothing ever removed it: restoreInterruptedPromotion returned as soon as destDir existed, so every later replacement stranded another whole install beside the live one. Recovery now reaps a holder the live install superseded. A holder is only ever filled by renaming destDir aside, so a destDir that holds something means a later promotion published over it. An EMPTY destDir is deliberately not that evidence: a husk can outlive a failed or partial promotion, and reaping on its account would delete the only surviving copy, so those holders are still left alone. The scan both paths use is now one function so they cannot drift. The bundle side already reclaimed its equivalent leftover on the next daemon start, which bounds that leak to one daemon lifetime rather than for good, but it did so silently. Since the upload reported success to its client while a whole copy of the prior tree was still on disk, and the cleanup failure itself is only logged, recovery now names what it reclaims. --- internal/daemon/remote/bundle.go | 6 +++ internal/daemon/remote/bundle_test.go | 25 +++++++++++ internal/dictation/download.go | 58 +++++++++++++++++++----- internal/dictation/download_test.go | 64 +++++++++++++++++++++++++-- 4 files changed, 138 insertions(+), 15 deletions(-) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index 8a0bfdb12..9cc369ce1 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -518,6 +518,12 @@ func restoreStagedBackup(dir string, s stagedExtract, restored map[string]staged parkKeptBackup(dir, staging, id, logf) return true } + // The upload that published dest reported success to its client while + // this whole copy of the prior tree was still on disk, and + // extractBundle's own cleanup failure is only logged. Say what is being + // reclaimed, so a bridge that ran without a logger configured is not the + // difference between the space being accounted for and not. + logf("remote: reclaiming the staged tree in %s that %s's live tree superseded", staging, id) if err := os.RemoveAll(staging); err != nil { logf("remote: could not remove superseded staging dir %s: %v", staging, err) } diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index 917206f0b..cf81f5c74 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -1182,6 +1182,31 @@ func TestStagingNamesAllocateInOrderAndParse(t *testing.T) { }) } +// A publish that succeeded but could not clear its staging dir reports success to +// the client while a whole copy of the prior tree stays on disk. The next +// recovery pass reclaims it, and must say so: with no bridge logger configured +// nothing else ever mentions the space that was being held. +func TestRecoverBundleDirReportsReclaimingASupersededTree(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + // The publish landed, so dest holds the new tree. + if err := os.MkdirAll(filepath.Join(dest, ".git"), 0o700); err != nil { + t.Fatal(err) + } + // The cleanup did not, so the old tree is still staged beside it. + staging := stageBackup(t, dir, "leftover", "proj-1", "v-old", 100) + + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }) + + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Errorf("a staged tree the live one superseded should be reclaimed, got %v", err) + } + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, staging) }) { + t.Errorf("reclaiming a superseded staged tree should name it, got %v", logged) + } +} + // A bundle dir's ordinary contents are the per-link work trees, and a link id is // whatever the uploading client sent. The allocator must read only the names it // owns: a link id is not a sequence, however much it looks like one. diff --git a/internal/dictation/download.go b/internal/dictation/download.go index cfd69c18f..7dd92df93 100644 --- a/internal/dictation/download.go +++ b/internal/dictation/download.go @@ -868,23 +868,27 @@ func createSequencedHolder(destDir string, n int64) (string, error) { // Best effort by design, since the caller can still download a fresh engine. func restoreInterruptedPromotion(destDir string) { if _, err := os.Lstat(destDir); err == nil { + // destDir is live. A holder is only ever filled by renaming destDir + // aside, so a destDir that holds something means a later promotion + // published over every holder beside it. Those are superseded copies of + // whole installs, and promoteStagedDir's cleanup is the only thing that + // removes them: when it fails the copy is stranded for good, and each + // later replacement strands another. + // + // An EMPTY destDir is not that evidence. A husk can outlive a failed or + // partial promotion, and reaping on its account would delete the only + // surviving install, so a holder beside one is left alone. + if dirHasEntries(destDir) { + for _, holder := range holdersBeside(destDir) { + _ = os.RemoveAll(holder) + } + } return } // ReadDir and a prefix rather than filepath.Glob: destDir is a real path, // and a '[' anywhere in it opens a character class to Glob, which then // matches nothing and strands the install this exists to put back. - parent := filepath.Dir(destDir) - entries, err := os.ReadDir(parent) - if err != nil { - return - } - prefix := filepath.Base(destDir) + holderSuffix - holders := make([]string, 0, len(entries)) - for _, entry := range entries { - if entry.IsDir() && strings.HasPrefix(entry.Name(), prefix) { - holders = append(holders, filepath.Join(parent, entry.Name())) - } - } + holders := holdersBeside(destDir) // Newest first: an unstamped holder is the least recent thing we can claim // to know about, so it is only reached once every stamped one has failed. slices.SortStableFunc(holders, func(a, b string) int { @@ -913,6 +917,36 @@ func restoreInterruptedPromotion(destDir string) { } } +// dirHasEntries reports whether dir holds anything at all. An unreadable or +// absent dir reads as empty, which is the conservative answer everywhere this is +// used: it withholds the evidence a reap needs rather than supplying it. +func dirHasEntries(dir string) bool { + entries, err := os.ReadDir(dir) + return err == nil && len(entries) > 0 +} + +// holdersBeside lists the holders promoteStagedDir may have left for destDir. +// The prefix is the attribution: a holder for a different install is a separate +// concern and is never touched on this one's account. +func holdersBeside(destDir string) []string { + // ReadDir and a prefix rather than filepath.Glob: destDir is a real path, + // and a '[' anywhere in it opens a character class to Glob, which then + // matches nothing and strands the install this exists to put back. + parent := filepath.Dir(destDir) + entries, err := os.ReadDir(parent) + if err != nil { + return nil + } + prefix := filepath.Base(destDir) + holderSuffix + holders := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() && strings.HasPrefix(entry.Name(), prefix) { + holders = append(holders, filepath.Join(parent, entry.Name())) + } + } + return holders +} + // promoteStagedDir moves stageDir into place at destDir. os.Rename refuses to // overwrite a non-empty directory, so any previous install has to move out of // the way first; it is set aside rather than deleted, and put back if the diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index 1d9e7fc56..1b09e89da 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -393,9 +393,60 @@ func TestRestoreInterruptedPromotionPutsTheInstallBack(t *testing.T) { } } +// A promotion that published its install but could not remove the holder leaves +// a complete copy of the OLD install beside a live one. Nothing else ever +// removes it, and every later replacement adds another, so recovery reaps it. +// The holder is filled by renaming destDir aside, so a destDir that holds +// something means a later promotion published over this holder. +func TestRestoreInterruptedPromotionReapsAHolderSupersededByALiveInstall(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + if err := os.MkdirAll(dest, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dest, "engine"), []byte("new"), 0o644); err != nil { + t.Fatal(err) + } + stranded := plantHolder(t, dest, 100, "old") + older := plantHolder(t, dest, 50, "older") + + restoreInterruptedPromotion(dest) + + // The live install is never touched. This is the assertion that matters most. + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "new" { + t.Fatalf("the live install must be left alone: got %q err %v", got, err) + } + for _, holder := range []string{stranded, older} { + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("a holder superseded by the live install should be reaped, got %v", err) + } + } +} + +// An empty destDir is not evidence that anything was published: it is a husk a +// failed or partial promotion can leave. Reaping on its account would delete the +// only surviving install, so the holder stays. +func TestRestoreInterruptedPromotionKeepsAHolderWhenDestIsAnEmptyHusk(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + if err := os.MkdirAll(dest, 0o755); err != nil { + t.Fatal(err) + } + holder := plantHolder(t, dest, 100, "the only copy") + + restoreInterruptedPromotion(dest) + + got, err := os.ReadFile(filepath.Join(holder, "install", "engine")) + if err != nil || string(got) != "the only copy" { + t.Errorf("an empty dest must not cost the holder its install: got %q err %v", got, err) + } +} + // A holder is a leftover, never a replacement for whatever is already at destDir, // empty or not. os.Rename refuses an existing directory either way, so this -// pins the behavior rather than one implementation of it. +// pins the behavior rather than one implementation of it. What happens to the +// holder afterwards differs by case and is asserted below. func TestRestoreInterruptedPromotionLeavesAnExistingDestAlone(t *testing.T) { for _, tc := range []struct{ name, live string }{ {"empty dest", ""}, @@ -431,8 +482,15 @@ func TestRestoreInterruptedPromotionLeavesAnExistingDestAlone(t *testing.T) { } else if err != nil || string(got) != tc.live { t.Fatalf("engine = %q, err %v, want the live %q", got, err, tc.live) } - if _, err := os.Stat(install); err != nil { - t.Errorf("the holder must be left intact when dest exists: %v", err) + // A live dest supersedes the holder and reaps it; an empty husk is + // no such evidence and the holder stays. Either way the assertion + // above stands: dest is never replaced by a holder. + _, err = os.Stat(install) + if tc.live == "" && err != nil { + t.Errorf("an empty dest must leave the holder intact: %v", err) + } + if tc.live != "" && !os.IsNotExist(err) { + t.Errorf("a live dest should reap the holder it superseded, got %v", err) } }) } From b57cacb55e8b56f88cb14e951c687ddbb135ea53 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 1 Sep 2026 01:12:20 -0500 Subject: [PATCH 18/34] fix(dictation): reap a holder only for a destination that is actually usable The reap added in e17255ea gated on destDir being non-empty, which is not the same claim as a promotion having published there. A destination holding a half populated tree, left by anything outside this transaction, reads as non-empty while the only usable copy of the engine or model sits in the holder beside it. Recovery then deleted that copy, and EnsureLocalEngine went to the network for a replacement, which an offline caller does not have. That is the loss the holder exists to prevent. restoreInterruptedPromotion now takes the same predicate its caller already uses to decide whether a download is needed: the engine binary resolving for the engine, dirHasModel for the model. A holder only loses to a destination that predicate accepts. The empty husk case is covered by the same rule rather than by a separate check, so dirHasEntries is gone. Reported by CodeRabbit on #993. --- internal/dictation/download.go | 37 +++++++----- internal/dictation/download_test.go | 90 ++++++++++++++++++++++------- 2 files changed, 89 insertions(+), 38 deletions(-) diff --git a/internal/dictation/download.go b/internal/dictation/download.go index 7dd92df93..64758350f 100644 --- a/internal/dictation/download.go +++ b/internal/dictation/download.go @@ -496,7 +496,10 @@ func EnsureLocalEngine(ctx context.Context, opts DownloadOptions) (EngineCompone // A previous run may have been stopped mid-promotion, leaving the only // install in a holder beside engineDir. Put it back before deciding whether // anything needs downloading. - restoreInterruptedPromotion(engineDir) + restoreInterruptedPromotion(engineDir, func(dir string) bool { + bin, _ := resolveEnginePaths(dir, targetWindows) + return fileExists(bin) + }) binPath, serverPath := resolveEnginePaths(engineDir, targetWindows) if !fileExists(binPath) { pinned := "" @@ -528,7 +531,7 @@ func EnsureLocalEngine(ctx context.Context, opts DownloadOptions) (EngineCompone // promoteStagedDir is shared with the model, so a stop mid-promotion leaves // the model in a holder too. Put it back before deciding anything is // missing: without this an offline user has no download to fall back on. - restoreInterruptedPromotion(modelDir) + restoreInterruptedPromotion(modelDir, dirHasModel) if !dirHasModel(modelDir) { asset, err := resolveAsset(ctx, client, apiBase, modelReleaseTag, modelName, "") if err != nil { @@ -866,19 +869,29 @@ func createSequencedHolder(destDir string, n int64) (string, error) { // nothing else looks at. Anything already at destDir wins, and the check for it // is explicit rather than leaning on os.Rename refusing an existing directory. // Best effort by design, since the caller can still download a fresh engine. -func restoreInterruptedPromotion(destDir string) { +// published reports whether destDir holds an install this caller can actually +// use. Recovery needs it because "there is something at destDir" and "a +// promotion published there" are different claims, and only the second one +// makes a holder beside it superseded. +func restoreInterruptedPromotion(destDir string, published func(string) bool) { if _, err := os.Lstat(destDir); err == nil { // destDir is live. A holder is only ever filled by renaming destDir - // aside, so a destDir that holds something means a later promotion + // aside, so a destDir holding a USABLE install means a later promotion // published over every holder beside it. Those are superseded copies of // whole installs, and promoteStagedDir's cleanup is the only thing that // removes them: when it fails the copy is stranded for good, and each // later replacement strands another. // - // An EMPTY destDir is not that evidence. A husk can outlive a failed or - // partial promotion, and reaping on its account would delete the only - // surviving install, so a holder beside one is left alone. - if dirHasEntries(destDir) { + // Merely non-empty is not that evidence. An empty husk, or a half + // populated directory left by something outside this transaction, can + // sit at destDir while the only usable copy is in the holder; reaping on + // either would delete exactly what the caller is about to need, and an + // offline caller has no download to fall back on. So the holder only + // loses to a destination that is genuinely usable. + // + // A removal that fails is left for the next call, which reaches this + // same branch and tries again. + if published != nil && published(destDir) { for _, holder := range holdersBeside(destDir) { _ = os.RemoveAll(holder) } @@ -917,14 +930,6 @@ func restoreInterruptedPromotion(destDir string) { } } -// dirHasEntries reports whether dir holds anything at all. An unreadable or -// absent dir reads as empty, which is the conservative answer everywhere this is -// used: it withholds the evidence a reap needs rather than supplying it. -func dirHasEntries(dir string) bool { - entries, err := os.ReadDir(dir) - return err == nil && len(entries) > 0 -} - // holdersBeside lists the holders promoteStagedDir may have left for destDir. // The prefix is the attribution: a holder for a different install is a separate // concern and is never touched on this one's account. diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index 1b09e89da..a3bc5c568 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -379,7 +379,7 @@ func TestRestoreInterruptedPromotionPutsTheInstallBack(t *testing.T) { t.Fatal(err) } - restoreInterruptedPromotion(dest) + restoreInterruptedPromotion(dest, testPublished) got, err := os.ReadFile(filepath.Join(dest, "engine")) if err != nil { @@ -410,7 +410,7 @@ func TestRestoreInterruptedPromotionReapsAHolderSupersededByALiveInstall(t *test stranded := plantHolder(t, dest, 100, "old") older := plantHolder(t, dest, 50, "older") - restoreInterruptedPromotion(dest) + restoreInterruptedPromotion(dest, testPublished) // The live install is never touched. This is the assertion that matters most. got, err := os.ReadFile(filepath.Join(dest, "engine")) @@ -424,22 +424,61 @@ func TestRestoreInterruptedPromotionReapsAHolderSupersededByALiveInstall(t *test } } -// An empty destDir is not evidence that anything was published: it is a husk a -// failed or partial promotion can leave. Reaping on its account would delete the -// only surviving install, so the holder stays. -func TestRestoreInterruptedPromotionKeepsAHolderWhenDestIsAnEmptyHusk(t *testing.T) { - root := t.TempDir() - dest := filepath.Join(root, "engine-1.2.3-linux-x64") - if err := os.MkdirAll(dest, 0o755); err != nil { - t.Fatal(err) - } - holder := plantHolder(t, dest, 100, "the only copy") +// A destDir that is merely NOT EMPTY is no evidence a promotion published there. +// An empty husk and a half-populated one are the same thing to recovery, and +// reaping on either would delete the copy the user still needs, so the holder +// only loses to a destination that holds a genuinely usable install. +func TestRestoreInterruptedPromotionKeepsAHolderWhenDestIsNotAUsableInstall(t *testing.T) { + for _, tc := range []struct { + name string + seed func(t *testing.T, dest string) + usable func(string) bool + }{ + { + name: "empty husk", + seed: func(t *testing.T, dest string) {}, + usable: func(dir string) bool { bin, _ := resolveEnginePaths(dir, false); return fileExists(bin) }, + }, + { + name: "non-empty but no engine binary", + seed: func(t *testing.T, dest string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(dest, "bin"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dest, "bin", "README"), []byte("partial"), 0o644); err != nil { + t.Fatal(err) + } + }, + usable: func(dir string) bool { bin, _ := resolveEnginePaths(dir, false); return fileExists(bin) }, + }, + { + name: "model dir without tokens.txt", + seed: func(t *testing.T, dest string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dest, "something.onnx"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + }, + usable: dirHasModel, + }, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + if err := os.MkdirAll(dest, 0o755); err != nil { + t.Fatal(err) + } + tc.seed(t, dest) + holder := plantHolder(t, dest, 100, "the only copy") - restoreInterruptedPromotion(dest) + restoreInterruptedPromotion(dest, tc.usable) - got, err := os.ReadFile(filepath.Join(holder, "install", "engine")) - if err != nil || string(got) != "the only copy" { - t.Errorf("an empty dest must not cost the holder its install: got %q err %v", got, err) + got, err := os.ReadFile(filepath.Join(holder, "install", "engine")) + if err != nil || string(got) != "the only copy" { + t.Errorf("a dest that is not a usable install must not cost the holder its copy: got %q err %v", got, err) + } + }) } } @@ -472,7 +511,7 @@ func TestRestoreInterruptedPromotionLeavesAnExistingDestAlone(t *testing.T) { t.Fatal(err) } - restoreInterruptedPromotion(dest) + restoreInterruptedPromotion(dest, testPublished) got, err := os.ReadFile(filepath.Join(dest, "engine")) if tc.live == "" { @@ -546,6 +585,13 @@ func offlineAPIBase(t *testing.T) string { return url } +// testPublished is the "is this a real install" predicate these tests use: the +// fixtures write an "engine" file, so its presence is what publication means here. +func testPublished(dir string) bool { + _, err := os.Stat(filepath.Join(dir, "engine")) + return err == nil +} + // plantHolder writes an install into a holder named the way promoteStagedDir // names one, so recovery sees the same shape it does in production. func plantHolder(t *testing.T, destDir string, stamp int64, content string) string { @@ -570,7 +616,7 @@ func TestRestoreInterruptedPromotionPrefersTheNewestHolder(t *testing.T) { stale := plantHolder(t, dest, 100, "stale") current := plantHolder(t, dest, 200, "current") - restoreInterruptedPromotion(dest) + restoreInterruptedPromotion(dest, testPublished) got, err := os.ReadFile(filepath.Join(dest, "engine")) if err != nil || string(got) != "current" { @@ -715,7 +761,7 @@ func TestRestoreInterruptedPromotionFindsHoldersUnderAnAwkwardPath(t *testing.T) t.Fatal(err) } - restoreInterruptedPromotion(dest) + restoreInterruptedPromotion(dest, testPublished) got, err := os.ReadFile(filepath.Join(dest, "engine")) if err != nil || string(got) != "kept" { @@ -745,7 +791,7 @@ func TestRestoreInterruptedPromotionPrefersTheRealNewerInstallOverAFutureStamped // The real transaction sets "new" aside and never publishes. interruptPromotion(t, dest, "engine") - restoreInterruptedPromotion(dest) + restoreInterruptedPromotion(dest, testPublished) got, err := os.ReadFile(filepath.Join(dest, "engine")) if err != nil || string(got) != "new" { @@ -954,7 +1000,7 @@ func TestRestoreInterruptedPromotionPrefersAStampedHolderOverAnUnstampedOne(t *t } stamped := plantHolder(t, dest, 100, "stamped") - restoreInterruptedPromotion(dest) + restoreInterruptedPromotion(dest, testPublished) got, err := os.ReadFile(filepath.Join(dest, "engine")) if err != nil || string(got) != "stamped" { @@ -983,7 +1029,7 @@ func TestRestoreInterruptedPromotionSkipsAHolderWithNoInstall(t *testing.T) { t.Fatal(err) } - restoreInterruptedPromotion(dest) + restoreInterruptedPromotion(dest, testPublished) got, err := os.ReadFile(filepath.Join(dest, "engine")) if err != nil || string(got) != "kept" { From 2f41eedb690e62af3f3cc2ff6f71254bc053093c Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 1 Sep 2026 01:45:16 -0500 Subject: [PATCH 19/34] test: name the mode the allocator sets, not the one MkdirTemp used to Both permission tests said the directory keeps the mode os.MkdirTemp gave it, but neither allocator has called os.MkdirTemp since the switch to exclusive create. os.Mkdir takes a mode where MkdirTemp did not, which is why the allocator names 0o700 explicitly and why these tests exist. --- internal/daemon/remote/bundle_test.go | 3 ++- internal/dictation/download_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index cf81f5c74..5f8fdf9b5 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -1334,7 +1334,8 @@ func TestStagingSeqAllocatesDistinctValuesConcurrently(t *testing.T) { } // The staging dir holds the only copy of a link's tree mid-swap, so it keeps the -// owner-only mode os.MkdirTemp gave it. +// owner-only mode it has always had. os.Mkdir takes +// a mode where os.MkdirTemp did not, so the allocator has to name it. func TestStagingDirKeepsOwnerOnlyPermissions(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Go permission bits do not map to Windows ACLs") diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index a3bc5c568..0e88fbd35 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -942,7 +942,8 @@ func TestHolderSeqAllocatesDistinctValuesConcurrently(t *testing.T) { } // The holder wraps a complete previous install for as long as it exists, so it -// keeps the owner-only mode os.MkdirTemp gave it. +// keeps the owner-only mode it has always had. os.Mkdir takes +// a mode where os.MkdirTemp did not, so the allocator has to name it. func TestHolderKeepsOwnerOnlyPermissions(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Go permission bits do not map to Windows ACLs") From 6d65b717ff4b8262138e08d13f4d225887d32b19 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:47:16 -0500 Subject: [PATCH 20/34] test(daemon/remote,dictation): route every filesystem step through one seam The crash-recovery tests could inject a failure at the publish rename and nowhere else, so the steps whose interruption the recovery passes exist to survive, the set-aside rename, every recursive remove, the marker write and the directory reads, could only be reasoned about. A guard nobody can make fail is indistinguishable from one that does nothing. Each package now holds one struct of function fields that every filesystem call in its write path, allocator and recovery goes through, replacing the two single-purpose rename vars. Tests swap a field, match the call by argument rather than by ordinal, and restore it; the counter behind the matcher is atomic so one seam can be driven from two goroutines under -race. Widening the seam widened what a blanket injection catches: five existing tests injected on every rename and now also killed the set-aside they assert against, so they select the call by argument instead. No production branch, order or error text changed. --- internal/daemon/remote/bundle.go | 71 +++-- internal/daemon/remote/bundle_test.go | 405 ++++++++++++++++++++++++-- internal/dictation/download.go | 63 ++-- internal/dictation/download_test.go | 383 +++++++++++++++++++++++- 4 files changed, 850 insertions(+), 72 deletions(-) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index 9cc369ce1..503f9e359 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -200,9 +200,38 @@ const stagingLinkFile = "link" // extractLockPoll is how often a cross-process extract lock is retried. const extractLockPoll = 50 * time.Millisecond -// renameDir moves a directory into its published location. It is a var so tests -// can force a failure at the steps whose errors would otherwise be unrecoverable. -var renameDir = os.Rename +// fsOps is the filesystem seam the write path, the allocator, and recovery take +// every step through. A step that calls os directly cannot be made to fail, and +// the crash each of these steps exists to survive is then only reasoned about. +// Every field is one call, so a test can fail the second remove or the rename +// whose source is one particular backup and leave the rest of the pass real. +type fsOps struct { + rename func(from, to string) error + removeAll func(path string) error + stat func(name string) (os.FileInfo, error) + lstat func(name string) (os.FileInfo, error) + readDir func(name string) ([]os.DirEntry, error) + readFile func(name string) ([]byte, error) + mkdir func(name string, perm os.FileMode) error + writeFile func(name string, data []byte, perm os.FileMode) error + create func(name string, flag int, perm os.FileMode) (*os.File, error) + createTemp func(dir, pattern string) (*os.File, error) +} + +// stagingFS is the seam every filesystem step in this file goes through. Tests +// swap a field and restore it; nothing else writes to it. +var stagingFS = fsOps{ + rename: os.Rename, + removeAll: os.RemoveAll, + stat: os.Stat, + lstat: os.Lstat, + readDir: os.ReadDir, + readFile: os.ReadFile, + mkdir: os.Mkdir, + writeFile: os.WriteFile, + create: os.OpenFile, + createTemp: os.CreateTemp, +} // extractLocks serializes extracts per destination. Each bundle upload runs in // its own connection goroutine, so two uploads of one link id would otherwise @@ -291,7 +320,7 @@ func recoverBundleDir(dir string, logf func(string, ...any)) { if logf == nil { logf = func(string, ...any) {} } - entries, err := os.ReadDir(dir) + entries, err := stagingFS.readDir(dir) if err != nil { if !os.IsNotExist(err) { logf("remote: could not scan bundle dir %s: %v", dir, err) @@ -336,17 +365,17 @@ func recoverBundleDir(dir string, logf func(string, ...any)) { // all: link ids starting with '.' used to be accepted, so this may be a // work tree someone published under a name that now looks reserved. // Never reap that. - if _, err := os.Stat(filepath.Join(staging, ".git")); err == nil { + if _, err := stagingFS.stat(filepath.Join(staging, ".git")); err == nil { logf("remote: %s holds a work tree, not a staged extract; leaving it in place", staging) continue } // Only reap once no clone can still be running: gitTimeout bounds a // clone, so anything older than that is abandoned. - info, err := os.Stat(staging) + info, err := stagingFS.stat(staging) if err != nil || time.Since(info.ModTime()) < 2*gitTimeout { continue } - if err := os.RemoveAll(staging); err != nil { + if err := stagingFS.removeAll(staging); err != nil { logf("remote: could not remove abandoned staging dir %s: %v", staging, err) } } @@ -400,7 +429,7 @@ const stagingSeqAttempts = 10000 // pass deletes it as superseded. Counting kept names keeps a parked number out // of circulation. Recovery still does not enumerate them. func nextStagingSeq(dir string) (int64, error) { - entries, err := os.ReadDir(dir) + entries, err := stagingFS.readDir(dir) if err != nil { return 0, err } @@ -452,7 +481,7 @@ func createSequencedStagingDir(dir string, n int64) (string, error) { return "", fmt.Errorf("remote: staging name %q does not read back as sequence %d", name, n) } path := filepath.Join(dir, name) - err := os.Mkdir(path, 0o700) + err := stagingFS.mkdir(path, 0o700) if err == nil { return path, nil } @@ -473,10 +502,10 @@ func createSequencedStagingDir(dir string, n int64) (string, error) { func restoreStagedBackup(dir string, s stagedExtract, restored map[string]stagedExtract, logf func(string, ...any)) bool { staging := s.path backup := filepath.Join(staging, "backup") - if _, err := os.Stat(backup); err != nil { + if _, err := stagingFS.stat(backup); err != nil { return false } - raw, err := os.ReadFile(filepath.Join(staging, stagingLinkFile)) + raw, err := stagingFS.readFile(filepath.Join(staging, stagingLinkFile)) if err != nil { logf("remote: staged tree in %s has no link marker; leaving it in place", staging) return true @@ -503,7 +532,7 @@ func restoreStagedBackup(dir string, s stagedExtract, restored map[string]staged return true } defer release() - if _, err := os.Stat(dest); err == nil { + if _, err := stagingFS.stat(dest); err == nil { // The link already has a tree, and a backup only ever holds the tree that // was live BEFORE it: backup is filled by renaming dest aside, so dest // holding anything at all means a later extract published over it. That @@ -524,18 +553,18 @@ func restoreStagedBackup(dir string, s stagedExtract, restored map[string]staged // reclaimed, so a bridge that ran without a logger configured is not the // difference between the space being accounted for and not. logf("remote: reclaiming the staged tree in %s that %s's live tree superseded", staging, id) - if err := os.RemoveAll(staging); err != nil { + if err := stagingFS.removeAll(staging); err != nil { logf("remote: could not remove superseded staging dir %s: %v", staging, err) } return true } - if err := os.Rename(backup, dest); err != nil { + if err := stagingFS.rename(backup, dest); err != nil { logf("remote: could not restore the staged tree for %s from %s: %v", id, staging, err) return true } restored[dest] = s logf("remote: restored the work tree for %s from %s after an interrupted extract", id, staging) - if err := os.RemoveAll(staging); err != nil { + if err := stagingFS.removeAll(staging); err != nil { logf("remote: could not remove staging dir %s after restoring %s: %v", staging, id, err) } return true @@ -549,7 +578,7 @@ func restoreStagedBackup(dir string, s stagedExtract, restored map[string]staged // it is, which is the same fail-safe one pass later. func parkKeptBackup(dir, staging, id string, logf func(string, ...any)) { parked := filepath.Join(dir, keptPrefix+strings.TrimPrefix(filepath.Base(staging), stagingPrefix)) - if err := os.Rename(staging, parked); err != nil { + if err := stagingFS.rename(staging, parked); err != nil { logf("remote: could not park the kept backup for %s at %s: %v", id, parked, err) } } @@ -603,7 +632,7 @@ func extractBundle(ctx context.Context, bundleFile, dest string, logf func(strin // A failure here strands a whole copy of the prior tree under a // dot-prefixed dir nothing else enumerates, so say so rather than // leaking it silently. - if err := os.RemoveAll(staging); err != nil { + if err := stagingFS.removeAll(staging); err != nil { logf("remote: could not remove bundle staging dir %s: %v", staging, err) } }() @@ -618,16 +647,16 @@ func extractBundle(ctx context.Context, bundleFile, dest string, logf func(strin backup := filepath.Join(staging, "backup") // Record the link before moving its tree, so a crash in the swap window // leaves something recoverBundleDir can attribute and put back. - if err := os.WriteFile(filepath.Join(staging, stagingLinkFile), []byte(filepath.Base(dest)), 0o600); err != nil { + if err := stagingFS.writeFile(filepath.Join(staging, stagingLinkFile), []byte(filepath.Base(dest)), 0o600); err != nil { return err } restore := func() error { return nil } - if err := os.Rename(dest, backup); err == nil { - restore = func() error { return renameDir(backup, dest) } + if err := stagingFS.rename(dest, backup); err == nil { + restore = func() error { return stagingFS.rename(backup, dest) } } else if !os.IsNotExist(err) { return err } - if err := renameDir(cloneDest, dest); err != nil { + if err := stagingFS.rename(cloneDest, dest); err != nil { if restoreErr := restore(); restoreErr != nil { // dest is empty and the only copy of the prior tree is the backup, // so keep staging rather than deleting the tree on the way out. diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index 5f8fdf9b5..0360e96c9 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -8,10 +8,13 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "runtime" "slices" + "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -286,12 +289,12 @@ func TestExtractBundleConcurrentSameDestAlwaysLeavesATree(t *testing.T) { // The unsafe window is the two back-to-back renames after the clone. Without // this delay it is narrow enough that the test still passes a good fraction // of the time with the lock removed, which would make it a guard in name only. - real := renameDir - renameDir = func(from, to string) error { + real := stagingFS.rename + stagingFS.rename = func(from, to string) error { time.Sleep(2 * time.Millisecond) return real(from, to) } - t.Cleanup(func() { renameDir = real }) + t.Cleanup(func() { stagingFS.rename = real }) var wg sync.WaitGroup var mu sync.Mutex @@ -351,16 +354,16 @@ func TestExtractBundleRestoresPriorTreeWhenPublishFails(t *testing.T) { } // Fail only the publish, so the restore rename that follows it still runs. - real := renameDir - calls := 0 - renameDir = func(from, to string) error { - calls++ - if calls == 1 { + // The clone is what moves in from repo; the set-aside and the restore both + // have to stay real for the prior tree to come back. + real := stagingFS.rename + stagingFS.rename = func(from, to string) error { + if filepath.Base(from) == "repo" { return errors.New("injected publish failure") } return real(from, to) } - t.Cleanup(func() { renameDir = real }) + t.Cleanup(func() { stagingFS.rename = real }) if err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, nil); err == nil { t.Fatal("a failed publish must be reported") @@ -385,9 +388,16 @@ func TestExtractBundleKeepsBackupWhenRestoreAlsoFails(t *testing.T) { t.Fatalf("seed extract: %v", err) } - real := renameDir - renameDir = func(string, string) error { return errors.New("injected rename failure") } - t.Cleanup(func() { renameDir = real }) + // The publish and the restore are the two renames into dest; the set-aside + // that fills the backup stays real, or there is no retained copy to assert on. + real := stagingFS.rename + stagingFS.rename = func(from, to string) error { + if to == dest { + return errors.New("injected rename failure") + } + return real(from, to) + } + t.Cleanup(func() { stagingFS.rename = real }) err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, nil) if err == nil { @@ -419,7 +429,7 @@ func TestExtractBundleKeepsBackupWhenRestoreAlsoFails(t *testing.T) { // The retained tree carries its link marker, so the next bridge start puts // it back rather than leaving the link empty forever. - renameDir = real + stagingFS.rename = real recoverBundleDir(bundleDir, nil) got, readErr = os.ReadFile(filepath.Join(dest, "a.txt")) if readErr != nil { @@ -716,9 +726,9 @@ func TestRecoverBundleDirLeavesALiveExtractAlone(t *testing.T) { } inPublish := make(chan struct{}) - real := renameDir + real := stagingFS.rename var once sync.Once - renameDir = func(from, to string) error { + stagingFS.rename = func(from, to string) error { // Stall only the publish (clone into dest), not the restore. if filepath.Base(from) == "repo" { once.Do(func() { close(inPublish) }) @@ -726,7 +736,7 @@ func TestRecoverBundleDirLeavesALiveExtractAlone(t *testing.T) { } return real(from, to) } - t.Cleanup(func() { renameDir = real }) + t.Cleanup(func() { stagingFS.rename = real }) var wg sync.WaitGroup var extractErr error @@ -1018,10 +1028,15 @@ func TestBridgeRecoversARealInterruptedExtractOnStart(t *testing.T) { // Both renames fail, which is the one path that leaves the tree in staging // with dest absent -- what a process killed between the two renames leaves. - real := renameDir - renameDir = func(string, string) error { return errors.New("injected rename failure") } + real := stagingFS.rename + stagingFS.rename = func(from, to string) error { + if to == dest { + return errors.New("injected rename failure") + } + return real(from, to) + } _, err := UploadRepoBundle(cfg, initTestRepo(t, "a.txt", "v2"), "proj-1") - renameDir = real + stagingFS.rename = real if err == nil { t.Fatal("an upload whose publish and restore both fail must report an error") } @@ -1096,10 +1111,15 @@ func TestExtractBundleOutOrdersALeftoverStampedInTheFuture(t *testing.T) { // Only now, with the bridge already up and its recovery pass behind us. stale := stageBackup(t, bundleRoot, "old", "proj-1", "v-old", farFuture) - real := renameDir - renameDir = func(string, string) error { return errors.New("injected rename failure") } + real := stagingFS.rename + stagingFS.rename = func(from, to string) error { + if to == dest { + return errors.New("injected rename failure") + } + return real(from, to) + } _, err := UploadRepoBundle(cfg, initTestRepo(t, "a.txt", "v2"), "proj-1") - renameDir = real + stagingFS.rename = real if err == nil { t.Fatal("an upload whose publish and restore both fail must report an error") } @@ -1486,3 +1506,344 @@ func TestRecoverBundleDirKeepsABackupItCannotTellApartFromTheRestore(t *testing. t.Fatalf("restored %q and kept %d of the two tied backups, want exactly 1 kept", got, kept) } } + +// ---- filesystem seam ------------------------------------------------------- + +// injectFault swaps one field of stagingFS for a wrapper that fails the call +// whose arguments satisfy match and passes every other call through to the real +// primitive. The call ordinal is appended to the arguments handed to match, so +// nthCall can select by call order without a second counter, and it is counted +// with an atomic because the racing scenarios drive one seam from two +// goroutines. The whole struct is restored in t.Cleanup, so a test that injects +// twice unwinds in reverse. +func injectFault(t *testing.T, step string, match func(args ...string) bool, err error) { + t.Helper() + var calls atomic.Int64 + real := stagingFS + t.Cleanup(func() { stagingFS = real }) + fire := func(args ...string) bool { + n := calls.Add(1) + if match == nil { + return true + } + return match(append(args, strconv.FormatInt(n, 10))...) + } + switch step { + case "rename": + stagingFS.rename = func(from, to string) error { + if fire(from, to) { + return err + } + return real.rename(from, to) + } + case "removeAll": + stagingFS.removeAll = func(path string) error { + if fire(path) { + return err + } + return real.removeAll(path) + } + case "stat": + stagingFS.stat = func(name string) (os.FileInfo, error) { + if fire(name) { + return nil, err + } + return real.stat(name) + } + case "lstat": + stagingFS.lstat = func(name string) (os.FileInfo, error) { + if fire(name) { + return nil, err + } + return real.lstat(name) + } + case "readDir": + stagingFS.readDir = func(name string) ([]os.DirEntry, error) { + if fire(name) { + return nil, err + } + return real.readDir(name) + } + case "readFile": + stagingFS.readFile = func(name string) ([]byte, error) { + if fire(name) { + return nil, err + } + return real.readFile(name) + } + case "mkdir": + stagingFS.mkdir = func(name string, perm os.FileMode) error { + if fire(name) { + return err + } + return real.mkdir(name, perm) + } + case "writeFile": + stagingFS.writeFile = func(name string, data []byte, perm os.FileMode) error { + if fire(name) { + return err + } + return real.writeFile(name, data, perm) + } + case "create": + stagingFS.create = func(name string, flag int, perm os.FileMode) (*os.File, error) { + if fire(name) { + return nil, err + } + return real.create(name, flag, perm) + } + case "createTemp": + stagingFS.createTemp = func(dir, pattern string) (*os.File, error) { + if fire(dir, pattern) { + return nil, err + } + return real.createTemp(dir, pattern) + } + default: + t.Fatalf("injectFault: unknown step %q", step) + } +} + +// nthCall selects the nth call through an injected seam. It reads the ordinal +// injectFault appends to the arguments rather than counting for itself, so the +// two share one counter and a concurrent test has one place to be correct. +func nthCall(n int) func(args ...string) bool { + return func(args ...string) bool { + return len(args) > 0 && args[len(args)-1] == strconv.Itoa(n) + } +} + +// blockStep holds every call to step until the returned release runs, which is +// how a test parks one transaction mid-step and lets another reach the same +// destination. Releasing twice is safe, so a test can release on the happy path +// and still defer it. +func blockStep(t *testing.T, step string) (release func()) { + t.Helper() + gate := make(chan struct{}) + var once sync.Once + release = func() { once.Do(func() { close(gate) }) } + t.Cleanup(release) + injectFaultGate(t, step, gate) + return release +} + +// injectFaultGate is blockStep's half of the swap, split out so the wrapper it +// installs sits beside the fault wrappers above. +func injectFaultGate(t *testing.T, step string, gate <-chan struct{}) { + t.Helper() + real := stagingFS + t.Cleanup(func() { stagingFS = real }) + switch step { + case "rename": + stagingFS.rename = func(from, to string) error { + <-gate + return real.rename(from, to) + } + case "removeAll": + stagingFS.removeAll = func(path string) error { + <-gate + return real.removeAll(path) + } + case "stat": + stagingFS.stat = func(name string) (os.FileInfo, error) { + <-gate + return real.stat(name) + } + default: + t.Fatalf("blockStep: unknown step %q", step) + } +} + +// The seam has to be the real filesystem until a test swaps a field. A nil field +// is a panic in whatever production path reaches it first, and a field that no +// longer behaves like its os function makes every test above it prove nothing. +func TestStagingFSDefaultsAreTheRealPrimitives(t *testing.T) { + for name, missing := range map[string]bool{ + "rename": stagingFS.rename == nil, + "removeAll": stagingFS.removeAll == nil, + "stat": stagingFS.stat == nil, + "lstat": stagingFS.lstat == nil, + "readDir": stagingFS.readDir == nil, + "readFile": stagingFS.readFile == nil, + "mkdir": stagingFS.mkdir == nil, + "writeFile": stagingFS.writeFile == nil, + "create": stagingFS.create == nil, + "createTemp": stagingFS.createTemp == nil, + } { + if missing { + t.Errorf("stagingFS.%s is nil", name) + } + } + + root := t.TempDir() + dir := filepath.Join(root, "one") + if err := stagingFS.mkdir(dir, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + file := filepath.Join(dir, "a.txt") + if err := stagingFS.writeFile(file, []byte("v0"), 0o600); err != nil { + t.Fatalf("writeFile: %v", err) + } + if _, err := stagingFS.stat(file); err != nil { + t.Fatalf("stat: %v", err) + } + if _, err := stagingFS.lstat(file); err != nil { + t.Fatalf("lstat: %v", err) + } + if got, err := stagingFS.readFile(file); err != nil || string(got) != "v0" { + t.Fatalf("readFile = %q, %v, want %q", got, err, "v0") + } + entries, err := stagingFS.readDir(dir) + if err != nil || len(entries) != 1 || entries[0].Name() != "a.txt" { + t.Fatalf("readDir = %v, %v, want one entry a.txt", entries, err) + } + tmp, err := stagingFS.createTemp(dir, "seam-*") + if err != nil { + t.Fatalf("createTemp: %v", err) + } + tmpName := tmp.Name() + if err := tmp.Close(); err != nil { + t.Fatalf("close temp: %v", err) + } + opened, err := stagingFS.create(tmpName, os.O_WRONLY, 0o600) + if err != nil { + t.Fatalf("create: %v", err) + } + if err := opened.Close(); err != nil { + t.Fatalf("close opened: %v", err) + } + moved := filepath.Join(root, "two") + if err := stagingFS.rename(dir, moved); err != nil { + t.Fatalf("rename: %v", err) + } + if _, err := stagingFS.stat(dir); !os.IsNotExist(err) { + t.Fatalf("stat of the old name = %v, want not-exist", err) + } + if err := stagingFS.removeAll(moved); err != nil { + t.Fatalf("removeAll: %v", err) + } + if _, err := stagingFS.stat(moved); !os.IsNotExist(err) { + t.Fatalf("stat after removeAll = %v, want not-exist", err) + } +} + +// The matrix needs both selections: "fail the rename whose source is seq 2's +// backup" is an argument match, and "fail the second remove" is an ordinal one. +// An ordinal alone cannot express the first, and an argument match cannot +// express a step that runs twice on the same path. +func TestInjectFaultMatchesByArgumentAndByOrdinal(t *testing.T) { + injected := errors.New("injected seam failure") + root := t.TempDir() + + t.Run("by argument", func(t *testing.T) { + injectFault(t, "rename", func(args ...string) bool { + return strings.HasSuffix(args[0], filepath.Join("seq", "backup")) + }, injected) + for _, name := range []string{"one", "seq", "three"} { + from := filepath.Join(root, name, "backup") + if err := os.MkdirAll(from, 0o700); err != nil { + t.Fatal(err) + } + err := stagingFS.rename(from, filepath.Join(root, name, "moved")) + if name == "seq" { + if !errors.Is(err, injected) { + t.Fatalf("rename of %s = %v, want the injected failure", from, err) + } + continue + } + if err != nil { + t.Fatalf("rename of %s = %v, want it to pass through", from, err) + } + } + }) + + t.Run("by ordinal", func(t *testing.T) { + injectFault(t, "removeAll", nthCall(2), injected) + for i := 1; i <= 3; i++ { + dir := filepath.Join(root, fmt.Sprintf("remove-%d", i)) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + err := stagingFS.removeAll(dir) + if i == 2 { + if !errors.Is(err, injected) { + t.Fatalf("removeAll call %d = %v, want the injected failure", i, err) + } + if _, statErr := os.Stat(dir); statErr != nil { + t.Fatalf("the failed call must not have removed %s: %v", dir, statErr) + } + continue + } + if err != nil { + t.Fatalf("removeAll call %d = %v, want it to pass through", i, err) + } + if _, statErr := os.Stat(dir); !os.IsNotExist(statErr) { + t.Fatalf("removeAll call %d left %s behind: %v", i, dir, statErr) + } + } + }) + + if got, want := reflect.ValueOf(stagingFS.removeAll).Pointer(), reflect.ValueOf(os.RemoveAll).Pointer(); got != want { + t.Fatal("stagingFS.removeAll was not restored after the subtest's cleanup") + } + if got, want := reflect.ValueOf(stagingFS.rename).Pointer(), reflect.ValueOf(os.Rename).Pointer(); got != want { + t.Fatal("stagingFS.rename was not restored after the subtest's cleanup") + } + + // One seam, two goroutines: the ordinal must come from a counter that is + // safe to increment concurrently, or the racing rows this seam exists for + // report a race instead of a result. + t.Run("concurrently", func(t *testing.T) { + injectFault(t, "stat", nthCall(1), injected) + start := make(chan struct{}) + errs := make([]error, 2) + var wg sync.WaitGroup + for i := range errs { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + _, errs[i] = stagingFS.stat(root) + }(i) + } + close(start) + wg.Wait() + failed := 0 + for _, err := range errs { + if errors.Is(err, injected) { + failed++ + } else if err != nil { + t.Fatalf("the call that was not selected = %v, want it to pass through", err) + } + } + if failed != 1 { + t.Fatalf("%d of 2 concurrent calls failed, want exactly 1", failed) + } + }) +} + +// A blocked step has to stay blocked: the racing rows park one transaction +// inside a step and only then let the second one reach the same destination. +func TestBlockStepHoldsTheCallUntilRelease(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "blocked") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + release := blockStep(t, "removeAll") + done := make(chan error, 1) + go func() { done <- stagingFS.removeAll(dir) }() + select { + case err := <-done: + t.Fatalf("removeAll returned %v before release", err) + case <-time.After(20 * time.Millisecond): + } + release() + release() + if err := <-done; err != nil { + t.Fatalf("removeAll after release: %v", err) + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Fatalf("the released call did not run: %v", err) + } +} diff --git a/internal/dictation/download.go b/internal/dictation/download.go index 64758350f..056671ff5 100644 --- a/internal/dictation/download.go +++ b/internal/dictation/download.go @@ -776,6 +776,39 @@ func resolveEnginePaths(engineDir string, targetWindows bool) (bin, server strin // and a directory mtime tracks the install's contents, not its promotion. The // number is one past the highest already beside this install, so a clock moving // backward cannot invert it the way the wall-clock stamp it replaces could. +// fsOps is the filesystem seam the promotion path, the holder allocator, and +// recovery take every step through. A step that calls os directly cannot be made +// to fail, and the crash each of these steps exists to survive is then only +// reasoned about. Every field is one call, so a test can fail the second remove +// or the rename whose source is one particular holder and leave the rest real. +type fsOps struct { + rename func(from, to string) error + removeAll func(path string) error + stat func(name string) (os.FileInfo, error) + lstat func(name string) (os.FileInfo, error) + readDir func(name string) ([]os.DirEntry, error) + readFile func(name string) ([]byte, error) + mkdir func(name string, perm os.FileMode) error + writeFile func(name string, data []byte, perm os.FileMode) error + create func(name string, flag int, perm os.FileMode) (*os.File, error) + createTemp func(dir, pattern string) (*os.File, error) +} + +// holderFS is the seam every filesystem step in the promotion path goes through. +// Tests swap a field and restore it; nothing else writes to it. +var holderFS = fsOps{ + rename: os.Rename, + removeAll: os.RemoveAll, + stat: os.Stat, + lstat: os.Lstat, + readDir: os.ReadDir, + readFile: os.ReadFile, + mkdir: os.Mkdir, + writeFile: os.WriteFile, + create: os.OpenFile, + createTemp: os.CreateTemp, +} + const holderSuffix = ".previous-" // holderStamp reads back the ordering number in a holder name, reporting false @@ -812,7 +845,7 @@ const holderSeqAttempts = 10000 // always allocates above it. Holders for a different install are a separate // sequence and are never compared against this one. func nextHolderSeq(destDir string) (int64, error) { - entries, err := os.ReadDir(filepath.Dir(destDir)) + entries, err := holderFS.readDir(filepath.Dir(destDir)) if err != nil { return 0, err } @@ -848,7 +881,7 @@ func createSequencedHolder(destDir string, n int64) (string, error) { if stamp, ok := holderStamp(destDir, path); !ok || stamp != n { return "", fmt.Errorf("holder name %q does not read back as sequence %d", filepath.Base(path), n) } - err := os.Mkdir(path, 0o700) + err := holderFS.mkdir(path, 0o700) if err == nil { return path, nil } @@ -874,7 +907,7 @@ func createSequencedHolder(destDir string, n int64) (string, error) { // promotion published there" are different claims, and only the second one // makes a holder beside it superseded. func restoreInterruptedPromotion(destDir string, published func(string) bool) { - if _, err := os.Lstat(destDir); err == nil { + if _, err := holderFS.lstat(destDir); err == nil { // destDir is live. A holder is only ever filled by renaming destDir // aside, so a destDir holding a USABLE install means a later promotion // published over every holder beside it. Those are superseded copies of @@ -893,7 +926,7 @@ func restoreInterruptedPromotion(destDir string, published func(string) bool) { // same branch and tries again. if published != nil && published(destDir) { for _, holder := range holdersBeside(destDir) { - _ = os.RemoveAll(holder) + _ = holderFS.removeAll(holder) } } return @@ -917,15 +950,15 @@ func restoreInterruptedPromotion(destDir string, published func(string) bool) { }) for _, holder := range holders { install := filepath.Join(holder, "install") - if _, err := os.Stat(install); err != nil { + if _, err := holderFS.stat(install); err != nil { continue } - if err := renameStagedDir(install, destDir); err != nil { + if err := holderFS.rename(install, destDir); err != nil { continue } // Only the holder this install came out of is removed; an older one is // left for a human, never deleted on a guess about which is current. - _ = os.RemoveAll(holder) + _ = holderFS.removeAll(holder) return } } @@ -938,7 +971,7 @@ func holdersBeside(destDir string) []string { // and a '[' anywhere in it opens a character class to Glob, which then // matches nothing and strands the install this exists to put back. parent := filepath.Dir(destDir) - entries, err := os.ReadDir(parent) + entries, err := holderFS.readDir(parent) if err != nil { return nil } @@ -962,12 +995,12 @@ func promoteStagedDir(stageDir, destDir, label string) error { cleanupHolder := true defer func() { if holder != "" && cleanupHolder { - _ = os.RemoveAll(holder) + _ = holderFS.removeAll(holder) } }() restore := func() error { return nil } - if _, err := os.Lstat(destDir); err == nil { + if _, err := holderFS.lstat(destDir); err == nil { var seq int64 seq, err = nextHolderSeq(destDir) if err != nil { @@ -978,15 +1011,15 @@ func promoteStagedDir(stageDir, destDir, label string) error { return fmt.Errorf("setting aside previous %s install: %w", label, err) } previous := filepath.Join(holder, "install") - if err := os.Rename(destDir, previous); err != nil { + if err := holderFS.rename(destDir, previous); err != nil { return fmt.Errorf("setting aside previous %s install: %w", label, err) } - restore = func() error { return renameStagedDir(previous, destDir) } + restore = func() error { return holderFS.rename(previous, destDir) } } else if !os.IsNotExist(err) { return fmt.Errorf("checking previous %s install: %w", label, err) } - if err := renameStagedDir(stageDir, destDir); err != nil { + if err := holderFS.rename(stageDir, destDir); err != nil { if restoreErr := restore(); restoreErr != nil { cleanupHolder = false return fmt.Errorf("promoting staged %s: %w (previous install left in %s: %v)", label, err, holder, restoreErr) @@ -996,10 +1029,6 @@ func promoteStagedDir(stageDir, destDir, label string) error { return nil } -// renameStagedDir moves a directory into its published location. It is a var so -// tests can force the failures the restore path above exists for. -var renameStagedDir = os.Rename - // extractTarBz2 unpacks a bzip2-compressed tar into destDir, guarding against // path-traversal entries. It reports extraction progress by how much of the // (compressed) archive it has consumed — the same MB scale as the download, so diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index 0e88fbd35..99374f386 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -12,10 +12,14 @@ import ( "net/http/httptest" "os" "path/filepath" + "reflect" "runtime" + "strconv" "strings" "sync" + "sync/atomic" "testing" + "time" ) // Tiny tar.bz2 fixtures (generated in-repo): the engine has top/bin/sherpa-onnx- @@ -303,16 +307,17 @@ func TestPromoteStagedDirKeepsThePreviousInstallWhenPromotionFails(t *testing.T) stagedTree(t, dest, "old") stage := stagedTree(t, filepath.Join(root, "stage"), "new") - real := renameStagedDir - calls := 0 - renameStagedDir = func(from, to string) error { - calls++ - if calls == 1 { + // Fail only the promotion, so the restore that follows it still runs. The + // set-aside rename goes through the same seam now, and it has to stay real + // or there is no previous install to put back. + real := holderFS.rename + holderFS.rename = func(from, to string) error { + if from == stage { return errors.New("injected promotion failure") } return real(from, to) } - t.Cleanup(func() { renameStagedDir = real }) + t.Cleanup(func() { holderFS.rename = real }) if err := promoteStagedDir(stage, dest, "engine"); err == nil { t.Fatal("a failed promotion must be reported") @@ -333,9 +338,16 @@ func TestPromoteStagedDirKeepsTheSetAsideCopyWhenRestoreAlsoFails(t *testing.T) stagedTree(t, dest, "old") stage := stagedTree(t, filepath.Join(root, "stage"), "new") - real := renameStagedDir - renameStagedDir = func(string, string) error { return errors.New("injected rename failure") } - t.Cleanup(func() { renameStagedDir = real }) + // The promotion and the restore are the two renames into dest; the set-aside + // that fills the holder stays real, or there is no retained copy to assert on. + real := holderFS.rename + holderFS.rename = func(from, to string) error { + if to == dest { + return errors.New("injected rename failure") + } + return real(from, to) + } + t.Cleanup(func() { holderFS.rename = real }) err := promoteStagedDir(stage, dest, "engine") if err == nil { @@ -641,10 +653,15 @@ func interruptPromotion(t *testing.T, destDir, label string) { if err := os.MkdirAll(stage, 0o755); err != nil { t.Fatal(err) } - real := renameStagedDir - renameStagedDir = func(string, string) error { return errors.New("injected rename failure") } + real := holderFS.rename + holderFS.rename = func(from, to string) error { + if to == destDir { + return errors.New("injected rename failure") + } + return real(from, to) + } err := promoteStagedDir(stage, destDir, label) - renameStagedDir = real + holderFS.rename = real if err == nil { t.Fatalf("a promotion whose publish and restore both fail must report an error") } @@ -1040,3 +1057,345 @@ func TestRestoreInterruptedPromotionSkipsAHolderWithNoInstall(t *testing.T) { t.Errorf("the restored holder should be cleared, got %v", err) } } + +// ---- filesystem seam ------------------------------------------------------- + +// injectFault swaps one field of holderFS for a wrapper that fails the call +// whose arguments satisfy match and passes every other call through to the real +// primitive. The call ordinal is appended to the arguments handed to match, so +// nthCall can select by call order without a second counter, and it is counted +// with an atomic because the racing scenarios drive one seam from two +// goroutines. The whole struct is restored in t.Cleanup, so a test that injects +// twice unwinds in reverse. +func injectFault(t *testing.T, step string, match func(args ...string) bool, err error) { + t.Helper() + var calls atomic.Int64 + real := holderFS + t.Cleanup(func() { holderFS = real }) + fire := func(args ...string) bool { + n := calls.Add(1) + if match == nil { + return true + } + return match(append(args, strconv.FormatInt(n, 10))...) + } + switch step { + case "rename": + holderFS.rename = func(from, to string) error { + if fire(from, to) { + return err + } + return real.rename(from, to) + } + case "removeAll": + holderFS.removeAll = func(path string) error { + if fire(path) { + return err + } + return real.removeAll(path) + } + case "stat": + holderFS.stat = func(name string) (os.FileInfo, error) { + if fire(name) { + return nil, err + } + return real.stat(name) + } + case "lstat": + holderFS.lstat = func(name string) (os.FileInfo, error) { + if fire(name) { + return nil, err + } + return real.lstat(name) + } + case "readDir": + holderFS.readDir = func(name string) ([]os.DirEntry, error) { + if fire(name) { + return nil, err + } + return real.readDir(name) + } + case "readFile": + holderFS.readFile = func(name string) ([]byte, error) { + if fire(name) { + return nil, err + } + return real.readFile(name) + } + case "mkdir": + holderFS.mkdir = func(name string, perm os.FileMode) error { + if fire(name) { + return err + } + return real.mkdir(name, perm) + } + case "writeFile": + holderFS.writeFile = func(name string, data []byte, perm os.FileMode) error { + if fire(name) { + return err + } + return real.writeFile(name, data, perm) + } + case "create": + holderFS.create = func(name string, flag int, perm os.FileMode) (*os.File, error) { + if fire(name) { + return nil, err + } + return real.create(name, flag, perm) + } + case "createTemp": + holderFS.createTemp = func(dir, pattern string) (*os.File, error) { + if fire(dir, pattern) { + return nil, err + } + return real.createTemp(dir, pattern) + } + default: + t.Fatalf("injectFault: unknown step %q", step) + } +} + +// nthCall selects the nth call through an injected seam. It reads the ordinal +// injectFault appends to the arguments rather than counting for itself, so the +// two share one counter and a concurrent test has one place to be correct. +func nthCall(n int) func(args ...string) bool { + return func(args ...string) bool { + return len(args) > 0 && args[len(args)-1] == strconv.Itoa(n) + } +} + +// blockStep holds every call to step until the returned release runs, which is +// how a test parks one transaction mid-step and lets another reach the same +// destination. Releasing twice is safe, so a test can release on the happy path +// and still defer it. +func blockStep(t *testing.T, step string) (release func()) { + t.Helper() + gate := make(chan struct{}) + var once sync.Once + release = func() { once.Do(func() { close(gate) }) } + t.Cleanup(release) + injectFaultGate(t, step, gate) + return release +} + +// injectFaultGate is blockStep's half of the swap, split out so the wrapper it +// installs sits beside the fault wrappers above. +func injectFaultGate(t *testing.T, step string, gate <-chan struct{}) { + t.Helper() + real := holderFS + t.Cleanup(func() { holderFS = real }) + switch step { + case "rename": + holderFS.rename = func(from, to string) error { + <-gate + return real.rename(from, to) + } + case "removeAll": + holderFS.removeAll = func(path string) error { + <-gate + return real.removeAll(path) + } + case "stat": + holderFS.stat = func(name string) (os.FileInfo, error) { + <-gate + return real.stat(name) + } + default: + t.Fatalf("blockStep: unknown step %q", step) + } +} + +// The seam has to be the real filesystem until a test swaps a field. A nil field +// is a panic in whatever production path reaches it first, and a field that no +// longer behaves like its os function makes every test above it prove nothing. +func TestHolderFSDefaultsAreTheRealPrimitives(t *testing.T) { + for name, missing := range map[string]bool{ + "rename": holderFS.rename == nil, + "removeAll": holderFS.removeAll == nil, + "stat": holderFS.stat == nil, + "lstat": holderFS.lstat == nil, + "readDir": holderFS.readDir == nil, + "readFile": holderFS.readFile == nil, + "mkdir": holderFS.mkdir == nil, + "writeFile": holderFS.writeFile == nil, + "create": holderFS.create == nil, + "createTemp": holderFS.createTemp == nil, + } { + if missing { + t.Errorf("holderFS.%s is nil", name) + } + } + + root := t.TempDir() + dir := filepath.Join(root, "one") + if err := holderFS.mkdir(dir, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + file := filepath.Join(dir, "a.txt") + if err := holderFS.writeFile(file, []byte("v0"), 0o600); err != nil { + t.Fatalf("writeFile: %v", err) + } + if _, err := holderFS.stat(file); err != nil { + t.Fatalf("stat: %v", err) + } + if _, err := holderFS.lstat(file); err != nil { + t.Fatalf("lstat: %v", err) + } + if got, err := holderFS.readFile(file); err != nil || string(got) != "v0" { + t.Fatalf("readFile = %q, %v, want %q", got, err, "v0") + } + entries, err := holderFS.readDir(dir) + if err != nil || len(entries) != 1 || entries[0].Name() != "a.txt" { + t.Fatalf("readDir = %v, %v, want one entry a.txt", entries, err) + } + tmp, err := holderFS.createTemp(dir, "seam-*") + if err != nil { + t.Fatalf("createTemp: %v", err) + } + tmpName := tmp.Name() + if err := tmp.Close(); err != nil { + t.Fatalf("close temp: %v", err) + } + opened, err := holderFS.create(tmpName, os.O_WRONLY, 0o600) + if err != nil { + t.Fatalf("create: %v", err) + } + if err := opened.Close(); err != nil { + t.Fatalf("close opened: %v", err) + } + moved := filepath.Join(root, "two") + if err := holderFS.rename(dir, moved); err != nil { + t.Fatalf("rename: %v", err) + } + if _, err := holderFS.stat(dir); !os.IsNotExist(err) { + t.Fatalf("stat of the old name = %v, want not-exist", err) + } + if err := holderFS.removeAll(moved); err != nil { + t.Fatalf("removeAll: %v", err) + } + if _, err := holderFS.stat(moved); !os.IsNotExist(err) { + t.Fatalf("stat after removeAll = %v, want not-exist", err) + } +} + +// The matrix needs both selections: "fail the rename whose source is seq 2's +// set-aside copy" is an argument match, and "fail the second remove" is an +// ordinal one. +// An ordinal alone cannot express the first, and an argument match cannot +// express a step that runs twice on the same path. +func TestInjectFaultMatchesByArgumentAndByOrdinal(t *testing.T) { + injected := errors.New("injected seam failure") + root := t.TempDir() + + t.Run("by argument", func(t *testing.T) { + injectFault(t, "rename", func(args ...string) bool { + return strings.HasSuffix(args[0], filepath.Join("seq", "backup")) + }, injected) + for _, name := range []string{"one", "seq", "three"} { + from := filepath.Join(root, name, "backup") + if err := os.MkdirAll(from, 0o700); err != nil { + t.Fatal(err) + } + err := holderFS.rename(from, filepath.Join(root, name, "moved")) + if name == "seq" { + if !errors.Is(err, injected) { + t.Fatalf("rename of %s = %v, want the injected failure", from, err) + } + continue + } + if err != nil { + t.Fatalf("rename of %s = %v, want it to pass through", from, err) + } + } + }) + + t.Run("by ordinal", func(t *testing.T) { + injectFault(t, "removeAll", nthCall(2), injected) + for i := 1; i <= 3; i++ { + dir := filepath.Join(root, fmt.Sprintf("remove-%d", i)) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + err := holderFS.removeAll(dir) + if i == 2 { + if !errors.Is(err, injected) { + t.Fatalf("removeAll call %d = %v, want the injected failure", i, err) + } + if _, statErr := os.Stat(dir); statErr != nil { + t.Fatalf("the failed call must not have removed %s: %v", dir, statErr) + } + continue + } + if err != nil { + t.Fatalf("removeAll call %d = %v, want it to pass through", i, err) + } + if _, statErr := os.Stat(dir); !os.IsNotExist(statErr) { + t.Fatalf("removeAll call %d left %s behind: %v", i, dir, statErr) + } + } + }) + + if got, want := reflect.ValueOf(holderFS.removeAll).Pointer(), reflect.ValueOf(os.RemoveAll).Pointer(); got != want { + t.Fatal("holderFS.removeAll was not restored after the subtest's cleanup") + } + if got, want := reflect.ValueOf(holderFS.rename).Pointer(), reflect.ValueOf(os.Rename).Pointer(); got != want { + t.Fatal("holderFS.rename was not restored after the subtest's cleanup") + } + + // One seam, two goroutines: the ordinal must come from a counter that is + // safe to increment concurrently, or the racing rows this seam exists for + // report a race instead of a result. + t.Run("concurrently", func(t *testing.T) { + injectFault(t, "stat", nthCall(1), injected) + start := make(chan struct{}) + errs := make([]error, 2) + var wg sync.WaitGroup + for i := range errs { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + _, errs[i] = holderFS.stat(root) + }(i) + } + close(start) + wg.Wait() + failed := 0 + for _, err := range errs { + if errors.Is(err, injected) { + failed++ + } else if err != nil { + t.Fatalf("the call that was not selected = %v, want it to pass through", err) + } + } + if failed != 1 { + t.Fatalf("%d of 2 concurrent calls failed, want exactly 1", failed) + } + }) +} + +// A blocked step has to stay blocked: the racing rows park one transaction +// inside a step and only then let the second one reach the same destination. +func TestBlockStepHoldsTheCallUntilRelease(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "blocked") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + release := blockStep(t, "removeAll") + done := make(chan error, 1) + go func() { done <- holderFS.removeAll(dir) }() + select { + case err := <-done: + t.Fatalf("removeAll returned %v before release", err) + case <-time.After(20 * time.Millisecond): + } + release() + release() + if err := <-done; err != nil { + t.Fatalf("removeAll after release: %v", err) + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Fatalf("the released call did not run: %v", err) + } +} From efc472ea201b695a8184fa7c678ae22a019b1221 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:03:44 -0500 Subject: [PATCH 21/34] fix(daemon/remote): prove which transaction owns a staged backup, and whether it committed The staging dir recorded only which link its backup belonged to, so recovery could tell the destination but not whether this code created the directory, nor whether the publish rename ever landed. Both facts have to come off disk, because the next process starts with no memory of the one that crashed. extractBundle now writes a transaction marker naming its kind, destination and sequence before the first destructive rename, and creates a commit flag after the publish rename and before cleanup. The marker is published by renaming a complete temp file into place, so a crash cannot leave a half-written one that parses. A commit flag that cannot be created leaves the staging dir alone and reports it, alongside the existing retain for a failed restore. restoreStagedBackup reads the marker rather than the link file the writer no longer produces; classifying on it is the following change, not this one. All three swap renames go through fsutil.RenameWithRetry, which absorbs a momentary Windows sharing violation and nothing longer. --- internal/daemon/remote/bundle.go | 153 ++++++++++++--- internal/daemon/remote/bundle_test.go | 269 +++++++++++++++++++++++++- 2 files changed, 394 insertions(+), 28 deletions(-) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index 503f9e359..e54f0d3f5 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -22,6 +22,7 @@ import ( "time" "github.com/Gitlawb/zero/internal/daemon" + "github.com/Gitlawb/zero/internal/fsutil" "github.com/Gitlawb/zero/internal/lockutil" ) @@ -193,9 +194,21 @@ const keptPrefix = ".kept-" // across processes. Dot-prefixed for the same reason stagingPrefix is. const lockDirName = ".extract-locks" -// stagingLinkFile records, inside a staging dir, which link the backup beside it -// belongs to. Without it a crash leaves an orphan nothing can attribute. -const stagingLinkFile = "link" +// stagingMarkerFile records, inside a staging dir, the transaction that created +// it: its kind, the link it is for, and the sequence in its own name. Without it +// a crash leaves an orphan nothing can attribute, and recovery has to retain a +// copy it can never name. +const stagingMarkerFile = "txn" + +// committedFile records, inside a staging dir, that the publish rename landed. +// It is the only evidence that the copy in backup was superseded, so recovery +// has to keep a backup that carries no flag. +const committedFile = "committed" + +// txnKindBundleExtract is the marker kind extractBundle writes. A marker naming +// another kind belongs to another site's transaction and is not this code's to +// act on. +const txnKindBundleExtract = "bundle-extract" // extractLockPoll is how often a cross-process extract lock is retried. const extractLockPoll = 50 * time.Millisecond @@ -505,12 +518,12 @@ func restoreStagedBackup(dir string, s stagedExtract, restored map[string]staged if _, err := stagingFS.stat(backup); err != nil { return false } - raw, err := stagingFS.readFile(filepath.Join(staging, stagingLinkFile)) + m, err := readMarker(staging) if err != nil { - logf("remote: staged tree in %s has no link marker; leaving it in place", staging) + logf("remote: staged tree in %s has no usable transaction marker (%v); leaving it in place", staging, err) return true } - id, err := sanitizeLinkID(string(raw)) + id, err := sanitizeLinkID(m.Dest) if err != nil { logf("remote: staged tree in %s names an invalid link (%v); leaving it in place", staging, err) return true @@ -583,14 +596,91 @@ func parkKeptBackup(dir, staging, id string, logf func(string, ...any)) { } } +// txnMarker is what a staging dir carries to prove which transaction created it. +// Seq repeats the number in the directory's own name: a marker that disagrees +// with its name proves nothing about who wrote either, so the pair is what makes +// the directory owned rather than the name alone. +type txnMarker struct { + Kind string `json:"kind"` + Dest string `json:"dest"` + Seq int64 `json:"seq"` +} + +// errMarkerMissing reports that a staging dir carries no marker at all, which is +// a different fact from a marker that could not be read: the first is a crash +// before the marker was written, the second is a filesystem fault. Deciding +// between them off a bare error would fold a fault into "not ours". +var errMarkerMissing = errors.New("remote: staging dir carries no transaction marker") + +// writeMarker publishes m into dir under stagingMarkerFile, through a temp file +// in that same directory so the rename is the only step a reader can observe. A +// plain write can be torn by a crash, and a half-written marker parses as +// nothing while still occupying the name recovery reads to decide ownership. +func writeMarker(dir string, m txnMarker) error { + payload, err := json.Marshal(m) + if err != nil { + return err + } + tmp, err := stagingFS.createTemp(dir, stagingMarkerFile+"-*") + if err != nil { + return err + } + name := tmp.Name() + // Sync before the rename: the rename can reach disk ahead of the bytes it + // names, which is exactly the partial marker the temp file exists to avoid. + if _, err := tmp.Write(payload); err != nil { + _ = tmp.Close() + _ = stagingFS.removeAll(name) + return err + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + _ = stagingFS.removeAll(name) + return err + } + if err := tmp.Close(); err != nil { + _ = stagingFS.removeAll(name) + return err + } + if err := fsutil.RenameWithRetry(name, filepath.Join(dir, stagingMarkerFile), stagingFS.rename); err != nil { + _ = stagingFS.removeAll(name) + return err + } + return nil +} + +// readMarker reads back what writeMarker published. A dir with no marker yields +// errMarkerMissing; every other failure is wrapped, so a caller can tell "this +// was never ours" from "this could not be read", which are opposite decisions. +func readMarker(dir string) (txnMarker, error) { + raw, err := stagingFS.readFile(filepath.Join(dir, stagingMarkerFile)) + if err != nil { + if os.IsNotExist(err) { + return txnMarker{}, errMarkerMissing + } + return txnMarker{}, fmt.Errorf("read transaction marker in %s: %w", dir, err) + } + var m txnMarker + if err := json.Unmarshal(raw, &m); err != nil { + return txnMarker{}, fmt.Errorf("parse transaction marker in %s: %w", dir, err) + } + return m, nil +} + // extractBundle clones bundleFile into a staging dir beside dest, then swaps the // clone into place (replacing any prior extraction for this link id). git clone -// needs a non-existent target, hence the staging dir. The live tree is moved -// aside rather than deleted and is put back if the publish fails, so on every -// error return dest holds either the prior extraction or the new one, never -// neither. Swapping a directory is two renames and cannot be made atomic, so a -// crash between them leaves dest absent with the prior tree in staging/backup; -// nothing reaps that on restart. logf may be nil. +// needs a non-existent target, hence the staging dir. The steps run in this +// order: allocate the sequenced staging dir, write its transaction marker, clone +// into it, set the live tree aside, publish the clone, record the commit flag, +// remove the staging dir. The marker precedes the first destructive rename and +// the flag follows the publish, so every copy this leaves on disk names the +// transaction that wrote it and says whether that transaction committed. The +// live tree is moved aside rather than deleted and is put back if the publish +// fails, so on every error return dest holds either the prior extraction or the +// new one, never neither. Swapping a directory is two renames and cannot be made +// atomic, so a crash between them leaves dest absent with the prior tree in +// staging/backup, which the marker is what lets recovery attribute and put back. +// logf may be nil. func extractBundle(ctx context.Context, bundleFile, dest string, logf func(string, ...any)) error { if logf == nil { logf = func(string, ...any) {} @@ -611,9 +701,10 @@ func extractBundle(ctx context.Context, bundleFile, dest string, logf func(strin // recovery tell an older leftover staging dir from a newer one. It is one // past the highest number already in the directory, claimed by exclusive // creation, so a concurrent extract for another link cannot take the same - // value and a clock that moves backward cannot invert the order. Numbers - // written by released versions are wall-clock nanoseconds; seeding from the - // highest present keeps those sorting older with no migration step. + // value and a clock that moves backward cannot invert the order. Every number + // on disk is one of these per-directory sequences: no released version wrote + // a stamped name at all (v0.8.0 staged under os.MkdirTemp's random suffix), + // so nothing here is ordering against a wall clock it inherited. seq, err := nextStagingSeq(parent) if err != nil { return err @@ -636,6 +727,16 @@ func extractBundle(ctx context.Context, bundleFile, dest string, logf func(strin logf("remote: could not remove bundle staging dir %s: %v", staging, err) } }() + // Attribute the staging dir before anything else touches the filesystem: from + // here on every failure can leave a copy of a tree behind, and a copy no + // marker names is one recovery can neither put back nor ever reclaim. The + // name is the authority on the sequence, because createSequencedStagingDir + // walks up from seq when a name is taken, and a marker that disagrees with + // its own directory name proves nothing. + claimed, _ := stagingStamp(filepath.Base(staging)) + if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: filepath.Base(dest), Seq: claimed}); err != nil { + return err + } cloneCtx, cancelClone := context.WithTimeout(ctx, gitTimeout) defer cancelClone() cloneDest := filepath.Join(staging, "repo") @@ -645,18 +746,13 @@ func extractBundle(ctx context.Context, bundleFile, dest string, logf func(strin // Every rename stays inside parent, so none of them crosses a filesystem. backup := filepath.Join(staging, "backup") - // Record the link before moving its tree, so a crash in the swap window - // leaves something recoverBundleDir can attribute and put back. - if err := stagingFS.writeFile(filepath.Join(staging, stagingLinkFile), []byte(filepath.Base(dest)), 0o600); err != nil { - return err - } restore := func() error { return nil } - if err := stagingFS.rename(dest, backup); err == nil { - restore = func() error { return stagingFS.rename(backup, dest) } + if err := fsutil.RenameWithRetry(dest, backup, stagingFS.rename); err == nil { + restore = func() error { return fsutil.RenameWithRetry(backup, dest, stagingFS.rename) } } else if !os.IsNotExist(err) { return err } - if err := stagingFS.rename(cloneDest, dest); err != nil { + if err := fsutil.RenameWithRetry(cloneDest, dest, stagingFS.rename); err != nil { if restoreErr := restore(); restoreErr != nil { // dest is empty and the only copy of the prior tree is the backup, // so keep staging rather than deleting the tree on the way out. @@ -665,6 +761,17 @@ func extractBundle(ctx context.Context, bundleFile, dest string, logf func(strin } return err } + // The flag is the only evidence that the copy in backup was published over. + // Without it that copy has to be kept, so a failure here costs one retained + // tree and never the publish, which has already landed and is reported as + // the success it is. + flag, err := stagingFS.create(filepath.Join(staging, committedFile), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + cleanupStaging = false + logf("remote: published %s but could not record the commit flag in %s: %v", dest, staging, err) + return nil + } + _ = flag.Close() return nil } diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index 0360e96c9..94fd52f6d 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -15,6 +15,7 @@ import ( "strings" "sync" "sync/atomic" + "syscall" "testing" "time" @@ -440,13 +441,270 @@ func TestExtractBundleKeepsBackupWhenRestoreAlsoFails(t *testing.T) { } } +// soleStaging returns the one staging dir under dir, and fails if there is not +// exactly one. A test that asserts on "the" staging dir has to prove that is +// what it found, or a second one left by an earlier step reads as a pass. +func soleStaging(t *testing.T, dir string) string { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + found := []string{} + for _, e := range entries { + if e.IsDir() && strings.HasPrefix(e.Name(), stagingPrefix) { + found = append(found, filepath.Join(dir, e.Name())) + } + } + if len(found) != 1 { + t.Fatalf("staging dirs in %s = %v, want exactly one", dir, found) + } + return found[0] +} + +// The marker is what lets recovery name the copy a crash leaves behind, so it +// has to be on disk before anything destructive runs. A clone that fails is the +// first step after it, and the marker must already be readable there. +func TestExtractBundleWritesTheMarkerBeforeTheClone(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + // The staging dir is cleaned up on the way out of a failed extract, so the + // cleanup is blocked to leave the on-disk state this test is about. + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + err := extractBundle(context.Background(), filepath.Join(dir, "missing.bundle"), dest, nil) + if err == nil { + t.Fatal("cloning a bundle that does not exist must fail") + } + + staging := soleStaging(t, dir) + seq, ok := stagingStamp(filepath.Base(staging)) + if !ok { + t.Fatalf("staging name %s carries no sequence", staging) + } + m, err := readMarker(staging) + if err != nil { + t.Fatalf("read the marker the extract should have written: %v", err) + } + if want := (txnMarker{Kind: txnKindBundleExtract, Dest: "proj-1", Seq: seq}); m != want { + t.Fatalf("marker = %+v, want %+v", m, want) + } + if _, err := os.Stat(filepath.Join(staging, "backup")); !os.IsNotExist(err) { + t.Errorf("nothing should have been set aside before the clone, got %v", err) + } +} + +// A torn marker occupies the name recovery reads and parses as nothing, so the +// bytes reach their name through a rename or not at all. +func TestExtractBundleMarkerIsAtomic(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + injected := errors.New("injected marker rename failure") + injectFault(t, "rename", func(args ...string) bool { return filepath.Base(args[1]) == stagingMarkerFile }, injected) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, nil) + if !errors.Is(err, injected) { + t.Fatalf("extract = %v, want the injected marker rename failure", err) + } + + staging := soleStaging(t, dir) + entries, readErr := os.ReadDir(staging) + if readErr != nil { + t.Fatal(readErr) + } + for _, e := range entries { + if e.Name() == stagingMarkerFile { + t.Errorf("a marker that never got renamed must not occupy %s", filepath.Join(staging, stagingMarkerFile)) + } + } + got, readErr := os.ReadFile(filepath.Join(dest, "a.txt")) + if readErr != nil || string(got) != "v0" { + t.Fatalf("dest a.txt = %q, err %v, want the prior tree %q untouched", got, readErr, "v0") + } + if _, err := os.Stat(filepath.Join(staging, "backup")); !os.IsNotExist(err) { + t.Errorf("a failed marker must stop the extract before the set-aside, got %v", err) + } +} + +// The flag is the evidence a later recovery needs that the copy in backup was +// superseded. It only means that if the publish already landed. +func TestExtractBundleCreatesTheCommitFlagAfterPublish(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + // Blocking the final cleanup is what leaves the committed staging dir on + // disk to assert on; a cleanup failure is logged, never returned. + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + if err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, nil); err != nil { + t.Fatalf("extract = %v, want the cleanup failure to be logged, not returned", err) + } + + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("dest a.txt = %q, err %v, want %q", got, err, "v1") + } + staging := soleStaging(t, dir) + got, err = os.ReadFile(filepath.Join(staging, "backup", "a.txt")) + if err != nil || string(got) != "v0" { + t.Fatalf("backup a.txt = %q, err %v, want the prior tree %q", got, err, "v0") + } + if _, err := os.Stat(filepath.Join(staging, committedFile)); err != nil { + t.Fatalf("a published extract must record the commit flag: %v", err) + } +} + +// Without the flag the copy in backup has no proof it was superseded, so the +// staging dir stays: deleting it would drop the only evidence recovery has. +func TestExtractBundleFailedCommitFlagKeepsTheStagingDir(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + injectFault(t, "create", func(args ...string) bool { return filepath.Base(args[0]) == committedFile }, errors.New("injected commit flag failure")) + var logged []string + err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, func(format string, args ...any) { + logged = append(logged, fmt.Sprintf(format, args...)) + }) + if err != nil { + t.Fatalf("extract = %v, want a committed publish reported as success", err) + } + + got, readErr := os.ReadFile(filepath.Join(dest, "a.txt")) + if readErr != nil || string(got) != "v1" { + t.Fatalf("dest a.txt = %q, err %v, want the published tree %q", got, readErr, "v1") + } + staging := soleStaging(t, dir) + got, readErr = os.ReadFile(filepath.Join(staging, "backup", "a.txt")) + if readErr != nil || string(got) != "v0" { + t.Fatalf("backup a.txt = %q, err %v, want the retained prior tree %q", got, readErr, "v0") + } + if _, err := os.Stat(filepath.Join(staging, committedFile)); !os.IsNotExist(err) { + t.Errorf("the flag failed to be created, so it must not exist: %v", err) + } + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, staging) }) { + t.Errorf("the retained staging dir should be named in the log, got %v", logged) + } +} + +// A publish that never landed supersedes nothing, so no flag may appear beside +// the copy that is still the live tree's only other copy. +func TestExtractBundlePublishFailureLeavesNoCommitFlag(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + injected := errors.New("injected publish failure") + injectFault(t, "rename", func(args ...string) bool { return filepath.Base(args[0]) == "repo" }, injected) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, nil) + if !errors.Is(err, injected) { + t.Fatalf("extract = %v, want the injected publish failure", err) + } + + got, readErr := os.ReadFile(filepath.Join(dest, "a.txt")) + if readErr != nil || string(got) != "v0" { + t.Fatalf("dest a.txt = %q, err %v, want the restored prior tree %q", got, readErr, "v0") + } + staging := soleStaging(t, dir) + if _, err := os.Stat(filepath.Join(staging, committedFile)); !os.IsNotExist(err) { + t.Errorf("a failed publish must leave no commit flag, got %v", err) + } +} + +// Every rename goes through fsutil.RenameWithRetry, which absorbs the momentary +// sharing violation an open file produces on Windows. The retry is Windows-only +// (fsutil/rename.go: ten attempts, 10 ms apart, guarded by runtime.GOOS), so on +// every other platform the assertion is only that the error comes back +// unchanged. This guard is falsifiable on Windows CI alone. +func TestExtractBundleRenamesRetryOnTransientErrors(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + // The third rename of the second extract is the publish: the marker, then + // the set-aside, then the swap into dest. + injected := &os.LinkError{Op: "rename", Old: "repo", New: dest, Err: syscall.Errno(32)} + injectFault(t, "rename", nthCall(3), injected) + + err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, nil) + want := "v0" + if runtime.GOOS == "windows" { + if err != nil { + t.Fatalf("extract = %v, want the sharing violation absorbed by the retry", err) + } + want = "v1" + } else if !errors.Is(err, injected) { + t.Fatalf("extract = %v, want the sharing violation returned unchanged", err) + } + got, readErr := os.ReadFile(filepath.Join(dest, "a.txt")) + if readErr != nil || string(got) != want { + t.Fatalf("dest a.txt = %q, err %v, want %q", got, readErr, want) + } +} + +// Missing and unreadable are opposite decisions for recovery: the first says the +// directory was never this code's to act on, the second says the filesystem +// failed and nothing may be concluded. Folding one into the other is what turns +// a fault into a licence. +func TestReadMarkerDistinguishesMissingFromUnreadable(t *testing.T) { + dir := t.TempDir() + + if _, err := readMarker(dir); !errors.Is(err, errMarkerMissing) { + t.Fatalf("readMarker of a dir with no marker = %v, want errMarkerMissing", err) + } + + // What a released version left at this name is not JSON; it parses as + // nothing, which is a marker that exists and cannot be trusted. + if err := os.WriteFile(filepath.Join(dir, stagingMarkerFile), []byte("proj-1"), 0o600); err != nil { + t.Fatal(err) + } + _, err := readMarker(dir) + if err == nil { + t.Fatal("an unparseable marker must be an error") + } + if errors.Is(err, errMarkerMissing) { + t.Fatalf("an unparseable marker = %v, want an error distinct from errMarkerMissing", err) + } + + injected := errors.New("injected marker read failure") + injectFault(t, "readFile", nil, injected) + _, err = readMarker(dir) + if !errors.Is(err, injected) { + t.Fatalf("readMarker over a failing read = %v, want the injected failure", err) + } + if errors.Is(err, errMarkerMissing) { + t.Fatalf("a read failure = %v, want an error distinct from errMarkerMissing", err) + } +} + // The link marker is read off disk, so a hostile or corrupt one must not steer a // rename anywhere outside the bundle dir. func TestRecoverBundleDirRefusesAMarkerThatEscapesTheBundleDir(t *testing.T) { for _, marker := range []string{"../evil", "/etc/evil", "..", "a/b", ".hidden"} { dir := t.TempDir() staging := plantInterruptedExtract(t, dir, "proj-1", "v0") - if err := os.WriteFile(filepath.Join(staging, stagingLinkFile), []byte(marker), 0o600); err != nil { + if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: marker}); err != nil { t.Fatal(err) } outside := filepath.Join(filepath.Dir(dir), "evil") @@ -478,7 +736,7 @@ func plantInterruptedExtract(t *testing.T, bundleDir, linkID, content string) st if err := os.WriteFile(filepath.Join(backup, "a.txt"), []byte(content), 0o644); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(staging, stagingLinkFile), []byte(linkID), 0o600); err != nil { + if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: linkID}); err != nil { t.Fatal(err) } return staging @@ -531,7 +789,7 @@ func TestRecoverBundleDirDropsBackupWhenTheLinkAlreadyHasATree(t *testing.T) { func TestRecoverBundleDirKeepsABackupItCannotAttribute(t *testing.T) { dir := t.TempDir() staging := plantInterruptedExtract(t, dir, "proj-1", "v0") - if err := os.Remove(filepath.Join(staging, stagingLinkFile)); err != nil { + if err := os.Remove(filepath.Join(staging, stagingMarkerFile)); err != nil { t.Fatal(err) } @@ -541,7 +799,7 @@ func TestRecoverBundleDirKeepsABackupItCannotAttribute(t *testing.T) { if _, err := os.Stat(filepath.Join(staging, "backup", "a.txt")); err != nil { t.Errorf("an unattributable tree must be left alone, not deleted: %v", err) } - if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "no link marker") }) { + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "no usable transaction marker") }) { t.Errorf("an unattributable tree should be reported, got %v", logged) } } @@ -779,7 +1037,8 @@ func stageBackup(t *testing.T, dir, name, linkID, content string, stamp int64) s if err := os.WriteFile(filepath.Join(backup, "a.txt"), []byte(content), 0o644); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(staging, stagingLinkFile), []byte(linkID), 0o600); err != nil { + seq, _ := stagingStamp(filepath.Base(staging)) + if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: linkID, Seq: seq}); err != nil { t.Fatal(err) } return staging From 60351fcc7069cbd0f74a75deae4816079a87aa92 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:03:44 -0500 Subject: [PATCH 22/34] fix(dictation): hold one lock across a destination's whole install lifecycle Nothing serialized the dictation install. Recovery could decide a destination was usable, a promotion in another process could move that destination into a holder, and recovery could then delete the holder as superseded, so the failed publish had no rollback source and the install was gone. Allocating a unique holder name arbitrates names, not the state stored under them. Each engine and model destination now has a cross-process lock, taken before recovery and held across the download decision, the promotion and the cleanup, so the check and the action it authorizes cannot be split. The lock is a handle threaded through recovery, download and promotion; each refuses to run on a nil handle or one naming another destination, so a call site that forgets it fails closed rather than open. Engine and model lock separately and never at once. A wait that outlives its budget re-checks the destination before failing: the likeliest reason the wait ran out is that the other process finished the same install, and only a destination that is still unusable reports one in progress. --- internal/dictation/download.go | 203 ++++++++++++-- internal/dictation/download_test.go | 420 ++++++++++++++++++++++++++-- 2 files changed, 580 insertions(+), 43 deletions(-) diff --git a/internal/dictation/download.go b/internal/dictation/download.go index 056671ff5..c75cc6598 100644 --- a/internal/dictation/download.go +++ b/internal/dictation/download.go @@ -20,6 +20,9 @@ import ( "slices" "strconv" "strings" + "time" + + "github.com/Gitlawb/zero/internal/lockutil" ) // Auto-download of the local engine + a default model (opt-in, behind a confirm @@ -489,32 +492,40 @@ func EnsureLocalEngine(ctx context.Context, opts DownloadOptions) (EngineCompone apiBase = defaultAPIBase } - engineDir := filepath.Join(opts.DestRoot, "engine-"+version+"-"+key) + engineName := "engine-" + version + "-" + key + engineDir := filepath.Join(opts.DestRoot, engineName) targetWindows := strings.HasPrefix(key, "windows-") // Resolve through the tarball's flattened subdir so an ALREADY-extracted // engine is found and not needlessly re-downloaded (the idempotency check). - // A previous run may have been stopped mid-promotion, leaving the only - // install in a holder beside engineDir. Put it back before deciding whether - // anything needs downloading. - restoreInterruptedPromotion(engineDir, func(dir string) bool { + enginePublished := func(dir string) bool { bin, _ := resolveEnginePaths(dir, targetWindows) return fileExists(bin) - }) - binPath, serverPath := resolveEnginePaths(engineDir, targetWindows) - if !fileExists(binPath) { + } + // The engine lock covers recovery, the decision to download, and the + // promotion, and it is released before the model lock is taken. Holding + // both would give two concurrent installs of different models an ordering + // to get wrong for no gain. + if err := withDestinationLock(ctx, opts.DestRoot, engineName, enginePublished, func(txn *destTxn) error { + // A previous run may have been stopped mid-promotion, leaving the only + // install in a holder beside engineDir. Put it back before deciding + // whether anything needs downloading. + restoreInterruptedPromotion(txn, engineDir, enginePublished, progress) + if enginePublished(engineDir) { + return nil + } pinned := "" if version == DefaultSherpaVersion && !opts.skipPinned { pinned = pinnedEngineDigest[key] } asset, err := resolveAsset(ctx, client, apiBase, version, "sherpa-onnx-", suffix) if err != nil { - return EngineComponents{}, err - } - if err := downloadVerifyExtract(ctx, client, asset, pinned, false, "Engine", engineDir, progress); err != nil { - return EngineComponents{}, err + return err } - binPath, serverPath = resolveEnginePaths(engineDir, targetWindows) + return downloadVerifyExtract(ctx, client, asset, pinned, false, "Engine", engineDir, txn, progress) + }); err != nil { + return EngineComponents{}, err } + binPath, serverPath := resolveEnginePaths(engineDir, targetWindows) if !fileExists(binPath) { return EngineComponents{}, fmt.Errorf("dictation download: engine binary not found after extraction in %s", engineDir) } @@ -528,14 +539,18 @@ func EnsureLocalEngine(ctx context.Context, opts DownloadOptions) (EngineCompone modelDirName = "model-moonshine-tiny-en-int8" } modelDir := filepath.Join(opts.DestRoot, modelDirName) - // promoteStagedDir is shared with the model, so a stop mid-promotion leaves - // the model in a holder too. Put it back before deciding anything is - // missing: without this an offline user has no download to fall back on. - restoreInterruptedPromotion(modelDir, dirHasModel) - if !dirHasModel(modelDir) { + if err := withDestinationLock(ctx, opts.DestRoot, modelDirName, dirHasModel, func(txn *destTxn) error { + // promoteStagedDir is shared with the model, so a stop mid-promotion + // leaves the model in a holder too. Put it back before deciding + // anything is missing: without this an offline user has no download to + // fall back on. + restoreInterruptedPromotion(txn, modelDir, dirHasModel, progress) + if dirHasModel(modelDir) { + return nil + } asset, err := resolveAsset(ctx, client, apiBase, modelReleaseTag, modelName, "") if err != nil { - return EngineComponents{}, err + return err } modelPinned := opts.ModelPinnedDigest // Only fall back to the built-in digest for the DEFAULT model — a different @@ -555,9 +570,9 @@ func EnsureLocalEngine(ctx context.Context, opts DownloadOptions) (EngineCompone // download (the model release predates GitHub's per-asset digests). The // engine BINARY above never allows this — a native executable is always // digest-verified. - if err := downloadVerifyExtract(ctx, client, asset, modelPinned, true, modelLabel, modelDir, progress); err != nil { - return EngineComponents{}, err - } + return downloadVerifyExtract(ctx, client, asset, modelPinned, true, modelLabel, modelDir, txn, progress) + }); err != nil { + return EngineComponents{}, err } resolvedModel := modelDir if !hasTokensFile(resolvedModel) { @@ -632,7 +647,12 @@ func resolveAsset(ctx context.Context, client *http.Client, apiBase, tag, namePr // (against the API digest, and — when pinned is set — a cross-check that the // resolved digest equals the audited value), and extracts the tar.bz2. A // mismatch aborts before anything is extracted or run. -func downloadVerifyExtract(ctx context.Context, client *http.Client, asset resolvedAsset, pinned string, allowUnverified bool, label, destDir string, progress func(string)) error { +func downloadVerifyExtract(ctx context.Context, client *http.Client, asset resolvedAsset, pinned string, allowUnverified bool, label, destDir string, txn *destTxn, progress func(string)) error { + // Refused here as well as in promoteStagedDir, so a caller that forgot the + // lock does not get a full download and extraction before finding out. + if !txn.holds(destDir) { + return fmt.Errorf("dictation download: refusing to install %s: no install lock is held for %s", label, destDir) + } // Determine the digest to verify against: the API digest and the pinned digest // must agree when both exist; at least one must exist unless allowUnverified // (models are data files from the official release — TLS-only is acceptable; @@ -712,7 +732,7 @@ func downloadVerifyExtract(ctx context.Context, client *http.Client, asset resol if err := extractTarBz2(tmpPath, stageDir, "Extracting "+label, progress); err != nil { return err } - if err := promoteStagedDir(stageDir, destDir, label); err != nil { + if err := promoteStagedDir(txn, stageDir, destDir, label, progress); err != nil { return err } cleanupStage = false @@ -809,6 +829,119 @@ var holderFS = fsOps{ createTemp: os.CreateTemp, } +// errInstallInProgress reports that another process held this destination's +// Install lock for the whole wait budget and the destination is still not +// usable. It is not an install failure: nothing was attempted, and the caller +// reports it and moves on. +var errInstallInProgress = errors.New("another install of this component is already running") + +// installLockWait bounds how long a caller waits for another process to finish +// with a destination. It covers a download, so a budget shorter than the +// critical section it guards is a scheduled failure. A var so a test can shorten +// it; nothing else assigns to it. +var installLockWait = 2 * time.Minute + +// installLockPoll is the retry interval while waiting. The lock is advisory and +// held by an open file handle, so there is nothing to wake us and polling is how +// the wait ends. +const installLockPoll = 50 * time.Millisecond + +// installLockDir holds one lock file per destination. It is a sibling of the +// destinations rather than a file beside each one, so a lock file is never +// mistaken for part of an install and never moves with one. +const installLockDir = ".install-locks" + +// destTxn is the handle proving its holder owns the Install lock for one +// destination. Recovery and promotion move and delete whole installs, so both +// take one: without it a second process can delete the copy this one is about +// to roll back to. +type destTxn struct { + root string + dest string + lock *lockutil.FileLock +} + +// destDir is the destination this handle locks. Callers keep passing their own +// destDir alongside the handle so a handle for a DIFFERENT destination is +// representable, which is what makes the mismatch testable rather than assumed. +func (t *destTxn) destDir() string { return filepath.Join(t.root, t.dest) } + +// holds reports whether this handle actually locks destDir. A nil handle and a +// handle for another destination both fail closed. +func (t *destTxn) holds(destDir string) bool { + return t != nil && t.lock != nil && t.destDir() == destDir +} + +// release drops the Install lock. Idempotent, so a caller can release early and +// still defer it. +func (t *destTxn) release() { + if t == nil || t.lock == nil { + return + } + _ = t.lock.Release() +} + +// lockDestination takes the Install lock for one destination under destRoot, +// waiting out another process that holds it until ctx ends or the budget runs +// out. The budget expiring is errInstallInProgress, which the caller answers by +// re-checking the destination rather than by failing. +func lockDestination(ctx context.Context, destRoot, dest string) (*destTxn, error) { + // A destination is one path component. Any other shape puts the lock file + // somewhere else, and two callers for the same destination would then lock + // different inodes and both proceed. + if dest == "" || dest != filepath.Base(dest) || dest == "." || dest == ".." { + return nil, fmt.Errorf("dictation download: %q is not an install destination name", dest) + } + // destRoot is otherwise created lazily by the download this lock covers, and + // lockutil opens the root with O_DIRECTORY, so on a first run there would be + // no directory to take the lock in. + if err := os.MkdirAll(destRoot, 0o755); err != nil { + return nil, err + } + lockDir := filepath.Join(destRoot, installLockDir) + if err := holderFS.mkdir(lockDir, 0o700); err != nil && !errors.Is(err, fs.ErrExist) { + return nil, err + } + lockPath := filepath.Join(lockDir, dest+".lock") + deadline := time.Now().Add(installLockWait) + for { + lock, err := lockutil.TryAcquireFileLockAt(destRoot, lockPath) + if err == nil { + return &destTxn{root: destRoot, dest: dest, lock: lock}, nil + } + if !errors.Is(err, lockutil.ErrLockHeld) { + return nil, err + } + // Waiting is the normal case: the holder is almost always another + // process installing this very component, and failing on the first + // contended try would turn that into a user-visible install failure. + if !time.Now().Before(deadline) { + return nil, fmt.Errorf("%w: %s is locked by another process", errInstallInProgress, dest) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(installLockPoll): + } + } +} + +// withDestinationLock runs fn under dest's Install lock. A wait that runs out is +// not by itself a failure: the likeliest reason is that the other process +// finished this exact install, so usable re-checks the destination before the +// caller is told anything is wrong. +func withDestinationLock(ctx context.Context, destRoot, dest string, usable func(string) bool, fn func(*destTxn) error) error { + txn, err := lockDestination(ctx, destRoot, dest) + if err != nil { + if errors.Is(err, errInstallInProgress) && usable(filepath.Join(destRoot, dest)) { + return nil + } + return err + } + defer txn.release() + return fn(txn) +} + const holderSuffix = ".previous-" // holderStamp reads back the ordering number in a holder name, reporting false @@ -906,7 +1039,16 @@ func createSequencedHolder(destDir string, n int64) (string, error) { // use. Recovery needs it because "there is something at destDir" and "a // promotion published there" are different claims, and only the second one // makes a holder beside it superseded. -func restoreInterruptedPromotion(destDir string, published func(string) bool) { +func restoreInterruptedPromotion(txn *destTxn, destDir string, published func(string) bool, report func(string)) { + // Recovery moves and deletes whole installs. Without destDir's Install lock + // it can restore a holder a promotion in another process is parked on and + // remove it, leaving that promotion's rollback nothing to put back. + if !txn.holds(destDir) { + if report != nil { + report(fmt.Sprintf("Skipping recovery of %s: no install lock is held for it", filepath.Base(destDir))) + } + return + } if _, err := holderFS.lstat(destDir); err == nil { // destDir is live. A holder is only ever filled by renaming destDir // aside, so a destDir holding a USABLE install means a later promotion @@ -990,7 +1132,16 @@ func holdersBeside(destDir string) []string { // the way first; it is set aside rather than deleted, and put back if the // promotion fails, so destDir is never left holding no install at all. If the // restore fails too, the set-aside copy is kept and named in the error. -func promoteStagedDir(stageDir, destDir, label string) error { +func promoteStagedDir(txn *destTxn, stageDir, destDir, label string, report func(string)) error { + // Same reason recovery refuses: the window between the two renames has the + // only copy of the install in a holder, and nothing else keeps a concurrent + // recovery out of it. + if !txn.holds(destDir) { + if report != nil { + report(fmt.Sprintf("Refusing to install %s: no install lock is held for %s", label, filepath.Base(destDir))) + } + return fmt.Errorf("promoting staged %s: no install lock is held for %s", label, destDir) + } holder := "" cleanupHolder := true defer func() { diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index 99374f386..25a3661c0 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -20,6 +20,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/Gitlawb/zero/internal/lockutil" ) // Tiny tar.bz2 fixtures (generated in-repo): the engine has top/bin/sherpa-onnx- @@ -267,7 +269,7 @@ func TestPromoteStagedDirReplacesAPreviousInstall(t *testing.T) { stagedTree(t, dest, "old") stage := stagedTree(t, filepath.Join(root, "stage"), "new") - if err := promoteStagedDir(stage, dest, "engine"); err != nil { + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil); err != nil { t.Fatalf("promote: %v", err) } got, err := os.ReadFile(filepath.Join(dest, "engine")) @@ -290,7 +292,7 @@ func TestPromoteStagedDirWorksWithNoPreviousInstall(t *testing.T) { dest := filepath.Join(root, "engine-dir") stage := stagedTree(t, filepath.Join(root, "stage"), "new") - if err := promoteStagedDir(stage, dest, "engine"); err != nil { + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil); err != nil { t.Fatalf("promote: %v", err) } got, err := os.ReadFile(filepath.Join(dest, "engine")) @@ -319,7 +321,7 @@ func TestPromoteStagedDirKeepsThePreviousInstallWhenPromotionFails(t *testing.T) } t.Cleanup(func() { holderFS.rename = real }) - if err := promoteStagedDir(stage, dest, "engine"); err == nil { + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil); err == nil { t.Fatal("a failed promotion must be reported") } got, err := os.ReadFile(filepath.Join(dest, "engine")) @@ -349,7 +351,7 @@ func TestPromoteStagedDirKeepsTheSetAsideCopyWhenRestoreAlsoFails(t *testing.T) } t.Cleanup(func() { holderFS.rename = real }) - err := promoteStagedDir(stage, dest, "engine") + err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil) if err == nil { t.Fatal("a failed promotion must be reported") } @@ -391,7 +393,7 @@ func TestRestoreInterruptedPromotionPutsTheInstallBack(t *testing.T) { t.Fatal(err) } - restoreInterruptedPromotion(dest, testPublished) + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) got, err := os.ReadFile(filepath.Join(dest, "engine")) if err != nil { @@ -422,7 +424,7 @@ func TestRestoreInterruptedPromotionReapsAHolderSupersededByALiveInstall(t *test stranded := plantHolder(t, dest, 100, "old") older := plantHolder(t, dest, 50, "older") - restoreInterruptedPromotion(dest, testPublished) + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) // The live install is never touched. This is the assertion that matters most. got, err := os.ReadFile(filepath.Join(dest, "engine")) @@ -484,7 +486,7 @@ func TestRestoreInterruptedPromotionKeepsAHolderWhenDestIsNotAUsableInstall(t *t tc.seed(t, dest) holder := plantHolder(t, dest, 100, "the only copy") - restoreInterruptedPromotion(dest, tc.usable) + restoreInterruptedPromotion(lockFor(t, dest), dest, tc.usable, nil) got, err := os.ReadFile(filepath.Join(holder, "install", "engine")) if err != nil || string(got) != "the only copy" { @@ -523,7 +525,7 @@ func TestRestoreInterruptedPromotionLeavesAnExistingDestAlone(t *testing.T) { t.Fatal(err) } - restoreInterruptedPromotion(dest, testPublished) + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) got, err := os.ReadFile(filepath.Join(dest, "engine")) if tc.live == "" { @@ -597,6 +599,20 @@ func offlineAPIBase(t *testing.T) string { return url } +// lockFor takes the Install lock the promotion path now requires, for the +// destination the test is about to drive. Released in cleanup, and release is +// idempotent, so a test that goes on to call EnsureLocalEngine releases it +// early rather than waiting out its own lock. +func lockFor(t *testing.T, destDir string) *destTxn { + t.Helper() + txn, err := lockDestination(context.Background(), filepath.Dir(destDir), filepath.Base(destDir)) + if err != nil { + t.Fatalf("locking %s: %v", destDir, err) + } + t.Cleanup(txn.release) + return txn +} + // testPublished is the "is this a real install" predicate these tests use: the // fixtures write an "engine" file, so its presence is what publication means here. func testPublished(dir string) bool { @@ -628,7 +644,7 @@ func TestRestoreInterruptedPromotionPrefersTheNewestHolder(t *testing.T) { stale := plantHolder(t, dest, 100, "stale") current := plantHolder(t, dest, 200, "current") - restoreInterruptedPromotion(dest, testPublished) + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) got, err := os.ReadFile(filepath.Join(dest, "engine")) if err != nil || string(got) != "current" { @@ -647,7 +663,7 @@ func TestRestoreInterruptedPromotionPrefersTheNewestHolder(t *testing.T) { // between its two renames leaves: destDir absent, the only install in a holder // promoteStagedDir named. Both renames fail, so nothing is put back in process // and the holder is retained rather than cleaned up. -func interruptPromotion(t *testing.T, destDir, label string) { +func interruptPromotion(t *testing.T, txn *destTxn, destDir, label string) { t.Helper() stage := destDir + ".incoming" if err := os.MkdirAll(stage, 0o755); err != nil { @@ -660,7 +676,7 @@ func interruptPromotion(t *testing.T, destDir, label string) { } return real(from, to) } - err := promoteStagedDir(stage, destDir, label) + err := promoteStagedDir(txn, stage, destDir, label, nil) holderFS.rename = real if err == nil { t.Fatalf("a promotion whose publish and restore both fail must report an error") @@ -725,7 +741,12 @@ func TestEnsureLocalEngineRecoversARealInterruptedPromotionOffline(t *testing.T) } target := tc.dirFor(t, dest, comp) - interruptPromotion(t, target, tc.label) + targetTxn := lockFor(t, target) + interruptPromotion(t, targetTxn, target, tc.label) + // Released before EnsureLocalEngine below, which takes the same + // lock: the wait is cross-process but flock conflicts between two + // opens in one process too. + targetTxn.release() holders := holdersFor(t, target) if len(holders) != 1 { t.Fatalf("want exactly one holder beside %s, got %v", target, holders) @@ -778,7 +799,7 @@ func TestRestoreInterruptedPromotionFindsHoldersUnderAnAwkwardPath(t *testing.T) t.Fatal(err) } - restoreInterruptedPromotion(dest, testPublished) + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) got, err := os.ReadFile(filepath.Join(dest, "engine")) if err != nil || string(got) != "kept" { @@ -806,9 +827,10 @@ func TestRestoreInterruptedPromotionPrefersTheRealNewerInstallOverAFutureStamped stale := plantHolder(t, dest, farFuture, "stale") // The real transaction sets "new" aside and never publishes. - interruptPromotion(t, dest, "engine") + txn := lockFor(t, dest) + interruptPromotion(t, txn, dest, "engine") - restoreInterruptedPromotion(dest, testPublished) + restoreInterruptedPromotion(txn, dest, testPublished, nil) got, err := os.ReadFile(filepath.Join(dest, "engine")) if err != nil || string(got) != "new" { @@ -1018,7 +1040,7 @@ func TestRestoreInterruptedPromotionPrefersAStampedHolderOverAnUnstampedOne(t *t } stamped := plantHolder(t, dest, 100, "stamped") - restoreInterruptedPromotion(dest, testPublished) + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) got, err := os.ReadFile(filepath.Join(dest, "engine")) if err != nil || string(got) != "stamped" { @@ -1047,7 +1069,7 @@ func TestRestoreInterruptedPromotionSkipsAHolderWithNoInstall(t *testing.T) { t.Fatal(err) } - restoreInterruptedPromotion(dest, testPublished) + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) got, err := os.ReadFile(filepath.Join(dest, "engine")) if err != nil || string(got) != "kept" { @@ -1399,3 +1421,367 @@ func TestBlockStepHoldsTheCallUntilRelease(t *testing.T) { t.Fatalf("the released call did not run: %v", err) } } + +// ---- the per-destination Install lock --------------------------------------- + +// shortenInstallLockWait cuts the wait budget so a test can observe an expiry +// without spending the production two minutes on it. +func shortenInstallLockWait(t *testing.T, d time.Duration) { + t.Helper() + real := installLockWait + installLockWait = d + t.Cleanup(func() { installLockWait = real }) +} + +// gateRename holds the one rename from -> to until the returned release runs and +// then fails it. blockStep gates every call to a step, which cannot tell a +// promotion's set-aside rename from the publish that follows it, and this test +// needs the transaction parked precisely between the two. +func gateRename(t *testing.T, from, to string, fail error) (reached <-chan struct{}, release func()) { + t.Helper() + gate := make(chan struct{}) + hit := make(chan struct{}) + var once, announced sync.Once + release = func() { once.Do(func() { close(gate) }) } + real := holderFS + t.Cleanup(release) + t.Cleanup(func() { holderFS = real }) + holderFS.rename = func(f, to2 string) error { + if f != from || to2 != to { + return real.rename(f, to2) + } + announced.Do(func() { close(hit) }) + <-gate + return fail + } + return hit, release +} + +// A promotion moves a whole install out of the way and puts it back, so running +// one without the destination's Install lock is what lets a second process +// delete the copy this one is about to roll back to. A handle for a DIFFERENT +// destination is the same defect wearing a lock, which is why destDir stays an +// explicit parameter. +func TestPromoteStagedDirRefusesWithoutADestinationLock(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "model-a") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + if err := promoteStagedDir(nil, stage, dest, "engine", nil); err == nil { + t.Error("a promotion with no Install lock must be refused") + } + if err := promoteStagedDir(lockFor(t, filepath.Join(root, "model-b")), stage, dest, "engine", nil); err == nil { + t.Error("a promotion holding another destination's Install lock must be refused") + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "old" { + t.Errorf("the destination was touched without its lock: %q err %v", got, err) + } + got, err = os.ReadFile(filepath.Join(stage, "engine")) + if err != nil || string(got) != "new" { + t.Errorf("the staged copy was touched without the lock: %q err %v", got, err) + } +} + +// Recovery restores and deletes whole installs, so it fails closed the same way, +// and it says so: it has no error to return and a silent skip reads as "there +// was nothing to recover". +func TestRestoreInterruptedPromotionRefusesWithoutADestinationLock(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + holder := plantHolder(t, dest, 100, "the only copy") + + var reported []string + restoreInterruptedPromotion(nil, dest, testPublished, func(s string) { reported = append(reported, s) }) + if len(reported) == 0 { + t.Error("a recovery pass that refuses to run must say so") + } + restoreInterruptedPromotion(lockFor(t, filepath.Join(root, "engine-other")), dest, testPublished, nil) + + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Errorf("recovery ran without the destination's lock: %v", err) + } + got, err := os.ReadFile(filepath.Join(holder, "install", "engine")) + if err != nil || string(got) != "the only copy" { + t.Errorf("the holder was touched without the lock: %q err %v", got, err) + } +} + +// The download half fails closed too, or a caller that forgot the lock still +// reaches promoteStagedDir with a full extraction behind it. +func TestDownloadVerifyExtractRefusesWithoutADestinationLock(t *testing.T) { + srv := fakeReleaseServer(t, engineSHA, modelSHA) + root := t.TempDir() + dest := filepath.Join(root, "engine-test-linux-amd64") + // A real asset the download would otherwise install, so the refusal is what + // keeps the destination empty rather than a download that was going to fail. + asset, err := resolveAsset(context.Background(), http.DefaultClient, srv.URL, "test", "sherpa-onnx-", engineAssetSuffix["linux-amd64"]) + if err != nil { + t.Fatal(err) + } + + var requests atomic.Int64 + client := &http.Client{Transport: countingTransport(&requests)} + if err := downloadVerifyExtract(context.Background(), client, asset, "", false, "Engine", dest, nil, func(string) {}); err == nil { + t.Fatal("a download with no Install lock must be refused") + } + // Refused before the transfer, not after it: the promotion at the end would + // refuse too, and then a caller that forgot the lock still pays for the + // whole download every start. + if got := requests.Load(); got != 0 { + t.Errorf("%d asset request(s) were made without the Install lock, want 0", got) + } + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Errorf("the destination was installed to without its lock: %v", err) + } +} + +// countingTransport counts the asset transfers a client performs, which is how a +// test tells "refused before the download" from "refused after it". +func countingTransport(n *atomic.Int64) http.RoundTripper { + return roundTripFunc(func(r *http.Request) (*http.Response, error) { + n.Add(1) + return http.DefaultTransport.RoundTrip(r) + }) +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +// The lock is the cross-process one, so a second open of the same lock file has +// to conflict, and a caller has to wait the holder out rather than fail on the +// first contended try. +func TestLockDestinationSerializesAcrossProcesses(t *testing.T) { + root := t.TempDir() + lockDir := filepath.Join(root, installLockDir) + if err := os.MkdirAll(lockDir, 0o700); err != nil { + t.Fatal(err) + } + held, err := lockutil.TryAcquireFileLockAt(root, filepath.Join(lockDir, "engine-a.lock")) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + start := time.Now() + if _, err := lockDestination(ctx, root, "engine-a"); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("lockDestination = %v, want the context error while another holder has the lock", err) + } + if waited := time.Since(start); waited < 150*time.Millisecond { + t.Errorf("lockDestination gave up after %v; it must wait the holder out", waited) + } + + if err := held.Release(); err != nil { + t.Fatal(err) + } + txn, err := lockDestination(context.Background(), root, "engine-a") + if err != nil { + t.Fatalf("the lock was not free after the holder released it: %v", err) + } + txn.release() +} + +// A destination name is one path component. Anything else would put the lock +// file somewhere other than beside its peers, so two callers for the same +// destination could hold different inodes and both proceed. +func TestLockDestinationRefusesADestinationThatIsNotABaseName(t *testing.T) { + root := t.TempDir() + for _, dest := range []string{"", ".", "..", "a/b", filepath.Join("..", "escape")} { + if _, err := lockDestination(context.Background(), root, dest); err == nil { + t.Errorf("lockDestination(%q) must be refused", dest) + } + } +} + +// The likeliest reason a caller finds the destination locked is that another +// process is installing the very thing it wants. Failing on the first contended +// try turns a normal race into a user-visible install failure. +func TestEnsureLocalEngineWaitsForAConcurrentInstall(t *testing.T) { + srv := fakeReleaseServer(t, engineSHA, modelSHA) + root := t.TempDir() + txn := lockFor(t, filepath.Join(root, "engine-test-linux-amd64")) + go func() { + time.Sleep(150 * time.Millisecond) + txn.release() + }() + + comp, err := EnsureLocalEngine(context.Background(), DownloadOptions{ + DestRoot: root, EngineVersion: "test", APIBase: srv.URL, platformKey: "linux-amd64", skipPinned: true, + }) + if err != nil { + t.Fatalf("an install that waited out a concurrent one must succeed: %v", err) + } + if !fileExists(comp.BinaryPath) { + t.Errorf("engine binary missing at %q", comp.BinaryPath) + } +} + +// The review's P2: a promotion parked between its two renames has the only copy +// of the install in a holder and destDir absent. A recovery pass that runs in +// that window restores the holder and removes it, and the promotion's rollback +// then has nothing to put back. The lock is what makes that window unreachable. +func TestRecoveryDoesNotDeleteALiveRollbackSource(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "live") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + reached, release := gateRename(t, stage, dest, errors.New("injected publish failure")) + txn := lockFor(t, dest) + promoted := make(chan error, 1) + go func() { promoted <- promoteStagedDir(txn, stage, dest, "engine", nil) }() + select { + case <-reached: + case <-time.After(10 * time.Second): + t.Fatal("the promotion never reached its publish rename") + } + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Fatalf("the parked promotion should leave %s absent, got %v", dest, err) + } + holders := holdersFor(t, dest) + if len(holders) != 1 { + t.Fatalf("want exactly one holder beside %s, got %v", dest, holders) + } + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + err := withDestinationLock(ctx, root, filepath.Base(dest), testPublished, func(rec *destTxn) error { + restoreInterruptedPromotion(rec, dest, testPublished, nil) + return nil + }) + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("recovery = %v, want the context error while a promotion holds the destination", err) + } + got, readErr := os.ReadFile(filepath.Join(holders[0], "install", "engine")) + if readErr != nil || string(got) != "live" { + t.Fatalf("recovery took the copy the promotion still needs: %q err %v", got, readErr) + } + + release() + if err := <-promoted; err == nil { + t.Fatal("the injected publish failure must be reported") + } + got, readErr = os.ReadFile(filepath.Join(dest, "engine")) + if readErr != nil || string(got) != "live" { + t.Fatalf("the rollback did not put the previous install back: %q err %v", got, readErr) + } + txn.release() +} + +// The two destinations are locked one after the other. Taking both up front +// would let a model install someone else is running block the engine step, which +// is shared and usually already done. +func TestEnsureLocalEngineLocksEngineAndModelSeparately(t *testing.T) { + shortenInstallLockWait(t, 200*time.Millisecond) + srv := fakeReleaseServer(t, engineSHA, modelSHA) + root := t.TempDir() + modelTxn := lockFor(t, filepath.Join(root, "model-moonshine-tiny-en-int8")) + + _, err := EnsureLocalEngine(context.Background(), DownloadOptions{ + DestRoot: root, EngineVersion: "test", APIBase: srv.URL, platformKey: "linux-amd64", skipPinned: true, + }) + if !errors.Is(err, errInstallInProgress) { + t.Fatalf("EnsureLocalEngine = %v, want the model step to report an install in progress", err) + } + bin, _ := resolveEnginePaths(filepath.Join(root, "engine-test-linux-amd64"), false) + if !fileExists(bin) { + t.Errorf("the engine step must not wait on the model's lock: %s missing", bin) + } + modelTxn.release() +} + +// The lock directory sits beside the installs and holds nothing anyone else +// needs to read. +func TestLockDirectoryIsOwnerOnly(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Go permission bits do not map to Windows ACLs") + } + root := t.TempDir() + requireUmaskAllowsWiderThan0700(t, root) + lockFor(t, filepath.Join(root, "engine-a")) + + info, err := os.Stat(filepath.Join(root, installLockDir)) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o700 { + t.Errorf("%s mode = %o, want 0700", installLockDir, got) + } +} + +// DestRoot is created lazily by the download that follows, and lockutil opens +// the root with O_DIRECTORY, so a first run has no root to lock in. +func TestLockDestinationCreatesTheDestRootFirst(t *testing.T) { + root := filepath.Join(t.TempDir(), "stt", "nested") + txn, err := lockDestination(context.Background(), root, "engine-a") + if err != nil { + t.Fatalf("lockDestination on a DestRoot that does not exist yet: %v", err) + } + defer txn.release() + if _, err := os.Stat(filepath.Join(root, installLockDir, "engine-a.lock")); err != nil { + t.Errorf("the lock file should sit under the created root: %v", err) + } +} + +// A wait that runs out is not an install failure. The other process was almost +// certainly installing the same thing, so the destination is re-checked before +// anyone is told anything went wrong, and what comes back when it really is +// missing is a named outcome the caller can report rather than a raw timeout. +func TestEnsureLocalEngineTreatsAnExpiredWaitAsBenign(t *testing.T) { + shortenInstallLockWait(t, 100*time.Millisecond) + + t.Run("a usable destination means the other install finished", func(t *testing.T) { + root := t.TempDir() + engineDir := filepath.Join(root, "engine-test-linux-amd64") + bin, server := enginePaths(engineDir, false) + if err := os.MkdirAll(filepath.Dir(bin), 0o755); err != nil { + t.Fatal(err) + } + for _, p := range []string{bin, server} { + if err := os.WriteFile(p, []byte("planted"), 0o755); err != nil { + t.Fatal(err) + } + } + modelDir := filepath.Join(root, "model-moonshine-tiny-en-int8") + if err := os.MkdirAll(modelDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(modelDir, "tokens.txt"), []byte("planted"), 0o644); err != nil { + t.Fatal(err) + } + txn := lockFor(t, engineDir) + defer txn.release() + + // Nothing is listening on the API base, so any download attempt fails + // outright rather than quietly re-installing what is already there. + comp, err := EnsureLocalEngine(context.Background(), DownloadOptions{ + DestRoot: root, EngineVersion: "test", APIBase: offlineAPIBase(t), platformKey: "linux-amd64", skipPinned: true, + }) + if err != nil { + t.Fatalf("an expired wait over a usable destination must read as installed: %v", err) + } + if comp.BinaryPath != bin { + t.Errorf("BinaryPath = %q, want the install already at the destination %q", comp.BinaryPath, bin) + } + }) + + t.Run("an unusable destination is a named in-progress outcome", func(t *testing.T) { + root := t.TempDir() + engineDir := filepath.Join(root, "engine-test-linux-amd64") + txn := lockFor(t, engineDir) + defer txn.release() + + _, err := EnsureLocalEngine(context.Background(), DownloadOptions{ + DestRoot: root, EngineVersion: "test", APIBase: offlineAPIBase(t), platformKey: "linux-amd64", skipPinned: true, + }) + if !errors.Is(err, errInstallInProgress) { + t.Fatalf("EnsureLocalEngine = %v, want an error satisfying errInstallInProgress", err) + } + if _, err := os.Lstat(engineDir); !os.IsNotExist(err) { + t.Errorf("nothing should have been created for a destination that was never locked: %v", err) + } + }) +} From 3ad135b019076c72c9cc40b8be7a48e7eaebe41f Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:21:46 -0500 Subject: [PATCH 23/34] fix(daemon/remote): let the allocator read only the names it wrote Older versions accepted link ids beginning with a dot, so a published work tree can legitimately sit under a name shaped like a staging dir. The stamp parser split on the first dash and took whatever followed the prefix, so a link named for the maximum int64 read as a sequence at the ceiling and every upload for every link then failed to allocate. The parser now accepts exactly the prefix, twenty digits and the -seq suffix, and the scan skips any name that passes the grammar but holds a .git at its own root, since that is a work tree whatever it is called. Names the grammar rejects read as unstamped, which already means retained and unordered, so the change widens what is kept and narrows nothing. The veto looks only at the entry's own root: a live extract holds a .git one level down in its clone and its number is still spoken for. Several fixtures were named in shapes the strict parser now ignores, including the one the overflow test depends on, so they are renamed to the grammar they are meant to exercise rather than passing vacuously. --- internal/daemon/remote/bundle.go | 62 +++++++--- internal/daemon/remote/bundle_test.go | 164 +++++++++++++++++++++++--- 2 files changed, 197 insertions(+), 29 deletions(-) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index e54f0d3f5..c6fef22fb 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -403,28 +403,51 @@ type stagedExtract struct { stamped bool } -// stagingStamp reads back the ordering stamp extractBundle put in a staging -// name. New names carry a per-directory sequence; names written by released -// versions carry wall-clock nanoseconds. Both are plain int64s and compare the -// same way, which is what lets one directory hold a mix of them. +// stagingStamp reads back the per-directory sequence createSequencedStagingDir +// put in a staging name. The grammar is exact: the prefix, stagingSeqDigits +// ASCII digits, the suffix, nothing else. No released version wrote a stamped +// name at all, so a looser parse buys no migration and costs ownership: v0.8.0 +// staged under os.MkdirTemp(parent, ".staging-*"), whose decimal suffix carries +// no second '-', and dot-prefixed link ids were once accepted, so a published +// work tree can sit under any name at all. One that reads as a sequence gets a +// say in the ordering, and one at the maximum stops the allocator for every link +// in the directory. func stagingStamp(name string) (int64, bool) { - digits, _, found := strings.Cut(strings.TrimPrefix(name, stagingPrefix), "-") - if !found { + digits, ok := strings.CutPrefix(name, stagingPrefix) + if !ok { return 0, false } + digits, ok = strings.CutSuffix(digits, stagingSeqSuffix) + if !ok || len(digits) != stagingSeqDigits { + return 0, false + } + for i := 0; i < len(digits); i++ { + // ParseInt accepts a leading sign, and a link id may contain '-', so + // without this a legacy name reads back as a negative sequence the + // writer could never have emitted. + if digits[i] < '0' || digits[i] > '9' { + return 0, false + } + } stamp, err := strconv.ParseInt(digits, 10, 64) if err != nil { + // Digits alone can still name a number past int64. return 0, false } return stamp, true } -// stagingSeqSuffix closes a sequenced staging name. The parsers cut on the first -// '-' after the digits, so a name that ends at the digits reads back as -// unstamped, which is silent: an unstamped entry sorts last and is always -// retained, so the ordering key would simply stop existing with nothing failing. -// os.MkdirTemp used to supply this separator with its random suffix. -const stagingSeqSuffix = "-seq" +// stagingSeqSuffix closes a sequenced staging name, and stagingSeqDigits is the +// width createSequencedStagingDir's %020d writes. The parser requires both +// exactly, and createSequencedStagingDir reads its own name back before using +// it, so a format string that drifted from either would fail there rather than +// putting a name on disk that reads as unstamped: an unstamped entry sorts last +// and is always retained, so the ordering key would simply stop existing with +// nothing failing. +const ( + stagingSeqSuffix = "-seq" + stagingSeqDigits = 20 +) // stagingSeqAttempts bounds the walk up from a taken number, in the spirit of // the retry limit os.MkdirTemp applies to its own random names. @@ -465,9 +488,20 @@ func nextStagingSeq(dir string) (int64, error) { default: continue } - if stamp, ok := stagingStamp(name); ok && stamp > high { - high = stamp + stamp, ok := stagingStamp(name) + if !ok || stamp <= high { + continue + } + // A dir with a .git at its own root is a work tree, not something this + // package wrote: link ids starting with '.' used to be accepted, so one + // can carry the exact generated name. The clone and the set-aside tree a + // live staging dir holds are one level down, in repo/ and backup/, so + // this vetoes the legacy tree without releasing a number that is still + // spoken for. + if _, err := stagingFS.stat(filepath.Join(dir, entry.Name(), ".git")); err == nil { + continue } + high = stamp } if high == math.MaxInt64 { // The addition below would wrap negative, and %020d of a negative diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index 94fd52f6d..05c3ff9a2 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -1022,12 +1022,15 @@ func TestRecoverBundleDirLeavesALiveExtractAlone(t *testing.T) { } // stageBackup plants a staging dir holding a backup tree for linkID, named the -// way extractBundle names one so recovery can order it. A stamp of 0 plants an -// unstamped name, which is the shape recovery must refuse to order. +// way extractBundle names one so recovery can order it. A stamp above 0 gets the +// allocator's exact grammar, because that is now the only shape recovery orders; +// the name argument then only distinguishes unstamped fixtures, which are the +// shape recovery must refuse to order. Two stamped fixtures in one directory +// need distinct stamps, as two extracts do. func stageBackup(t *testing.T, dir, name, linkID, content string, stamp int64) string { t.Helper() if stamp > 0 { - name = fmt.Sprintf("%020d-%s", stamp, name) + name = fmt.Sprintf("%020d%s", stamp, stagingSeqSuffix) } staging := filepath.Join(dir, stagingPrefix+name) backup := filepath.Join(staging, "backup") @@ -1428,20 +1431,27 @@ func TestStagingNamesAllocateInOrderAndParse(t *testing.T) { } }) - // The upgrade case: a released binary wrote wall-clock nanoseconds, so - // seeding from the highest present is the whole migration. - t.Run("seeds above a legacy nanosecond name", func(t *testing.T) { + // The upgrade case. v0.8.0 staged under os.MkdirTemp(parent, ".staging-*"), + // whose suffix is a decimal uint32 with no second '-', and this branch's + // intermediate commits wrote a random one. Neither shape is a sequence this + // package ever handed out, so the allocator reads none of them and starts at + // one; the residue is left where it is for recovery to report. + t.Run("ignores a legacy MkdirTemp-shaped name", func(t *testing.T) { dir := t.TempDir() const legacy = int64(1_700_000_000_000_000_000) - if err := os.Mkdir(filepath.Join(dir, fmt.Sprintf("%s%020d-x7Kq3", stagingPrefix, legacy)), 0o700); err != nil { + name := fmt.Sprintf("%s%020d-x7Kq3", stagingPrefix, legacy) + if err := os.Mkdir(filepath.Join(dir, name), 0o700); err != nil { t.Fatal(err) } seq, err := nextStagingSeq(dir) if err != nil { t.Fatal(err) } - if seq <= legacy { - t.Errorf("next sequence = %d, want strictly greater than the legacy stamp %d", seq, legacy) + if seq != 1 { + t.Errorf("next sequence = %d, want 1: a name this package never wrote is not a sequence", seq) + } + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + t.Errorf("the legacy entry must be left alone: %v", err) } }) @@ -1461,6 +1471,129 @@ func TestStagingNamesAllocateInOrderAndParse(t *testing.T) { }) } +// The grammar is the first ownership filter: a name that is not exactly what +// createSequencedStagingDir writes was written by something else, and reading a +// sequence out of it is how a legacy work tree gets a say in the ordering. +func TestStagingStampRequiresTheExactGrammar(t *testing.T) { + // No '*', '?' or ':' in any name here: a table entry Windows cannot name + // would fail its own setup rather than test the parser. + cases := []struct { + name string + seq int64 + ok bool + }{ + {stagingPrefix + "00000000000000000042" + stagingSeqSuffix, 42, true}, + {stagingPrefix + "42" + stagingSeqSuffix, 0, false}, + // The shape this branch's intermediate commits wrote before the suffix + // was fixed, and the shape os.MkdirTemp writes. + {stagingPrefix + "00000000000000000042-x7Kq3", 0, false}, + // The v0.8.0 shape: MkdirTemp's decimal suffix with no second '-'. + {stagingPrefix + "1234567890", 0, false}, + {stagingPrefix + "00000000000000000042" + stagingSeqSuffix + "-extra", 0, false}, + {stagingPrefix + "0000000000000000004a" + stagingSeqSuffix, 0, false}, + // Twenty characters, and ParseInt would take the sign. A link id may + // contain '-', so this is a name a legacy work tree can carry. + {stagingPrefix + "-0000000000000000042" + stagingSeqSuffix, 0, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + seq, ok := stagingStamp(tc.name) + if ok != tc.ok { + t.Fatalf("stagingStamp(%q) ok = %v, want %v", tc.name, ok, tc.ok) + } + if ok && seq != tc.seq { + t.Errorf("stagingStamp(%q) = %d, want %d", tc.name, seq, tc.seq) + } + }) + } +} + +// Link ids starting with '.' used to be accepted, so a published work tree can +// sit under a name that now reads as a staging sequence. One at the maximum +// would make the allocator refuse to allocate anything, permanently, and no +// extract for any link in the directory could run again. +func TestNextStagingSeqIgnoresALegacyWorkTreeAtTheMaximum(t *testing.T) { + dir := t.TempDir() + // Twenty digits, so it passes the grammar and only the .git veto keeps it + // out of the ordering. + planted := []string{ + fmt.Sprintf("%s%020d%s", stagingPrefix, int64(math.MaxInt64), stagingSeqSuffix), + // Nineteen digits, the name from the review: the grammar alone rejects it. + stagingPrefix + "9223372036854775807" + stagingSeqSuffix, + } + for _, name := range planted { + if err := os.MkdirAll(filepath.Join(dir, name, ".git"), 0o700); err != nil { + t.Fatal(err) + } + } + // A legacy staging dir at the maximum with no work tree in it: the grammar + // is the only thing that can keep this one out, so loosening the grammar is + // visible here even with the veto in place. + if err := os.Mkdir(filepath.Join(dir, fmt.Sprintf("%s%020d-x7Kq3", stagingPrefix, int64(math.MaxInt64))), 0o700); err != nil { + t.Fatal(err) + } + + seq, err := nextStagingSeq(dir) + if err != nil { + t.Fatalf("a legacy entry must not stop the allocator: %v", err) + } + if seq != 1 { + t.Errorf("next sequence = %d, want 1: none of these names is one this package wrote", seq) + } + for _, name := range planted { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + t.Errorf("the allocator must not touch %s: %v", name, err) + } + } +} + +// The veto is keyed on .git at the entry's own root. A live extract's staging +// dir holds its clone one level down, in repo/, so its number still counts; +// handing it out again would put two transactions on one name. +func TestNextStagingSeqCountsAStagingDirHoldingAClone(t *testing.T) { + dir := t.TempDir() + staging := filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, 4, stagingSeqSuffix)) + if err := os.MkdirAll(filepath.Join(staging, "repo", ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(staging, "backup", ".git"), 0o700); err != nil { + t.Fatal(err) + } + + seq, err := nextStagingSeq(dir) + if err != nil { + t.Fatal(err) + } + if seq != 5 { + t.Errorf("next sequence = %d, want 5: a live extract's number is still spoken for", seq) + } +} + +// Kept backups are named off the staging name, so they are counted, and they are +// counted under the same grammar: a loose read of one is the same defect as a +// loose read of a staging name, one prefix over. +func TestNextStagingSeqCountsKeptNamesUnderTheSameGrammar(t *testing.T) { + dir := t.TempDir() + kept := filepath.Join(dir, fmt.Sprintf("%s%020d%s", keptPrefix, 7, stagingSeqSuffix)) + if err := os.Mkdir(kept, 0o700); err != nil { + t.Fatal(err) + } + if err := writeMarker(kept, txnMarker{Kind: txnKindBundleExtract, Dest: "proj-1", Seq: 7}); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(dir, fmt.Sprintf("%s%020d-x", keptPrefix, 99)), 0o700); err != nil { + t.Fatal(err) + } + + seq, err := nextStagingSeq(dir) + if err != nil { + t.Fatal(err) + } + if seq != 8 { + t.Errorf("next sequence = %d, want 8: the parked 7 is spoken for and the loose 99 is not a sequence", seq) + } +} + // A publish that succeeded but could not clear its staging dir reports success to // the client while a whole copy of the prior tree stays on disk. The next // recovery pass reclaims it, and must say so: with no bridge logger configured @@ -1541,7 +1674,7 @@ func TestCreateSequencedStagingDirSkipsAnOccupiedNumber(t *testing.T) { // belongs where the addition is. func TestNextStagingSeqRefusesOverflow(t *testing.T) { dir := t.TempDir() - if err := os.Mkdir(filepath.Join(dir, fmt.Sprintf("%s%020d-x", stagingPrefix, int64(math.MaxInt64))), 0o700); err != nil { + if err := os.Mkdir(filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, int64(math.MaxInt64), stagingSeqSuffix)), 0o700); err != nil { t.Fatal(err) } if seq, err := nextStagingSeq(dir); err == nil { @@ -1739,13 +1872,14 @@ func TestRecoverBundleDirIsIdempotent(t *testing.T) { } } -// Two extracts can stamp the same instant. Recovery restores one of them and -// must then keep the other rather than deleting it as superseded: equal stamps -// are not evidence that one came first. +// Two backups can both carry names recovery cannot order: v0.8.0 residue, or a +// crash before the marker. Recovery restores one of them and must then keep the +// other rather than deleting it as superseded, because no order exists to say +// which came first. func TestRecoverBundleDirKeepsABackupItCannotTellApartFromTheRestore(t *testing.T) { dir := t.TempDir() - first := stageBackup(t, dir, "one", "proj-1", "v1", 100) - second := stageBackup(t, dir, "two", "proj-1", "v2", 100) + first := stageBackup(t, dir, "one", "proj-1", "v1", 0) + second := stageBackup(t, dir, "two", "proj-1", "v2", 0) recoverBundleDir(dir, nil) From 064b4e132d13ecc38fef6dab572fcc6de24217b2 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:21:46 -0500 Subject: [PATCH 24/34] fix(dictation): attribute a holder by its marker, not by its name Any sibling directory whose name began with the destination plus .previous- was treated as a holder this code owned and reaped wholesale. Engine release tags are configurable and model directory names come from a browse listing, so a legitimate cache directory can share that prefix, and ensuring one usable destination deleted the other outright. A holder now carries a marker naming the transaction kind, the destination and its sequence, written before the set-aside rename, and a commit flag written after the publish. Attribution is the grammar and then agreement with that marker: a prefix collision is not a candidate at all, a marked holder naming another destination is not this destination's to classify, and a grammar passing directory with no readable marker is retained rather than reaped. Holders also gain the kept namespace the bundle site already had, so a copy a later pass must not enumerate can be moved out of the scanned prefix. A failed commit flag keeps the superseded copy, beside the existing retain for a failed restore. No released version wrote a holder, so the comments claiming an older wall-clock name shape described only this branch and are corrected. --- internal/dictation/download.go | 348 ++++++++++++++++---- internal/dictation/download_test.go | 491 +++++++++++++++++++++++++--- 2 files changed, 719 insertions(+), 120 deletions(-) diff --git a/internal/dictation/download.go b/internal/dictation/download.go index c75cc6598..7ef3b4bad 100644 --- a/internal/dictation/download.go +++ b/internal/dictation/download.go @@ -22,6 +22,7 @@ import ( "strings" "time" + "github.com/Gitlawb/zero/internal/fsutil" "github.com/Gitlawb/zero/internal/lockutil" ) @@ -794,8 +795,10 @@ func resolveEnginePaths(engineDir string, targetWindows bool) (bin, server strin // because recovery has to pick the NEWEST holder when a failed cleanup left an // older one behind, and nothing else records that order: Glob sorts lexically // and a directory mtime tracks the install's contents, not its promotion. The -// number is one past the highest already beside this install, so a clock moving -// backward cannot invert it the way the wall-clock stamp it replaces could. +// number is a per-install sequence, one past the highest already beside this +// install, so no decision here reads a clock. There is no older shape to accept: +// released versions staged under .stage-* and published with a plain rename, +// and none of them ever created a holder. // fsOps is the filesystem seam the promotion path, the holder allocator, and // recovery take every step through. A step that calls os directly cannot be made // to fail, and the crash each of these steps exists to survive is then only @@ -881,15 +884,22 @@ func (t *destTxn) release() { _ = t.lock.Release() } +// isInstallDestName reports whether name is one path component naming an install +// destination. A destination is one path component: any other shape puts the +// lock file somewhere else, and two callers for the same destination would then +// lock different inodes and both proceed. Recovery runs the same check over the +// destination name it reads out of a marker, so a name off disk is never trusted +// further than one the caller supplied. +func isInstallDestName(name string) bool { + return name != "" && name == filepath.Base(name) && name != "." && name != ".." +} + // lockDestination takes the Install lock for one destination under destRoot, // waiting out another process that holds it until ctx ends or the budget runs // out. The budget expiring is errInstallInProgress, which the caller answers by // re-checking the destination rather than by failing. func lockDestination(ctx context.Context, destRoot, dest string) (*destTxn, error) { - // A destination is one path component. Any other shape puts the lock file - // somewhere else, and two callers for the same destination would then lock - // different inodes and both proceed. - if dest == "" || dest != filepath.Base(dest) || dest == "." || dest == ".." { + if !isInstallDestName(dest) { return nil, fmt.Errorf("dictation download: %q is not an install destination name", dest) } // destRoot is otherwise created lazily by the download this lock covers, and @@ -944,16 +954,48 @@ func withDestinationLock(ctx context.Context, destRoot, dest string, usable func const holderSuffix = ".previous-" +// keptSuffix names a holder recovery decided to retain rather than restore or +// delete. It is a different prefix from holderSuffix so recovery's scan never +// enumerates a copy it has already ruled on, while the allocator still counts +// both and cannot reissue a number a Kept backup already carries. +const keptSuffix = ".kept-" + +// holderSeqDigits is how many digits a sequenced name carries. Fixed width is +// what makes the grammar exact: a name one digit short is a name this code did +// not write, whatever else it looks like. +const holderSeqDigits = 20 + // holderStamp reads back the ordering number in a holder name, reporting false -// for a name it cannot order (one this package did not write). New names carry a -// per-install sequence; names written by released versions carry wall-clock -// nanoseconds. Both compare the same way. +// for any name this code did not write. The grammar is exact and it is the +// first ownership filter: the install's own base name, then .previous- or +// .kept-, then twenty digits, then holderSeqSuffix, and nothing else. Loose +// parsing is what let a sibling merely NAMED like a holder be attributed to the +// install and moved on its account. No released version wrote a holder at all, +// so there is no older shape to accept. func holderStamp(destDir, holder string) (int64, bool) { - rest := strings.TrimPrefix(filepath.Base(holder), filepath.Base(destDir)+holderSuffix) - digits, _, found := strings.Cut(rest, "-") - if !found { + name := filepath.Base(holder) + base := filepath.Base(destDir) + var rest string + switch { + case strings.HasPrefix(name, base+holderSuffix): + rest = name[len(base)+len(holderSuffix):] + case strings.HasPrefix(name, base+keptSuffix): + rest = name[len(base)+len(keptSuffix):] + default: return 0, false } + digits, ok := strings.CutSuffix(rest, holderSeqSuffix) + if !ok || len(digits) != holderSeqDigits { + return 0, false + } + for _, c := range []byte(digits) { + // ParseInt would take a leading '+' or '-', and a signed number renders + // back through %020d as a different string, so the writer and the reader + // would disagree about a name they both accept. + if c < '0' || c > '9' { + return 0, false + } + } stamp, err := strconv.ParseInt(digits, 10, 64) if err != nil { return 0, false @@ -961,31 +1003,203 @@ func holderStamp(destDir, holder string) (int64, bool) { return stamp, true } -// holderSeqSuffix closes a sequenced holder name. holderStamp cuts on the first -// '-' after the digits, so a name ending at the digits reads back as unstamped, -// which is silent: an unstamped holder sorts last and is still restorable, so -// the ordering key would stop existing with nothing failing. os.MkdirTemp used -// to supply this separator with its random suffix. +// holderSeqSuffix closes a sequenced holder name. It is what the grammar ends +// on, so a name that stops at the digits, or carries anything after the suffix, +// is not one this code wrote. const holderSeqSuffix = "-seq" // holderSeqAttempts bounds the walk up from a taken number, in the spirit of the // retry limit os.MkdirTemp applies to its own random names. const holderSeqAttempts = 10000 +// holderMarkerFile names the transaction marker written inside every holder +// before anything is moved into it. The name alone is not ownership: any +// directory can be named to look like a holder, and moving or deleting one on +// its name is how a sibling that merely collides with the prefix loses its +// contents. +const holderMarkerFile = "txn" + +// committedFile is the flag promoteStagedDir creates inside a holder after the +// publish rename succeeds. It is the only evidence that the copy in the holder +// was superseded by an install that actually landed, and so the only thing that +// licenses removing it. +const committedFile = "committed" + +// holderMarkerKind is what this site writes in a marker's kind field. The two +// install sites use one marker format and different kinds, so a directory +// written by the other one is never read as this one's. +const holderMarkerKind = "dictation-promote" + +// txnMarker is the ownership proof inside a holder: the transaction that created +// it, the destination it was created for, and the sequence in its name. All +// three have to agree with what recovery already knows before the holder is +// this code's to touch. +type txnMarker struct { + Kind string `json:"kind"` + Dest string `json:"dest"` + Seq int64 `json:"seq"` +} + +// errMarkerMissing separates "there is no marker" from "the marker could not be +// read". The first is a directory this code cannot claim; the second is a +// filesystem fault, and treating them alike would let a transient read error +// silently reclassify a copy. +var errMarkerMissing = errors.New("no transaction marker") + +// writeHolderMarker writes the marker atomically: a temp file in the same +// directory, then a rename. A torn write cannot then leave a partial marker that +// parses, which is what makes "the marker is missing" and "the marker is ours" +// the only two answers a crash can produce. +func writeHolderMarker(dir string, m txnMarker) error { + data, err := json.Marshal(m) + if err != nil { + return err + } + f, err := holderFS.createTemp(dir, holderMarkerFile+"-*") + if err != nil { + return err + } + tmp := f.Name() + if err := writeAndSync(f, data); err != nil { + _ = holderFS.removeAll(tmp) + return err + } + if err := fsutil.RenameWithRetry(tmp, filepath.Join(dir, holderMarkerFile), holderFS.rename); err != nil { + _ = holderFS.removeAll(tmp) + return err + } + return nil +} + +// writeAndSync fills the marker's temp file and gets it to disk before the +// rename that publishes it. Without the Sync the rename can be visible after a +// crash while the bytes behind it are not. +func writeAndSync(f *os.File, data []byte) error { + if _, err := f.Write(data); err != nil { + f.Close() + return err + } + if err := f.Sync(); err != nil { + f.Close() + return err + } + return f.Close() +} + +// readHolderMarker reads a holder's marker. A missing one is errMarkerMissing; +// anything else is returned as it came back, so a caller can tell a directory it +// cannot claim from one it cannot read. +func readHolderMarker(dir string) (txnMarker, error) { + data, err := holderFS.readFile(filepath.Join(dir, holderMarkerFile)) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return txnMarker{}, fmt.Errorf("%w in %s", errMarkerMissing, dir) + } + return txnMarker{}, err + } + var m txnMarker + if err := json.Unmarshal(data, &m); err != nil { + return txnMarker{}, fmt.Errorf("unreadable transaction marker in %s: %w", dir, err) + } + return m, nil +} + +// writeCommitFlag records that the publish this holder's copy was set aside for +// actually landed. O_EXCL because a flag that is already there was written by a +// different transaction, and overwriting it would move the evidence. +func writeCommitFlag(dir string) error { + f, err := holderFS.create(filepath.Join(dir, committedFile), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + return f.Close() +} + +// holderCandidate is one holder recovery may act on, with the sequence its name +// and its marker agree on. +type holderCandidate struct { + path string + seq int64 +} + +// ownedHoldersBeside splits the entries beside destDir into the holders this +// code can prove it wrote for THIS destination and the ones it cannot. Owned +// means the exact name grammar and a marker whose kind, destination, and +// sequence all agree with it; unowned means a name this code would have written +// with no such marker behind it, which is retained and reported, never moved. +// An entry that only collides with the prefix is neither: it was never this +// install's, so it is not even reported on its account. +func ownedHoldersBeside(destDir string) (owned []holderCandidate, unowned []string) { + // ReadDir and a prefix rather than filepath.Glob: destDir is a real path, + // and a '[' anywhere in it opens a character class to Glob, which then + // matches nothing and strands the install this exists to put back. + parent := filepath.Dir(destDir) + base := filepath.Base(destDir) + entries, err := holderFS.readDir(parent) + if err != nil { + return nil, nil + } + prefix := base + holderSuffix + for _, entry := range entries { + name := entry.Name() + if !entry.IsDir() || !strings.HasPrefix(name, prefix) { + continue + } + path := filepath.Join(parent, name) + seq, ok := holderStamp(destDir, name) + if !ok { + continue + } + m, err := readHolderMarker(path) + if err != nil { + unowned = append(unowned, path) + continue + } + // Everything read off disk goes back through the check the write path + // applied, and the whole marker has to agree: a marker naming another + // destination or another sequence is evidence this directory belongs to + // a transaction that is not the one being recovered. + if m.Kind != holderMarkerKind || m.Seq != seq || m.Dest != base || !isInstallDestName(m.Dest) { + continue + } + owned = append(owned, holderCandidate{path: path, seq: seq}) + } + return owned, unowned +} + +// parkKeptHolder renames a holder under the Kept prefix, which is how a copy +// recovery will not restore and cannot prove superseded leaves the scan without +// leaving the disk. A plain rename: the Kept name is a distinct sequence nothing +// else claims, and a park that would have to clear something first is a park +// onto a copy that is not ours to remove. +func parkKeptHolder(holder string) error { + dir, name := filepath.Split(holder) + cut := strings.LastIndex(name, holderSuffix) + if cut < 0 { + return fmt.Errorf("%q is not a holder name", name) + } + destDir := filepath.Join(dir, name[:cut]) + if _, ok := holderStamp(destDir, holder); !ok { + return fmt.Errorf("%q is not a holder name", name) + } + kept := filepath.Join(dir, name[:cut]+keptSuffix+name[cut+len(holderSuffix):]) + return fsutil.RenameWithRetry(holder, kept, holderFS.rename) +} + // nextHolderSeq is the number a new holder should claim: one past the highest -// already set aside for this install. That is what makes the order survive a -// clock that moves backward, since a promotion that reads an existing holder -// always allocates above it. Holders for a different install are a separate -// sequence and are never compared against this one. +// already set aside for this install, counting BOTH the scanned prefix and the +// Kept one. Recovery parks a holder by renaming it under .kept-, and a sequence +// that stopped counting there would reissue a number a parked copy already +// carries. Holders for a different install are a separate sequence and are never +// compared against this one, and a name the grammar rejects raises nothing. func nextHolderSeq(destDir string) (int64, error) { entries, err := holderFS.readDir(filepath.Dir(destDir)) if err != nil { return 0, err } - prefix := filepath.Base(destDir) + holderSuffix var high int64 for _, entry := range entries { - if !entry.IsDir() || !strings.HasPrefix(entry.Name(), prefix) { + if !entry.IsDir() { continue } if stamp, ok := holderStamp(destDir, entry.Name()); ok && stamp > high { @@ -1067,30 +1281,24 @@ func restoreInterruptedPromotion(txn *destTxn, destDir string, published func(st // A removal that fails is left for the next call, which reaches this // same branch and tries again. if published != nil && published(destDir) { - for _, holder := range holdersBeside(destDir) { - _ = holderFS.removeAll(holder) + owned, _ := ownedHoldersBeside(destDir) + for _, candidate := range owned { + _ = holderFS.removeAll(candidate.path) } } return } - // ReadDir and a prefix rather than filepath.Glob: destDir is a real path, - // and a '[' anywhere in it opens a character class to Glob, which then - // matches nothing and strands the install this exists to put back. - holders := holdersBeside(destDir) - // Newest first: an unstamped holder is the least recent thing we can claim - // to know about, so it is only reached once every stamped one has failed. - slices.SortStableFunc(holders, func(a, b string) int { - sa, oka := holderStamp(destDir, a) - sb, okb := holderStamp(destDir, b) - if oka != okb { - if oka { - return -1 - } - return 1 - } - return cmp.Compare(sb, sa) + // Only holders this code can prove it wrote for THIS destination. A sibling + // that merely collides with the prefix is not a copy of this install and is + // never restored from or removed on its account. + owned, _ := ownedHoldersBeside(destDir) + // Newest first: the sequence in the name, which the marker agrees with, is + // the only record of which copy was live last. + slices.SortStableFunc(owned, func(a, b holderCandidate) int { + return cmp.Compare(b.seq, a.seq) }) - for _, holder := range holders { + for _, candidate := range owned { + holder := candidate.path install := filepath.Join(holder, "install") if _, err := holderFS.stat(install); err != nil { continue @@ -1105,28 +1313,6 @@ func restoreInterruptedPromotion(txn *destTxn, destDir string, published func(st } } -// holdersBeside lists the holders promoteStagedDir may have left for destDir. -// The prefix is the attribution: a holder for a different install is a separate -// concern and is never touched on this one's account. -func holdersBeside(destDir string) []string { - // ReadDir and a prefix rather than filepath.Glob: destDir is a real path, - // and a '[' anywhere in it opens a character class to Glob, which then - // matches nothing and strands the install this exists to put back. - parent := filepath.Dir(destDir) - entries, err := holderFS.readDir(parent) - if err != nil { - return nil - } - prefix := filepath.Base(destDir) + holderSuffix - holders := make([]string, 0, len(entries)) - for _, entry := range entries { - if entry.IsDir() && strings.HasPrefix(entry.Name(), prefix) { - holders = append(holders, filepath.Join(parent, entry.Name())) - } - } - return holders -} - // promoteStagedDir moves stageDir into place at destDir. os.Rename refuses to // overwrite a non-empty directory, so any previous install has to move out of // the way first; it is set aside rather than deleted, and put back if the @@ -1161,22 +1347,50 @@ func promoteStagedDir(txn *destTxn, stageDir, destDir, label string, report func if err != nil { return fmt.Errorf("setting aside previous %s install: %w", label, err) } + // createSequencedHolder walks up from seq when a name is taken, so the + // marker has to carry the number actually claimed. A marker that + // disagrees with the name it sits in is not ownership, and recovery + // would leave the copy in place forever. + claimed, ok := holderStamp(destDir, holder) + if !ok { + return fmt.Errorf("setting aside previous %s install: holder name %q is not one recovery can order", label, filepath.Base(holder)) + } + // The marker goes in BEFORE the first destructive rename. A holder that + // takes the only copy of an install before it can be proven ours is one + // nothing on disk lets recovery claim. + if err := writeHolderMarker(holder, txnMarker{Kind: holderMarkerKind, Dest: filepath.Base(destDir), Seq: claimed}); err != nil { + return fmt.Errorf("setting aside previous %s install: %w", label, err) + } previous := filepath.Join(holder, "install") - if err := holderFS.rename(destDir, previous); err != nil { + if err := fsutil.RenameWithRetry(destDir, previous, holderFS.rename); err != nil { return fmt.Errorf("setting aside previous %s install: %w", label, err) } - restore = func() error { return holderFS.rename(previous, destDir) } + restore = func() error { return fsutil.RenameWithRetry(previous, destDir, holderFS.rename) } } else if !os.IsNotExist(err) { return fmt.Errorf("checking previous %s install: %w", label, err) } - if err := holderFS.rename(stageDir, destDir); err != nil { + if err := fsutil.RenameWithRetry(stageDir, destDir, holderFS.rename); err != nil { if restoreErr := restore(); restoreErr != nil { cleanupHolder = false return fmt.Errorf("promoting staged %s: %w (previous install left in %s: %v)", label, err, holder, restoreErr) } return fmt.Errorf("promoting staged %s: %w", label, err) } + if holder != "" { + // The flag follows the publish and precedes the cleanup, so the only + // thing a crash in that window costs is a retained copy. Writing it any + // earlier would license deleting a copy that is still the only one. + if err := writeCommitFlag(holder); err != nil { + // The install itself landed. Deleting a whole previous install on + // the word of a step that just failed is the one outcome worse than + // leaving it on disk, so the copy stays and the report says where. + cleanupHolder = false + if report != nil { + report(fmt.Sprintf("Installed %s, but the previous install could not be marked superseded and is kept in %s (%v)", label, holder, err)) + } + } + } return nil } diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index 25a3661c0..0a601b735 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -14,6 +14,7 @@ import ( "path/filepath" "reflect" "runtime" + "slices" "strconv" "strings" "sync" @@ -384,14 +385,7 @@ func TestPromoteStagedDirKeepsTheSetAsideCopyWhenRestoreAlsoFails(t *testing.T) func TestRestoreInterruptedPromotionPutsTheInstallBack(t *testing.T) { root := t.TempDir() dest := filepath.Join(root, "engine-1.2.3-linux-x64") - holder := filepath.Join(root, filepath.Base(dest)+".previous-abc") - install := filepath.Join(holder, "install") - if err := os.MkdirAll(install, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(install, "engine"), []byte("kept"), 0o644); err != nil { - t.Fatal(err) - } + holder := plantHolder(t, dest, 1, "kept") restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) @@ -516,14 +510,7 @@ func TestRestoreInterruptedPromotionLeavesAnExistingDestAlone(t *testing.T) { t.Fatal(err) } } - holder := filepath.Join(root, filepath.Base(dest)+".previous-abc") - install := filepath.Join(holder, "install") - if err := os.MkdirAll(install, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(install, "engine"), []byte("stale"), 0o644); err != nil { - t.Fatal(err) - } + install := filepath.Join(plantHolder(t, dest, 1, "stale"), "install") restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) @@ -566,8 +553,11 @@ func TestEnsureLocalEngineRestoresAnInterruptedModelPromotion(t *testing.T) { // Stop the world where promoteStagedDir has moved the model aside but has // not yet published the staged copy. modelDir := filepath.Dir(comp.ModelPath) - holder := modelDir + ".previous-abc" - if err := os.MkdirAll(holder, 0o755); err != nil { + holder := fmt.Sprintf("%s%s%020d%s", modelDir, holderSuffix, 1, holderSeqSuffix) + if err := os.MkdirAll(holder, 0o700); err != nil { + t.Fatal(err) + } + if err := writeHolderMarker(holder, txnMarker{Kind: holderMarkerKind, Dest: filepath.Base(modelDir), Seq: 1}); err != nil { t.Fatal(err) } if err := os.Rename(modelDir, filepath.Join(holder, "install")); err != nil { @@ -620,11 +610,13 @@ func testPublished(dir string) bool { return err == nil } -// plantHolder writes an install into a holder named the way promoteStagedDir -// names one, so recovery sees the same shape it does in production. -func plantHolder(t *testing.T, destDir string, stamp int64, content string) string { +// plantHolder writes an install into a holder named and marked the way +// promoteStagedDir writes one, so recovery sees the same shape it does in +// production: the name alone is not ownership, and a fixture without the marker +// would test a directory recovery is supposed to leave alone. +func plantHolder(t *testing.T, destDir string, seq int64, content string) string { t.Helper() - holder := fmt.Sprintf("%s%s%020d-%d", destDir, holderSuffix, stamp, stamp) + holder := fmt.Sprintf("%s%s%020d%s", destDir, holderSuffix, seq, holderSeqSuffix) install := filepath.Join(holder, "install") if err := os.MkdirAll(install, 0o755); err != nil { t.Fatal(err) @@ -632,6 +624,9 @@ func plantHolder(t *testing.T, destDir string, stamp int64, content string) stri if err := os.WriteFile(filepath.Join(install, "engine"), []byte(content), 0o644); err != nil { t.Fatal(err) } + if err := writeHolderMarker(holder, txnMarker{Kind: holderMarkerKind, Dest: filepath.Base(destDir), Seq: seq}); err != nil { + t.Fatal(err) + } return holder } @@ -867,23 +862,34 @@ func TestHolderNamesAllocateInOrderAndParse(t *testing.T) { } }) - t.Run("seeds above a legacy nanosecond name", func(t *testing.T) { + // No released version of this package ever wrote a holder: v0.8.0 staged + // under .stage-* and published with a plain rename. A nanosecond-shaped name + // beside an install is therefore something this code did not write, and + // counting it would let any such directory dictate the sequence, up to the + // maximum that refuses installs outright. + t.Run("ignores a nanosecond-shaped name it did not write", func(t *testing.T) { root := t.TempDir() dest := filepath.Join(root, "engine-1.2.3-linux-x64") - const legacy = int64(1_700_000_000_000_000_000) - // Both shapes a released binary could have left: the MkdirTemp random - // suffix, and the digits-only one the test helper plants. + const nano = int64(1_700_000_000_000_000_000) + var planted []string for _, suffix := range []string{"x7Kq3", "12345"} { - if err := os.MkdirAll(fmt.Sprintf("%s%s%020d-%s", dest, holderSuffix, legacy, suffix), 0o700); err != nil { + path := fmt.Sprintf("%s%s%020d-%s", dest, holderSuffix, nano, suffix) + if err := os.MkdirAll(path, 0o700); err != nil { t.Fatal(err) } + planted = append(planted, path) } seq, err := nextHolderSeq(dest) if err != nil { t.Fatal(err) } - if seq <= legacy { - t.Errorf("next sequence = %d, want strictly greater than the legacy stamp %d", seq, legacy) + if seq != 1 { + t.Errorf("next sequence = %d, want 1: neither planted name is one this code wrote", seq) + } + for _, path := range planted { + if _, err := os.Stat(path); err != nil { + t.Errorf("a name this code did not write must be left alone: %v", err) + } } }) } @@ -909,7 +915,7 @@ func TestCreateSequencedHolderSkipsAnOccupiedNumber(t *testing.T) { func TestNextHolderSeqRefusesOverflow(t *testing.T) { root := t.TempDir() dest := filepath.Join(root, "engine-1.2.3-linux-x64") - if err := os.MkdirAll(fmt.Sprintf("%s%s%020d-x", dest, holderSuffix, int64(math.MaxInt64)), 0o700); err != nil { + if err := os.MkdirAll(fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, int64(math.MaxInt64), holderSeqSuffix), 0o700); err != nil { t.Fatal(err) } if seq, err := nextHolderSeq(dest); err == nil { @@ -1022,36 +1028,41 @@ func requireUmaskAllowsWiderThan0700(t *testing.T, parent string) { } } -// A holder this package did not name carries no ordering anyone can read, so it -// is the least recent thing recovery can claim to know about and must lose to -// any stamped holder, however old that one's stamp is. -func TestRestoreInterruptedPromotionPrefersAStampedHolderOverAnUnstampedOne(t *testing.T) { +// Recovery restores only from a holder it can prove it wrote for THIS install. +// Both decoys carry content that would land at the destination if the name alone +// were the attribution, and both sort FIRST lexically, so only the ownership +// rule can produce the wanted answer. +func TestRestoreInterruptedPromotionRestoresOnlyFromAnOwnedHolder(t *testing.T) { root := t.TempDir() dest := filepath.Join(root, "engine-1.2.3-linux-x64") - // The unstamped name sorts after the stamped one lexically, so a pass that - // ignored the stamp entirely would still get this right; give it a name that - // sorts FIRST, so only the stamped-wins rule can produce the wanted answer. - unstamped := dest + holderSuffix + "aaa" - if err := os.MkdirAll(filepath.Join(unstamped, "install"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(unstamped, "install", "engine"), []byte("unstamped"), 0o644); err != nil { - t.Fatal(err) + // A user's own directory that happens to start like a holder name. + collide := dest + holderSuffix + "aaa" + // A name this code would have written, with nothing behind it saying it did. + unmarked := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 900, holderSeqSuffix) + for _, decoy := range []string{collide, unmarked} { + if err := os.MkdirAll(filepath.Join(decoy, "install"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(decoy, "install", "engine"), []byte("not ours"), 0o644); err != nil { + t.Fatal(err) + } } - stamped := plantHolder(t, dest, 100, "stamped") + owned := plantHolder(t, dest, 100, "ours") restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) got, err := os.ReadFile(filepath.Join(dest, "engine")) - if err != nil || string(got) != "stamped" { - t.Fatalf("a stamped holder must win over an unstamped one: got %q err %v", got, err) + if err != nil || string(got) != "ours" { + t.Fatalf("recovery restored from a holder it cannot prove it wrote: got %q err %v", got, err) } - if _, err := os.Stat(stamped); !os.IsNotExist(err) { + if _, err := os.Stat(owned); !os.IsNotExist(err) { t.Errorf("the restored holder should be cleared, got %v", err) } - // The one recovery did not use is left for a human, never deleted on a guess. - if _, err := os.Stat(filepath.Join(unstamped, "install", "engine")); err != nil { - t.Errorf("the unused holder must be kept: %v", err) + // Neither decoy is recovery's to move or delete. + for _, decoy := range []string{collide, unmarked} { + if _, err := os.Stat(filepath.Join(decoy, "install", "engine")); err != nil { + t.Errorf("%s must be left exactly as found: %v", filepath.Base(decoy), err) + } } } @@ -1063,9 +1074,14 @@ func TestRestoreInterruptedPromotionSkipsAHolderWithNoInstall(t *testing.T) { root := t.TempDir() dest := filepath.Join(root, "engine-1.2.3-linux-x64") usable := plantHolder(t, dest, 100, "kept") - // Newer, so ordering reaches it first, but it holds nothing. - empty := fmt.Sprintf("%s%s%020d-x", dest, holderSuffix, 200) - if err := os.MkdirAll(empty, 0o755); err != nil { + // Newer AND owned, so ordering reaches it first and it is a real candidate, + // but it holds nothing: exactly what a stop between the allocation and the + // set-aside rename leaves. + empty := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 200, holderSeqSuffix) + if err := os.MkdirAll(empty, 0o700); err != nil { + t.Fatal(err) + } + if err := writeHolderMarker(empty, txnMarker{Kind: holderMarkerKind, Dest: filepath.Base(dest), Seq: 200}); err != nil { t.Fatal(err) } @@ -1785,3 +1801,372 @@ func TestEnsureLocalEngineTreatsAnExpiredWaitAsBenign(t *testing.T) { } }) } + +// The grammar is the first ownership filter: a name that does not read back as +// one createSequencedHolder wrote is not this code's to move or delete. Loose +// parsing is what let a sibling merely NAMED like a holder be attributed to the +// install, which is the review's P3 finding on this site. +func TestHolderStampRequiresTheExactGrammar(t *testing.T) { + dest := filepath.Join(t.TempDir(), "engine-1.2.3-linux-x64") + base := filepath.Base(dest) + for _, tc := range []struct { + name string + want int64 + ok bool + }{ + {base + ".previous-00000000000000000042-seq", 42, true}, + {base + ".kept-00000000000000000042-seq", 42, true}, + {base + ".previous-42-seq", 0, false}, + {base + ".previous-00000000000000000042-x7Kq3", 0, false}, + {base + ".previous-1234567890", 0, false}, + {base + ".previous-00000000000000000042-seq-extra", 0, false}, + {base + ".previous-0000000000000000004a-seq", 0, false}, + {base + ".previous-notes", 0, false}, + {base + ".previous-", 0, false}, + {base + ".kept-42-seq", 0, false}, + {"other-install.previous-00000000000000000042-seq", 0, false}, + } { + t.Run(tc.name, func(t *testing.T) { + got, ok := holderStamp(dest, filepath.Join(filepath.Dir(dest), tc.name)) + if ok != tc.ok || got != tc.want { + t.Errorf("holderStamp(%q) = (%d, %v), want (%d, %v)", tc.name, got, ok, tc.want, tc.ok) + } + }) + } +} + +// The allocator counts the Kept prefix as well as the scanned one. Recovery +// parks a holder by renaming it under .kept-, and a sequence that stopped +// counting there would hand the next promotion a number a parked copy already +// carries, so the park would collide or the ordering would repeat. +func TestNextHolderSeqCountsKeptNames(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + kept := fmt.Sprintf("%s%s%020d%s", dest, keptSuffix, 9, holderSeqSuffix) + if err := os.MkdirAll(kept, 0o700); err != nil { + t.Fatal(err) + } + + seq, err := nextHolderSeq(dest) + if err != nil { + t.Fatal(err) + } + if seq != 10 { + t.Errorf("nextHolderSeq = %d, want 10: a parked copy already holds sequence 9", seq) + } +} + +// The review's P3 reproduction. A directory whose NAME collides with the holder +// prefix is not this install's copy, and attributing one by name is what put a +// user's unrelated directory in reach of recovery's moves and deletes. +func TestOwnedHoldersBesideIgnoresAPrefixCollidingSibling(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "model-a") + other := filepath.Join(root, "model-b") + + mine := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 1, holderSeqSuffix) + makeDir(t, mine) + if err := writeHolderMarker(mine, txnMarker{Kind: holderMarkerKind, Dest: "model-a", Seq: 1}); err != nil { + t.Fatal(err) + } + // Named for model-a, marked for model-b: evidence it belongs to another + // destination's transaction, so model-a's pass has no claim on it at all. + foreign := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 2, holderSeqSuffix) + makeDir(t, foreign) + if err := writeHolderMarker(foreign, txnMarker{Kind: holderMarkerKind, Dest: filepath.Base(other), Seq: 2}); err != nil { + t.Fatal(err) + } + // A name the grammar rejects: a user's own directory that happens to start + // the same way. + collide := dest + holderSuffix + "mine" + makeDir(t, collide) + // The grammar passes and no marker backs it: retained and reported, never + // restored and never deleted. + unmarked := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 3, holderSeqSuffix) + makeDir(t, unmarked) + + owned, unowned := ownedHoldersBeside(dest) + + gotOwned := make([]string, 0, len(owned)) + for _, c := range owned { + gotOwned = append(gotOwned, c.path) + } + if want := []string{mine}; !slices.Equal(gotOwned, want) { + t.Errorf("owned = %v, want %v", gotOwned, want) + } + if want := []string{unmarked}; !slices.Equal(unowned, want) { + t.Errorf("unowned = %v, want %v", unowned, want) + } + for _, path := range []string{foreign, collide} { + if slices.Contains(gotOwned, path) || slices.Contains(unowned, path) { + t.Errorf("%s is not this destination's to classify", filepath.Base(path)) + } + } + if len(owned) == 1 && owned[0].seq != 1 { + t.Errorf("owned seq = %d, want 1", owned[0].seq) + } +} + +// makeDir creates one directory the fixtures need, with the mode the allocator +// gives a holder. +func makeDir(t *testing.T, path string) { + t.Helper() + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } +} + +// Parking is how a copy recovery will not restore and cannot prove superseded +// leaves the scan without leaving the disk: the same directory under a prefix +// the scan does not enumerate, keeping its sequence so the operator can still +// name it. +func TestParkKeptHolderMovesTheCopyUnderTheKeptPrefix(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + holder := plantHolder(t, dest, 7, "kept") + + if err := parkKeptHolder(holder); err != nil { + t.Fatalf("park: %v", err) + } + kept := fmt.Sprintf("%s%s%020d%s", dest, keptSuffix, 7, holderSeqSuffix) + got, err := os.ReadFile(filepath.Join(kept, "install", "engine")) + if err != nil || string(got) != "kept" { + t.Fatalf("parked copy = %q (err %v), want %q under %s", got, err, "kept", filepath.Base(kept)) + } + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("the holder should be gone from the scanned prefix, got %v", err) + } + owned, unowned := ownedHoldersBeside(dest) + if len(owned) != 0 || len(unowned) != 0 { + t.Errorf("a parked copy must leave the scan: owned %v unowned %v", owned, unowned) + } +} + +// A park that would have to clear something first is a park onto a copy that is +// not ours to remove, so it fails and leaves both where they are. Only a +// clobbering implementation (remove the destination, then rename) can lose the +// occupant. +func TestParkKeptHolderDoesNotClobber(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + holder := plantHolder(t, dest, 7, "new") + occupied := fmt.Sprintf("%s%s%020d%s", dest, keptSuffix, 7, holderSeqSuffix) + makeDir(t, filepath.Join(occupied, "install")) + if err := os.WriteFile(filepath.Join(occupied, "install", "engine"), []byte("older"), 0o644); err != nil { + t.Fatal(err) + } + + if err := parkKeptHolder(holder); err == nil { + t.Error("parking onto an occupied Kept name must fail") + } + got, err := os.ReadFile(filepath.Join(occupied, "install", "engine")) + if err != nil || string(got) != "older" { + t.Errorf("the occupant was clobbered: %q err %v", got, err) + } + if _, err := os.Stat(filepath.Join(holder, "install", "engine")); err != nil { + t.Errorf("a failed park must leave the holder where it is: %v", err) + } +} + +// soleHolder returns the one holder beside destDir, failing when there is not +// exactly one. +func soleHolder(t *testing.T, destDir string) string { + t.Helper() + holders := holdersFor(t, destDir) + if len(holders) != 1 { + t.Fatalf("want exactly one holder beside %s, got %v", destDir, holders) + } + return holders[0] +} + +// hasFile reports whether path exists. +func hasFile(t *testing.T, path string) bool { + t.Helper() + _, err := os.Lstat(path) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + return err == nil +} + +// The marker goes in before the first destructive rename. A holder that takes +// the only copy of an install before it can be proven ours is one recovery has +// to leave in place forever, since nothing on disk says who wrote it. +func TestPromoteStagedDirWritesTheMarkerBeforeSettingAside(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + injectFault(t, "rename", func(args ...string) bool { + return filepath.Base(args[1]) == "install" + }, errors.New("injected set-aside failure")) + // The holder is cleaned up on this failure path, which is right and would + // also erase what this test is asserting on. + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil); err == nil { + t.Fatal("a failed set-aside must be reported") + } + holder := soleHolder(t, dest) + m, err := readHolderMarker(holder) + if err != nil { + t.Fatalf("the holder was filled before it carried a marker: %v", err) + } + seq, ok := holderStamp(dest, holder) + if !ok { + t.Fatalf("the allocator wrote a name the grammar rejects: %q", filepath.Base(holder)) + } + if m.Kind != holderMarkerKind || m.Dest != filepath.Base(dest) || m.Seq != seq { + t.Errorf("marker = %+v, want kind %q dest %q seq %d", m, holderMarkerKind, filepath.Base(dest), seq) + } + if hasFile(t, filepath.Join(holder, "install")) { + t.Error("the set-aside failed, so nothing should have moved into the holder") + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "old" { + t.Errorf("the previous install must be untouched: %q err %v", got, err) + } +} + +// A torn marker write must leave no marker at all: the atomic rename is what +// makes "missing" and "ours" the only two answers a crash can produce, so a +// partial file can never be read as ownership. +func TestPromoteStagedDirMarkerWriteIsAtomic(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + injectFault(t, "rename", func(args ...string) bool { + return filepath.Base(args[1]) == holderMarkerFile + }, errors.New("injected marker publish failure")) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil); err == nil { + t.Fatal("a failed marker write must be reported") + } + holder := soleHolder(t, dest) + if hasFile(t, filepath.Join(holder, holderMarkerFile)) { + t.Error("a marker that was never published must not be readable") + } + if _, err := readHolderMarker(holder); !errors.Is(err, errMarkerMissing) { + t.Errorf("readHolderMarker = %v, want errMarkerMissing", err) + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "old" { + t.Errorf("the previous install must be untouched: %q err %v", got, err) + } +} + +// The commit flag is what proves to a later pass that the copy in the holder was +// superseded by a publish that actually landed. It goes in after the publish and +// before the cleanup, so a crash in that window costs a retained copy rather +// than the install. +func TestPromoteStagedDirCreatesTheCommitFlagAfterPublish(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + // Fail only the final cleanup, so the holder the flag was written into is + // still there to assert on. + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil); err != nil { + t.Fatalf("promote: %v", err) + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "new" { + t.Fatalf("engine = %q err %v, want %q", got, err, "new") + } + holder := soleHolder(t, dest) + if !hasFile(t, filepath.Join(holder, committedFile)) { + t.Error("a published promotion must mark the copy it superseded") + } + if kept, err := os.ReadFile(filepath.Join(holder, "install", "engine")); err != nil || string(kept) != "old" { + t.Errorf("the superseded copy = %q err %v, want %q", kept, err, "old") + } +} + +// A publish that never happened must leave no evidence that it did. The flag is +// the only thing licensing a later pass to delete the copy in the holder, so a +// flag beside a copy that was never superseded authorizes deleting the only +// install the user has. Both renames into the destination fail, which is the +// state a crash mid-publish leaves: the holder retained, holding everything. +func TestPromoteStagedDirPublishFailureLeavesNoCommitFlag(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + injectFault(t, "rename", func(args ...string) bool { + return args[1] == dest + }, errors.New("injected publish failure")) + + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil); err == nil { + t.Fatal("a failed publish must be reported") + } + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Fatalf("neither rename into %s succeeded, so it should be absent, got %v", dest, err) + } + holder := soleHolder(t, dest) + if kept, err := os.ReadFile(filepath.Join(holder, "install", "engine")); err != nil || string(kept) != "old" { + t.Fatalf("the only copy = %q err %v, want it retained as %q", kept, err, "old") + } + if found := findNamed(t, root, committedFile); len(found) != 0 { + t.Errorf("a promotion that never published must leave no commit flag, found %v", found) + } +} + +// A commit flag that cannot be written leaves the superseded copy on disk. The +// alternative is deleting a whole install on the word of a step that just +// failed, and the report is what tells the user where the copy went. +func TestPromoteStagedDirFailedCommitFlagKeepsTheHolder(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + injectFault(t, "create", func(args ...string) bool { + return filepath.Base(args[0]) == committedFile + }, errors.New("injected commit flag failure")) + var reported []string + report := func(line string) { reported = append(reported, line) } + + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", report); err != nil { + t.Fatalf("a published install must not be reported as a failure: %v", err) + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "new" { + t.Fatalf("engine = %q err %v, want the published %q", got, err, "new") + } + holder := soleHolder(t, dest) + if kept, err := os.ReadFile(filepath.Join(holder, "install", "engine")); err != nil || string(kept) != "old" { + t.Errorf("the superseded copy = %q err %v, want it retained as %q", kept, err, "old") + } + if hasFile(t, filepath.Join(holder, committedFile)) { + t.Error("the flag write failed, so no flag should exist") + } + if !slices.ContainsFunc(reported, func(line string) bool { return strings.Contains(line, holder) }) { + t.Errorf("the report must name the retained copy, got %v", reported) + } +} + +// findNamed lists every path under root whose base name is name. +func findNamed(t *testing.T, root, name string) []string { + t.Helper() + var found []string + if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.Name() == name { + found = append(found, path) + } + return nil + }); err != nil { + t.Fatal(err) + } + return found +} From f21290f816536ba82f7f13cd8a9d688ce92ec518 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:53:24 -0500 Subject: [PATCH 25/34] fix(daemon/remote): decide recovery from proof on disk, not from what a name suggests Recovery walked directory entries and mutated as it went, so each decision saw one entry rather than everything competing for a destination. A staged backup was deleted whenever something existed at the destination, which is not evidence that a later transaction published over it, and a directory was reaped on its name plus an mtime, with no lock, while a live extract could be mid-swap. The pass now scans without touching anything, groups candidates by destination, and reconciles each one under that destination's locks, re-reading every fact it classified on before acting. A copy is deleted only when a commit flag proves a publish landed over it and the destination is a usable work tree, or when its marker proves the transaction never filled it. Anything else is kept: an unreadable candidate stops the destination, an unowned directory is left where it is, and both are reported. An unusable destination is set aside only once a replacement is chosen, and the husk goes back if that restore fails, so a fault cannot leave the destination emptier than it was found. Recovery reads no clock and keeps no memory between passes, so a second pass reaches the same verdict as the first. --- internal/daemon/remote/bundle.go | 565 +++++++++++++---- internal/daemon/remote/bundle_test.go | 868 +++++++++++++++++++++++--- 2 files changed, 1208 insertions(+), 225 deletions(-) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index c6fef22fb..6a1a5aae4 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "io/fs" + "maps" "math" "net" "os" @@ -262,24 +263,57 @@ type extractLock struct { // lockExtract blocks until dest is free and returns its release func. Entries // are refcounted so the map cannot grow with every link id ever uploaded. func lockExtract(dest string) func() { + entry := holdExtractRef(dest) + entry.mu.Lock() + return func() { + entry.mu.Unlock() + dropExtractRef(dest) + } +} + +// tryLockExtract takes the in-process lock without waiting, reporting held when +// a goroutine in this daemon owns dest. Recovery cannot use lockExtract: an +// extract holds its destination across the whole clone, so a recovering pass +// that blocked here would wait out that clone before deciding anything about a +// destination it was going to skip either way. +func tryLockExtract(dest string) (release func(), held bool) { + entry := holdExtractRef(dest) + if !entry.mu.TryLock() { + dropExtractRef(dest) + return nil, true + } + return func() { + entry.mu.Unlock() + dropExtractRef(dest) + }, false +} + +// holdExtractRef returns dest's lock entry with a reference taken, so the entry +// cannot be dropped from the map while a caller is waiting on it. +func holdExtractRef(dest string) *extractLock { extractLocks.mu.Lock() + defer extractLocks.mu.Unlock() entry := extractLocks.locks[dest] if entry == nil { entry = &extractLock{} extractLocks.locks[dest] = entry } entry.refs++ - extractLocks.mu.Unlock() + return entry +} - entry.mu.Lock() - return func() { - entry.mu.Unlock() - extractLocks.mu.Lock() - entry.refs-- - if entry.refs == 0 { - delete(extractLocks.locks, dest) - } - extractLocks.mu.Unlock() +// dropExtractRef releases one reference and forgets the entry once nothing holds +// it, which is what keeps the map from growing with every link id ever seen. +func dropExtractRef(dest string) { + extractLocks.mu.Lock() + defer extractLocks.mu.Unlock() + entry := extractLocks.locks[dest] + if entry == nil { + return + } + entry.refs-- + if entry.refs == 0 { + delete(extractLocks.locks, dest) } } @@ -325,82 +359,419 @@ func tryLockExtractFile(bundleDir, dest string) (release func(), held bool, err return func() { _ = lock.Release() }, false, nil } -// recoverBundleDir repairs what a crash left behind in dir. A staging dir whose -// backup belongs to a link with no live tree is put back; one that no extract -// can still own is removed. It is called once at bridge construction, before any -// upload is served, and never removes a staging dir a live extract may hold. +// recoverBundleDir repairs what a crash left behind in dir. It runs in two +// halves: a scan that discovers and classifies every candidate without touching +// anything, and a reconcile per destination that takes that destination's locks, +// re-validates everything it is about to act on, and only then acts. Splitting +// them is what keeps a decision from being made against state nobody was +// holding, and what keeps one destination's fault from reaching another's. +// +// Nothing here reads a clock, and nothing here remembers the previous pass. Both +// were sources of deletes the on-disk state did not license: an mtime says when +// a directory was written, not who owns it, and a per-pass map makes the second +// pass over the same directory reach a different answer from the first. +// +// It is called once at bridge construction, before any upload is served, and +// never removes a staging dir a live extract may hold. func recoverBundleDir(dir string, logf func(string, ...any)) { if logf == nil { logf = func(string, ...any) {} } + byDest := scanBundleDir(dir, logf) + // Sorted, so two passes over one directory visit destinations in the same + // order and a failure is reproducible rather than a function of readDir. + for _, id := range slices.Sorted(maps.Keys(byDest)) { + reconcileLink(dir, id, byDest[id], logf) + } +} + +// bundleCandidate is a staging directory the scan attributed to a destination: +// its name passed the strict grammar, it holds no work tree at its root, and its +// marker agrees with both the name and a link id the write path would accept. +type bundleCandidate struct { + path string + seq int64 + dest string +} + +// scanBundleDir reads dir once and groups every attributable staging directory +// by the destination its marker names. It mutates nothing: a directory that is +// going to be deleted is decided on under the destination's lock, and this runs +// before any lock is held. Entries it cannot attribute are reported here, once +// per pass, because a copy nothing names is one an operator cannot find. +func scanBundleDir(dir string, logf func(string, ...any)) map[string][]bundleCandidate { entries, err := stagingFS.readDir(dir) if err != nil { if !os.IsNotExist(err) { logf("remote: could not scan bundle dir %s: %v", dir, err) } - return + return nil } - // One link can have several staged backups: a cleanup that could not finish - // leaves one behind, and a later crash adds another. Newest first, so the - // tree that comes back is the most recent one rather than whichever the - // directory happened to list first. The order comes from the sequence the - // extract allocated against the entries already in the directory, not from a - // wall clock and not from a directory mtime. A clock can move backward and - // invert two stamps; an extract that read an existing entry always numbers - // above it. Mtimes stay rejected for their own reasons: they track a tree's - // contents, and a coarse filesystem gives two of them the same value anyway. - staged := make([]stagedExtract, 0, len(entries)) + byDest := map[string][]bundleCandidate{} for _, entry := range entries { if !entry.IsDir() || !strings.HasPrefix(entry.Name(), stagingPrefix) { continue } - stamp, stamped := stagingStamp(entry.Name()) - staged = append(staged, stagedExtract{path: filepath.Join(dir, entry.Name()), stamp: stamp, stamped: stamped}) + staging := filepath.Join(dir, entry.Name()) + seq, ok := stagingStamp(entry.Name()) + if !ok { + logf("remote: %s does not carry a name this code writes; leaving it in place", staging) + continue + } + id, ok := attributeStagingDir(dir, staging, seq, logf) + if !ok { + continue + } + byDest[id] = append(byDest[id], bundleCandidate{path: staging, seq: seq, dest: id}) + } + return byDest +} + +// attributeStagingDir decides whether staging is this code's to act on. The +// order is load-bearing: the work-tree veto runs before the marker is read, +// because link ids starting with '.' used to be accepted, so a published work +// tree can sit under the exact generated name AND carry a file called txn at its +// root. Reading the marker first would let that file license a delete. +func attributeStagingDir(dir, staging string, seq int64, logf func(string, ...any)) (string, bool) { + if _, err := stagingFS.stat(filepath.Join(staging, ".git")); err == nil { + logf("remote: %s holds a work tree, not a staged extract; leaving it in place", staging) + return "", false + } + m, err := readMarker(staging) + if err != nil { + logf("remote: %s has no usable transaction marker (%v); leaving it in place", staging, err) + return "", false + } + if m.Kind != txnKindBundleExtract { + logf("remote: %s carries a %q transaction marker, which no extract wrote; leaving it in place", staging, m.Kind) + return "", false + } + if m.Seq != seq { + // The name is the authority: createSequencedStagingDir walks up from the + // number it asked for, and it reads the name back before using it. A + // marker that disagrees with the name proves nothing about who wrote + // either of them. + logf("remote: %s carries a marker for sequence %d but is named %d; leaving it in place", staging, m.Seq, seq) + return "", false + } + // Everything read off disk goes back through the write path's own checks. A + // marker is a file, and a file can be edited by anything that can reach the + // bundle dir. + id, err := sanitizeLinkID(m.Dest) + if err != nil { + logf("remote: staged tree in %s names an invalid link (%v); leaving it in place", staging, err) + return "", false + } + if !withinDir(dir, filepath.Join(dir, id)) { + logf("remote: staged tree in %s names a link outside the bundle dir; leaving it in place", staging) + return "", false + } + return id, true +} + +// candidateState is one candidate classified per the recovery design. empty, +// committed and usable are three independent facts, and the disposition is a +// function of them plus the destination's own state; nothing else. +type candidateState struct { + bundleCandidate + empty bool + committed bool + usable bool +} + +// candidateVerdict separates the two ways a candidate leaves the reconcile +// without being acted on. Dropped means it is no longer attributable, which is +// this candidate's business alone. Unreadable means a filesystem fault, which +// says nothing about the candidate and stops the whole destination: acting on +// the rest would be deciding against a directory that could not be read. +type candidateVerdict int + +const ( + verdictOwned candidateVerdict = iota + verdictDropped + verdictUnreadable +) + +// reconcileLink classifies and then acts on every candidate for one destination, +// with that destination's in-process and cross-process locks held across both. +// The lock covers the check and the action together: a live extract mid-swap is +// indistinguishable on disk from a crashed one, and the lock is the only thing +// that tells them apart. +func reconcileLink(dir, id string, cands []bundleCandidate, logf func(string, ...any)) { + dest := filepath.Join(dir, id) + unlock, held := tryLockExtract(dest) + if held { + logf("remote: an extract in this process holds %s; leaving its staged copies alone", id) + return + } + defer unlock() + unlockFile, heldFile, err := tryLockExtractFile(dir, dest) + if err != nil { + logf("remote: could not lock %s while recovering it: %v", id, err) + return + } + if heldFile { + logf("remote: another process holds %s; leaving its staged copies alone", id) + return + } + defer unlockFile() + + states, ok := classifyCandidates(dir, id, cands, logf) + if !ok { + return + } + present, usable, ok := classifyDest(dest, logf) + if !ok { + return + } + if present && usable { + reconcileAgainstUsableDest(dir, id, states, logf) + return + } + restoreForDest(dir, id, dest, present, states, logf) +} + +// classifyCandidates re-validates every candidate under the lock and returns +// them newest first. The scan's reading is deliberately not trusted: it ran +// before the lock, so anything it saw could have been changed by the process the +// lock now excludes. +func classifyCandidates(dir, id string, cands []bundleCandidate, logf func(string, ...any)) ([]candidateState, bool) { + states := make([]candidateState, 0, len(cands)) + for _, c := range cands { + st, verdict := classifyCandidate(dir, c, logf) + switch verdict { + case verdictOwned: + states = append(states, st) + case verdictDropped: + continue + case verdictUnreadable: + logf("remote: leaving every staged copy for %s in place until %s can be read", id, c.path) + return nil, false + } + } + // Sequence order is transaction order: an extract that reads an existing + // entry always numbers above it, so a higher sequence is a later + // transaction whatever the clock did in between. + slices.SortStableFunc(states, func(a, b candidateState) int { return cmp.Compare(b.seq, a.seq) }) + return states, true +} + +func classifyCandidate(dir string, c bundleCandidate, logf func(string, ...any)) (candidateState, candidateVerdict) { + st := candidateState{bundleCandidate: c} + id, ok := attributeStagingDir(dir, c.path, c.seq, logf) + if !ok || id != c.dest { + return st, verdictDropped + } + backup := filepath.Join(c.path, "backup") + switch _, err := stagingFS.stat(backup); { + case err == nil: + case os.IsNotExist(err): + // A readable marker with no set-aside content is the one shape that + // proves the directory holds nothing: owned, and empty. + st.empty = true + return st, verdictOwned + default: + logf("remote: could not read the staged tree in %s: %v", c.path, err) + return st, verdictUnreadable + } + switch _, err := stagingFS.stat(filepath.Join(c.path, committedFile)); { + case err == nil: + st.committed = true + case os.IsNotExist(err): + default: + logf("remote: could not read the commit flag in %s: %v", c.path, err) + return st, verdictUnreadable + } + switch _, err := stagingFS.stat(filepath.Join(backup, ".git")); { + case err == nil: + st.usable = true + case os.IsNotExist(err): + // Durable: this copy will never pass the predicate, so it is skipped in + // selection and kept. That is a different fact from the error below. + default: + logf("remote: could not tell whether the staged tree in %s is usable: %v", c.path, err) + return st, verdictUnreadable + } + return st, verdictOwned +} + +// classifyDest reports whether dest exists and whether it can serve. Usability +// is dest/.git present, checked structurally. isGitWorktree shells out to git +// rev-parse, which discovers upward, so a bundle dir under any enclosing +// checkout would answer "usable" for an empty destination and license deleting +// every copy beside it. +func classifyDest(dest string, logf func(string, ...any)) (present, usable, ok bool) { + switch _, err := stagingFS.stat(dest); { + case err == nil: + case os.IsNotExist(err): + return false, false, true + default: + logf("remote: could not read %s: %v; leaving its staged copies in place", dest, err) + return false, false, false + } + switch _, err := stagingFS.stat(filepath.Join(dest, ".git")); { + case err == nil: + return true, true, true + case os.IsNotExist(err): + return true, false, true + default: + logf("remote: could not tell whether %s is usable: %v; leaving its staged copies in place", dest, err) + return true, false, false } - slices.SortStableFunc(staged, func(a, b stagedExtract) int { - if a.stamped != b.stamped { - if a.stamped { - return -1 +} + +// reconcileAgainstUsableDest handles the destination that is present and can +// serve. Only a commit flag licenses a delete here: the flag is written by the +// transaction that published over that exact copy, so it is evidence about this +// copy. "The destination exists" is not, and deleting on it is what loses the +// last copy of a tree when the destination came from anywhere else. +func reconcileAgainstUsableDest(dir, id string, states []candidateState, logf func(string, ...any)) { + for _, c := range states { + switch { + case c.empty: + reapOwnedEmpty(c.path, logf) + case c.committed: + // The upload that published dest reported success to its client + // while this whole copy of the prior tree was still on disk. Say + // what is being reclaimed, so a bridge running without a logger is + // not the difference between the space being accounted for and not. + logf("remote: reclaiming the staged tree in %s that %s's live tree superseded", c.path, id) + if err := stagingFS.removeAll(c.path); err != nil { + logf("remote: could not remove superseded staging dir %s: %v", c.path, err) } - return 1 + default: + logf("remote: keeping the staged tree in %s: nothing proves %s published over it", c.path, id) + parkKeptBackup(dir, c.path, id, logf) } - return cmp.Compare(b.stamp, a.stamp) - }) + } +} - // Which links this pass put a tree back on, and the staging it came from. - restored := map[string]stagedExtract{} - for _, s := range staged { - staging := s.path - if restoreStagedBackup(dir, s, restored, logf) { - continue +// restoreForDest handles the destination that is absent or cannot serve. It +// selects before it moves anything: a destination with no usable candidate is +// left exactly as it was found, because taking its husk apart with nothing to +// put in its place leaves the operator with strictly less than they had. +func restoreForDest(dir, id, dest string, present bool, states []candidateState, logf func(string, ...any)) { + winner := selectCandidate(states) + if winner < 0 { + logf("remote: %s has no usable staged copy to restore; leaving it as it is", id) + parkRemaining(dir, id, states, -1, logf) + return + } + husk := "" + if present { + var err error + husk, err = setAsideHusk(dir, id, dest, logf) + if err != nil { + logf("remote: could not set the unusable tree at %s aside: %v; leaving %s alone", dest, err, id) + return } - // No backup to attribute. A dir with a .git at its root is not staging at - // all: link ids starting with '.' used to be accepted, so this may be a - // work tree someone published under a name that now looks reserved. - // Never reap that. - if _, err := stagingFS.stat(filepath.Join(staging, ".git")); err == nil { - logf("remote: %s holds a work tree, not a staged extract; leaving it in place", staging) - continue + } + sel := states[winner] + if err := fsutil.RenameWithRetry(filepath.Join(sel.path, "backup"), dest, stagingFS.rename); err != nil { + logf("remote: could not restore the staged tree for %s from %s: %v", id, sel.path, err) + // No fallback to an older copy: installing one would put a tree at dest + // that the next pass reads as having published over the newer copy, + // which is how a retained copy turns into a deleted one. + if husk != "" { + putHuskBack(dest, husk, logf) + } + return + } + logf("remote: restored the work tree for %s from %s after an interrupted extract", id, sel.path) + // The winner's directory is now owned and empty, which is the one delete + // that needs no flag. + reapOwnedEmpty(sel.path, logf) + if husk != "" { + logf("remote: keeping the tree that could not serve at %s", dest) + parkKeptBackup(dir, husk, id, logf) + } + parkRemaining(dir, id, states, winner, logf) +} + +// selectCandidate is the whole selection rule: the newest uncommitted usable +// copy, and only when there is none, the newest committed usable one. A +// committed copy is second because the transaction that flagged it published +// something over it; an uncommitted one is the copy no publish is known to have +// replaced. +func selectCandidate(states []candidateState) int { + for i, c := range states { + if !c.empty && !c.committed && c.usable { + return i } - // Only reap once no clone can still be running: gitTimeout bounds a - // clone, so anything older than that is abandoned. - info, err := stagingFS.stat(staging) - if err != nil || time.Since(info.ModTime()) < 2*gitTimeout { + } + for i, c := range states { + if !c.empty && c.committed && c.usable { + return i + } + } + return -1 +} + +// parkRemaining keeps every candidate selection did not restore. Nothing here is +// deleted: without a commit flag beside a usable destination there is no +// evidence any of these was superseded, and a copy with no evidence against it +// may be the last one. +func parkRemaining(dir, id string, states []candidateState, winner int, logf func(string, ...any)) { + for i, c := range states { + if i == winner { continue } - if err := stagingFS.removeAll(staging); err != nil { - logf("remote: could not remove abandoned staging dir %s: %v", staging, err) + if c.empty { + reapOwnedEmpty(c.path, logf) + continue } + logf("remote: keeping the staged tree in %s that %s was not restored from", c.path, id) + parkKeptBackup(dir, c.path, id, logf) + } +} + +// reapOwnedEmpty removes a staging dir that carries a valid marker and holds no +// set-aside content. The marker is what makes this safe: a directory with none +// names no destination, so no lock excludes the live allocation that may be +// sitting between its Mkdir and its marker write right now. +func reapOwnedEmpty(staging string, logf func(string, ...any)) { + if err := stagingFS.removeAll(staging); err != nil { + logf("remote: could not remove the empty staging dir %s: %v", staging, err) } } -// stagedExtract is a staging dir plus the creation order recorded in its name. -// stamped is false for a name this package did not write, which is an ordering -// it must not claim to know. -type stagedExtract struct { - path string - stamp int64 - stamped bool +// setAsideHusk moves a destination that cannot serve into a fresh sequenced +// staging dir, so the restore has somewhere to land. It allocates and attributes +// before it moves anything, so a failure never leaves a copy of a tree in a +// directory no marker names. +func setAsideHusk(dir, id, dest string, logf func(string, ...any)) (string, error) { + seq, err := nextStagingSeq(dir) + if err != nil { + return "", err + } + staging, err := createSequencedStagingDir(dir, seq) + if err != nil { + return "", err + } + claimed, _ := stagingStamp(filepath.Base(staging)) + if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: id, Seq: claimed}); err != nil { + // Nothing has moved yet, so the directory holds nothing and removing it + // loses nothing. Leaving it would be residue no marker attributes. + reapOwnedEmpty(staging, logf) + return "", err + } + if err := fsutil.RenameWithRetry(dest, filepath.Join(staging, "backup"), stagingFS.rename); err != nil { + reapOwnedEmpty(staging, logf) + return "", err + } + return staging, nil +} + +// putHuskBack undoes a set-aside whose restore then failed, so the destination +// is left exactly as recovery found it. If the move back fails too, the husk +// stays where it is as an owned candidate of its own and both failures are +// named: the copy is still on disk, which is the property that matters. +func putHuskBack(dest, husk string, logf func(string, ...any)) { + if err := fsutil.RenameWithRetry(filepath.Join(husk, "backup"), dest, stagingFS.rename); err != nil { + logf("remote: and could not put the tree at %s back from %s: %v", dest, husk, err) + return + } + reapOwnedEmpty(husk, logf) } // stagingStamp reads back the per-directory sequence createSequencedStagingDir @@ -543,80 +914,6 @@ func createSequencedStagingDir(dir string, n int64) (string, error) { return "", fmt.Errorf("remote: could not claim a staging name in %s after %d attempts", dir, stagingSeqAttempts) } -// restoreStagedBackup puts a staged backup back if its link has no live tree. -// It reports whether staging was dealt with and needs no further handling. -// restored carries the links this recovery pass has already put a tree back on. -func restoreStagedBackup(dir string, s stagedExtract, restored map[string]stagedExtract, logf func(string, ...any)) bool { - staging := s.path - backup := filepath.Join(staging, "backup") - if _, err := stagingFS.stat(backup); err != nil { - return false - } - m, err := readMarker(staging) - if err != nil { - logf("remote: staged tree in %s has no usable transaction marker (%v); leaving it in place", staging, err) - return true - } - id, err := sanitizeLinkID(m.Dest) - if err != nil { - logf("remote: staged tree in %s names an invalid link (%v); leaving it in place", staging, err) - return true - } - dest := filepath.Join(dir, id) - if !withinDir(dir, dest) { - logf("remote: staged tree in %s names a link outside the bundle dir; leaving it in place", staging) - return true - } - // A live extract mid-swap looks exactly like a crashed one: its backup is - // aside and dest is briefly absent. Only the lock tells them apart, so skip - // any link something still owns rather than taking its tree. - release, held, err := tryLockExtractFile(dir, dest) - if err != nil { - logf("remote: could not lock %s while recovering %s: %v", id, staging, err) - return true - } - if held { - return true - } - defer release() - if _, err := stagingFS.stat(dest); err == nil { - // The link already has a tree, and a backup only ever holds the tree that - // was live BEFORE it: backup is filled by renaming dest aside, so dest - // holding anything at all means a later extract published over it. That - // is what makes the backup superseded -- not a timestamp comparison, - // which two directories can tie on and which tracks a tree's contents - // rather than when it was promoted. - if from, ours := restored[dest]; ours && (!s.stamped || !from.stamped || s.stamp >= from.stamp) { - // dest is a tree THIS pass put back, so nothing published over this - // backup and the reasoning above does not apply. Without an order - // both names agree on, which of the two is current is unknown. - logf("remote: staged tree in %s cannot be ordered against the tree just restored for %s; keeping it", staging, id) - parkKeptBackup(dir, staging, id, logf) - return true - } - // The upload that published dest reported success to its client while - // this whole copy of the prior tree was still on disk, and - // extractBundle's own cleanup failure is only logged. Say what is being - // reclaimed, so a bridge that ran without a logger configured is not the - // difference between the space being accounted for and not. - logf("remote: reclaiming the staged tree in %s that %s's live tree superseded", staging, id) - if err := stagingFS.removeAll(staging); err != nil { - logf("remote: could not remove superseded staging dir %s: %v", staging, err) - } - return true - } - if err := stagingFS.rename(backup, dest); err != nil { - logf("remote: could not restore the staged tree for %s from %s: %v", id, staging, err) - return true - } - restored[dest] = s - logf("remote: restored the work tree for %s from %s after an interrupted extract", id, staging) - if err := stagingFS.removeAll(staging); err != nil { - logf("remote: could not remove staging dir %s after restoring %s: %v", staging, id, err) - } - return true -} - // parkKeptBackup moves a staging dir out of the prefix recovery scans, so the // next pass leaves alone what this one deliberately kept instead of reading the // restored tree at dest as a later extract publishing over it. A rename onto an diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index 05c3ff9a2..ee2aa9403 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -704,7 +704,11 @@ func TestRecoverBundleDirRefusesAMarkerThatEscapesTheBundleDir(t *testing.T) { for _, marker := range []string{"../evil", "/etc/evil", "..", "a/b", ".hidden"} { dir := t.TempDir() staging := plantInterruptedExtract(t, dir, "proj-1", "v0") - if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: marker}); err != nil { + // Only the destination is hostile: a marker whose sequence disagrees + // with its own name is dropped before the escape check ever runs, and + // the test would then prove nothing about that check. + seq, _ := stagingStamp(filepath.Base(staging)) + if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: marker, Seq: seq}); err != nil { t.Fatal(err) } outside := filepath.Join(filepath.Dir(dir), "evil") @@ -728,18 +732,14 @@ func TestRecoverBundleDirRefusesAMarkerThatEscapesTheBundleDir(t *testing.T) { // renames leaves behind: the link's only tree sitting in a staging dir. func plantInterruptedExtract(t *testing.T, bundleDir, linkID, content string) string { t.Helper() - staging := filepath.Join(bundleDir, stagingPrefix+"crashed") - backup := filepath.Join(staging, "backup") - if err := os.MkdirAll(filepath.Join(backup, ".git"), 0o700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(backup, "a.txt"), []byte(content), 0o644); err != nil { - t.Fatal(err) - } - if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: linkID}); err != nil { + // The name has to pass the strict grammar and the marker has to agree with + // it, or recovery reads the fixture as unowned and every assertion built on + // it passes for the wrong reason. + seq, err := nextStagingSeq(bundleDir) + if err != nil { t.Fatal(err) } - return staging + return stageBackup(t, bundleDir, "", linkID, content, seq) } func TestRecoverBundleDirRestoresInterruptedExtract(t *testing.T) { @@ -764,16 +764,15 @@ func TestRecoverBundleDirRestoresInterruptedExtract(t *testing.T) { } } +// A copy the live tree provably published over is reclaimed. "Provably" is the +// commit flag inside that copy and a destination that can actually serve, not +// the destination merely existing: that reading deletes the last copy of a tree +// whenever the destination came from anywhere else. func TestRecoverBundleDirDropsBackupWhenTheLinkAlreadyHasATree(t *testing.T) { dir := t.TempDir() + live := plantUsableDest(t, dir, "proj-1", "live") staging := plantInterruptedExtract(t, dir, "proj-1", "stale") - live := filepath.Join(dir, "proj-1") - if err := os.MkdirAll(live, 0o700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(live, "a.txt"), []byte("live"), 0o644); err != nil { - t.Fatal(err) - } + markCommitted(t, staging) recoverBundleDir(dir, nil) @@ -818,30 +817,6 @@ func TestRecoverBundleDirLeavesAStagingDirAnExtractCouldStillOwn(t *testing.T) { } } -func TestRecoverBundleDirReapsAbandonedStaging(t *testing.T) { - // Both name shapes: the stamped one is what extractBundle writes now, the - // bare one is any staging dir whose name the reaper cannot read. - for _, name := range []string{"old", "00000000000000000100-old"} { - t.Run(name, func(t *testing.T) { - dir := t.TempDir() - staging := filepath.Join(dir, stagingPrefix+name) - if err := os.MkdirAll(filepath.Join(staging, "repo"), 0o700); err != nil { - t.Fatal(err) - } - old := time.Now().Add(-3 * gitTimeout) - if err := os.Chtimes(staging, old, old); err != nil { - t.Fatal(err) - } - - recoverBundleDir(dir, nil) - - if _, err := os.Stat(staging); !os.IsNotExist(err) { - t.Errorf("a staging dir older than any clone should be reaped, got %v", err) - } - }) - } -} - // A second daemon sharing the bundle dir must not extract the same link at the // same time; the in-process lock cannot see it, so an advisory file lock does. func TestExtractBundleWaitsForACrossProcessLock(t *testing.T) { @@ -1082,6 +1057,9 @@ func parkedStaging(staging string) string { return filepath.Join(filepath.Dir(staging), keptPrefix+strings.TrimPrefix(base, stagingPrefix)) } +// A name this code never wrote carries no order at all, so it is not a candidate +// for anything: not restored, not parked, not deleted. Moving it would be acting +// on a directory nothing attributes to this code. func TestRecoverBundleDirKeepsAnUnorderableBackupAgainstATreeItRestored(t *testing.T) { dir := t.TempDir() stamped := stageBackup(t, dir, "current", "proj-1", "v1", 200) @@ -1097,14 +1075,11 @@ func TestRecoverBundleDirKeepsAnUnorderableBackupAgainstATreeItRestored(t *testi if _, err := os.Stat(filepath.Join(stamped, "backup")); !os.IsNotExist(err) { t.Errorf("the restored staging dir should be cleared, got %v", err) } - if _, err := os.Stat(filepath.Join(parkedStaging(unordered), "backup", "a.txt")); err != nil { - t.Errorf("a backup that cannot be ordered against the restore must be kept: %v", err) - } - if _, err := os.Stat(unordered); !os.IsNotExist(err) { - t.Errorf("the kept backup should be parked out of the scanned prefix, got %v", err) + if _, err := os.Stat(filepath.Join(unordered, "backup", "a.txt")); err != nil { + t.Errorf("a backup under a name this code never wrote must stay where it is: %v", err) } - if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "cannot be ordered") }) { - t.Errorf("keeping an unorderable backup should be reported, got %v", logged) + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "does not carry a name this code writes") }) { + t.Errorf("an unowned backup should be reported, got %v", logged) } } @@ -1117,7 +1092,7 @@ func TestRecoverBundleDirKeepsAnUnorderableBackupAcrossRestarts(t *testing.T) { unordered := stageBackup(t, dir, "leftover", "proj-1", "v2", 0) recoverBundleDir(dir, nil) - if _, err := os.Stat(filepath.Join(parkedStaging(unordered), "backup", "a.txt")); err != nil { + if _, err := os.Stat(filepath.Join(unordered, "backup", "a.txt")); err != nil { t.Fatalf("the first pass should keep the unorderable backup: %v", err) } @@ -1165,10 +1140,10 @@ func keptBackupSurvives(t *testing.T, dir, content string) bool { // where it is, which is still a copy, and never eats the occupant. func TestRecoverBundleDirDoesNotClobberAnOccupiedParkedName(t *testing.T) { dir := t.TempDir() - stageBackup(t, dir, "current", "proj-1", "v1", 200) - unordered := stageBackup(t, dir, "leftover", "proj-1", "v2", 0) + plantUsableDest(t, dir, "proj-1", "live") + keepable := stageBackup(t, dir, "", "proj-1", "v2", 200) - occupied := parkedStaging(unordered) + occupied := parkedStaging(keepable) if err := os.MkdirAll(occupied, 0o700); err != nil { t.Fatal(err) } @@ -1183,16 +1158,26 @@ func TestRecoverBundleDirDoesNotClobberAnOccupiedParkedName(t *testing.T) { if err != nil || string(got) != "not mine" { t.Errorf("the occupant of the parked name must be untouched: got %q err %v", got, err) } - if _, err := os.Stat(filepath.Join(unordered, "backup", "a.txt")); err != nil { + if _, err := os.Stat(filepath.Join(keepable, "backup", "a.txt")); err != nil { t.Errorf("a backup that could not be parked must stay where it is: %v", err) } if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "could not park") }) { t.Errorf("a failed park should be reported, got %v", logged) } + + // The next pass finds it under the scanned prefix again. It still carries no + // commit flag, so it is parked again rather than reclaimed: a park that + // failed must never turn into a delete one pass later. + recoverBundleDir(dir, nil) + if _, err := os.Stat(filepath.Join(keepable, "backup", "a.txt")); err != nil { + t.Errorf("a copy that could not be parked must survive the next pass: %v", err) + } } -// An older backup IS dropped once the tree restored over it is provably newer, -// so the fail-safe above does not turn into a leak of every leftover. +// An older backup is moved out of the scanned prefix once a newer one has been +// restored, so the next pass does not reconsider it. It is not deleted: being +// older than the restored copy is not evidence that anything published over it, +// and only that evidence licenses a delete. func TestRecoverBundleDirDropsAnOlderBackupAfterRestoringANewerOne(t *testing.T) { dir := t.TempDir() stale := stageBackup(t, dir, "stale", "proj-1", "v0", 100) @@ -1205,7 +1190,10 @@ func TestRecoverBundleDirDropsAnOlderBackupAfterRestoringANewerOne(t *testing.T) t.Fatalf("the newest backup should be restored: got %q err %v", got, err) } if _, err := os.Stat(stale); !os.IsNotExist(err) { - t.Errorf("the superseded backup should be removed, got %v", err) + t.Errorf("the older backup should leave the scanned prefix, got %v", err) + } + if _, err := os.Stat(filepath.Join(parkedStaging(stale), "backup", "a.txt")); err != nil { + t.Errorf("the older backup must be kept, not deleted: %v", err) } } @@ -1239,14 +1227,9 @@ func TestRecoverBundleDirKeepsAWorkTreePublishedUnderAReservedName(t *testing.T) // one at dest, so a link that has a tree supersedes it either way. func TestRecoverBundleDirDropsBackupWhenMtimesAreEqual(t *testing.T) { dir := t.TempDir() + live := plantUsableDest(t, dir, "proj-1", "live") staging := plantInterruptedExtract(t, dir, "proj-1", "stale") - live := filepath.Join(dir, "proj-1") - if err := os.MkdirAll(live, 0o700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(live, "a.txt"), []byte("live"), 0o644); err != nil { - t.Fatal(err) - } + markCommitted(t, staging) // What a one-second-granularity filesystem produces for a backup and a // publish that happen in the same second. tie := time.Unix(1700000000, 0) @@ -1403,7 +1386,10 @@ func TestExtractBundleOutOrdersALeftoverStampedInTheFuture(t *testing.T) { } } if _, err := os.Stat(stale); !os.IsNotExist(err) { - t.Errorf("the genuinely older backup should be superseded once the newer one is restored, got %v", err) + t.Errorf("the genuinely older backup should leave the scanned prefix once the newer one is restored, got %v", err) + } + if _, err := os.Stat(filepath.Join(parkedStaging(stale), "backup", "a.txt")); err != nil { + t.Errorf("it must be kept, not deleted: %v", err) } } @@ -1605,8 +1591,10 @@ func TestRecoverBundleDirReportsReclaimingASupersededTree(t *testing.T) { if err := os.MkdirAll(filepath.Join(dest, ".git"), 0o700); err != nil { t.Fatal(err) } - // The cleanup did not, so the old tree is still staged beside it. + // The cleanup did not, so the old tree is still staged beside it, carrying + // the flag that proves the publish landed over it. staging := stageBackup(t, dir, "leftover", "proj-1", "v-old", 100) + markCommitted(t, staging) var logged []string recoverBundleDir(dir, func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }) @@ -1813,15 +1801,11 @@ func TestRecoverBundleDirHandlesLinksIndependently(t *testing.T) { dir := t.TempDir() // proj-1 has no tree, so its backup is restored. restorable := stageBackup(t, dir, "one", "proj-1", "v1", 100) - // proj-2 has a live tree, so its backup was published over and is dropped. + // proj-2 has a usable live tree and a copy whose flag proves that tree + // published over it, so that copy is dropped. + live := plantUsableDest(t, dir, "proj-2", "live") superseded := stageBackup(t, dir, "two", "proj-2", "old", 200) - live := filepath.Join(dir, "proj-2") - if err := os.MkdirAll(live, 0o700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(live, "a.txt"), []byte("live"), 0o644); err != nil { - t.Fatal(err) - } + markCommitted(t, superseded) recoverBundleDir(dir, nil) @@ -1872,32 +1856,28 @@ func TestRecoverBundleDirIsIdempotent(t *testing.T) { } } -// Two backups can both carry names recovery cannot order: v0.8.0 residue, or a -// crash before the marker. Recovery restores one of them and must then keep the -// other rather than deleting it as superseded, because no order exists to say -// which came first. +// Two backups can both carry names this code never wrote: v0.8.0 residue, or a +// crash before the marker. Neither is a candidate, so neither is restored and +// neither is touched. Restoring one would be picking between two copies with +// nothing on disk to order them by, and the loser is then a copy sitting beside +// a destination the next pass reads as having superseded it. func TestRecoverBundleDirKeepsABackupItCannotTellApartFromTheRestore(t *testing.T) { dir := t.TempDir() first := stageBackup(t, dir, "one", "proj-1", "v1", 0) second := stageBackup(t, dir, "two", "proj-1", "v2", 0) - recoverBundleDir(dir, nil) + for pass := 1; pass <= 2; pass++ { + recoverBundleDir(dir, nil) - got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) - if err != nil { - t.Fatalf("nothing was restored: %v", err) - } - kept := 0 - for _, staging := range []string{first, second} { - for _, at := range []string{staging, parkedStaging(staging)} { - if _, err := os.Stat(filepath.Join(at, "backup", "a.txt")); err == nil { - kept++ + if _, err := os.Stat(filepath.Join(dir, "proj-1")); !os.IsNotExist(err) { + t.Fatalf("pass %d: nothing may be restored from a name this code never wrote: %v", pass, err) + } + for _, staging := range []string{first, second} { + if _, err := os.Stat(filepath.Join(staging, "backup", "a.txt")); err != nil { + t.Errorf("pass %d: %s must be retained in place: %v", pass, staging, err) } } } - if kept != 1 { - t.Fatalf("restored %q and kept %d of the two tied backups, want exactly 1 kept", got, kept) - } } // ---- filesystem seam ------------------------------------------------------- @@ -2240,3 +2220,709 @@ func TestBlockStepHoldsTheCallUntilRelease(t *testing.T) { t.Fatalf("the released call did not run: %v", err) } } + +// ---- recovery: reconcile per destination ----------------------------------- + +// markCommitted writes the flag extractBundle writes after a publish lands. It +// is the only evidence a staged copy was superseded, so a fixture that omits it +// is a copy recovery is not allowed to delete. +func markCommitted(t *testing.T, staging string) { + t.Helper() + f, err := os.OpenFile(filepath.Join(staging, committedFile), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } +} + +// stageScratch plants an owned staging dir that holds no copy of any tree: the +// state a crash between the marker and the set-aside leaves. It is the one shape +// besides a committed copy that recovery may delete. +func stageScratch(t *testing.T, dir, linkID string, stamp int64) string { + t.Helper() + staging := filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, stamp, stagingSeqSuffix)) + if err := os.MkdirAll(filepath.Join(staging, "repo"), 0o700); err != nil { + t.Fatal(err) + } + if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: linkID, Seq: stamp}); err != nil { + t.Fatal(err) + } + return staging +} + +// plantUsableDest writes the shape the bundle site's usability predicate looks +// for: a work tree, which is dest/.git present. Nothing here shells out to git, +// because the predicate does not either. +func plantUsableDest(t *testing.T, dir, id, content string) string { + t.Helper() + dest := filepath.Join(dir, id) + if err := os.MkdirAll(filepath.Join(dest, ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dest, "a.txt"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return dest +} + +// plantUnusableDest writes a destination that exists and cannot serve: a husk +// with no .git, which is what a partial restore or a half-finished swap leaves. +func plantUnusableDest(t *testing.T, dir, id, content string) string { + t.Helper() + dest := filepath.Join(dir, id) + if err := os.MkdirAll(dest, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dest, "husk.txt"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return dest +} + +func recoverAndLog(t *testing.T, dir string) []string { + t.Helper() + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }) + return logged +} + +func logged(logs []string, want string) bool { + return slices.ContainsFunc(logs, func(m string) bool { return strings.Contains(m, want) }) +} + +// The commit flag is the only evidence a staged copy was published over. The +// predicate this replaces was "the destination exists", which is not evidence of +// anything: a destination can exist because an outside hand put it there, or +// because a crash left a husk, and deleting the last copy of a tree on that +// reading is the central defect of the old pass. +func TestRecoverBundleDirDeletesOnlyWithACommitFlag(t *testing.T) { + t.Run("beside a usable dest only the flagged copy goes", func(t *testing.T) { + dir := t.TempDir() + plantUsableDest(t, dir, "proj-1", "live") + committed := stageBackup(t, dir, "", "proj-1", "superseded", 100) + markCommitted(t, committed) + uncommitted := stageBackup(t, dir, "", "proj-1", "unproven", 200) + + logs := recoverAndLog(t, dir) + + if _, err := os.Stat(committed); !os.IsNotExist(err) { + t.Errorf("a copy the destination provably published over should be reclaimed, got %v", err) + } + got, err := os.ReadFile(filepath.Join(parkedStaging(uncommitted), "backup", "a.txt")) + if err != nil || string(got) != "unproven" { + t.Errorf("a copy with no commit flag must be kept: got %q err %v", got, err) + } + if _, err := os.Stat(uncommitted); !os.IsNotExist(err) { + t.Errorf("the kept copy should be parked out of the scanned prefix, got %v", err) + } + if !logged(logs, uncommitted) { + t.Errorf("the retained copy should be named in the report, got %v", logs) + } + }) + + t.Run("beside an unusable dest the flagged copy is restored", func(t *testing.T) { + dir := t.TempDir() + dest := plantUnusableDest(t, dir, "proj-1", "husk") + committed := stageBackup(t, dir, "", "proj-1", "last-copy", 100) + markCommitted(t, committed) + + recoverAndLog(t, dir) + + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "last-copy" { + t.Errorf("the only usable copy should be restored over a husk: got %q err %v", got, err) + } + if _, err := os.Stat(committed); !os.IsNotExist(err) { + t.Errorf("the restored copy's directory should be gone, got %v", err) + } + if !keptHuskSurvives(t, dir, "husk") { + t.Error("the husk that was at the destination must be kept, not deleted") + } + }) +} + +// keptHuskSurvives reports whether the husk a set-aside moved out of dest is +// still under a Kept name. keptBackupSurvives reads a.txt; a husk has none. +func keptHuskSurvives(t *testing.T, dir, content string) bool { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), keptPrefix) { + continue + } + got, err := os.ReadFile(filepath.Join(dir, entry.Name(), "backup", "husk.txt")) + if err == nil && string(got) == content { + return true + } + } + return false +} + +// A copy recovery parked is retained permanently. A later successful publish is +// not evidence about it: the commit flag is written into the copy the publishing +// transaction set aside, and a Kept backup is never that copy. +func TestRecoverBundleDirKeepsAnUncommittedCopyAcrossALaterPublish(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + kept := parkedStaging(stageBackup(t, dir, "", "proj-1", "unproven", 500)) + + recoverAndLog(t, dir) + if _, err := os.Stat(filepath.Join(kept, "backup", "a.txt")); err != nil { + t.Fatalf("the uncommitted copy should have been parked: %v", err) + } + + // A real publish over the same destination, then another pass. + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v2"), dest, nil); err != nil { + t.Fatalf("second extract: %v", err) + } + recoverAndLog(t, dir) + + got, err := os.ReadFile(filepath.Join(kept, "backup", "a.txt")) + if err != nil || string(got) != "unproven" { + t.Errorf("a Kept backup must survive a later publish: got %q err %v", got, err) + } +} + +// If the newest usable copy cannot be put back, recovery stops for that +// destination. Falling through to an older copy is what manufactures the +// provenance the next pass then reads as supersession of the newer one. +func TestRecoverBundleDirStopsWhenTheNewestRestoreFails(t *testing.T) { + dir := t.TempDir() + older := stageBackup(t, dir, "", "proj-1", "v1", 100) + newest := stageBackup(t, dir, "", "proj-1", "v2", 200) + dest := filepath.Join(dir, "proj-1") + + fail := func() { + injected := errors.New("injected restore failure") + injectFault(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(newest, "backup") + }, injected) + } + + for pass := 1; pass <= 2; pass++ { + var logs []string + func() { + saved := stagingFS + defer func() { stagingFS = saved }() + fail() + logs = recoverAndLog(t, dir) + }() + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatalf("pass %d: nothing may be installed when the newest copy cannot be: %v", pass, err) + } + for _, staging := range []string{older, newest} { + if _, err := os.Stat(filepath.Join(staging, "backup", "a.txt")); err != nil { + t.Fatalf("pass %d: %s must be retained in place: %v", pass, staging, err) + } + } + if !logged(logs, newest) { + t.Errorf("pass %d: the failed restore should be reported, got %v", pass, logs) + } + } + + // Once the fault is gone the newest comes back and the older is parked. + recoverAndLog(t, dir) + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v2" { + t.Fatalf("the newest copy should be restored once the fault is gone: got %q err %v", got, err) + } + if _, err := os.Stat(filepath.Join(parkedStaging(older), "backup", "a.txt")); err != nil { + t.Errorf("the older copy should be parked, not deleted: %v", err) + } +} + +// Selection is by sequence, and the loser is a copy of a tree like any other, so +// it is parked rather than deleted. +func TestRecoverBundleDirRestoresTheNewestUsableAndParksTheOlder(t *testing.T) { + dir := t.TempDir() + older := stageBackup(t, dir, "", "proj-1", "v1", 100) + newest := stageBackup(t, dir, "", "proj-1", "v2", 200) + + recoverAndLog(t, dir) + + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil || string(got) != "v2" { + t.Fatalf("restored a.txt = %q err %v, want the newest %q", got, err, "v2") + } + if _, err := os.Stat(newest); !os.IsNotExist(err) { + t.Errorf("the restored copy's directory should be gone, got %v", err) + } + if _, err := os.Stat(filepath.Join(parkedStaging(older), "backup", "a.txt")); err != nil { + t.Errorf("the older copy must be parked, not deleted: %v", err) + } +} + +// A copy that cannot serve is skipped in selection rather than installed. The +// old pass selected on "a backup directory exists", which installs a partial +// tree over an absent destination and calls it recovery. +func TestRecoverBundleDirSkipsAnUnusableNewestCandidate(t *testing.T) { + dir := t.TempDir() + older := stageBackup(t, dir, "", "proj-1", "v1", 100) + newest := stageBackup(t, dir, "", "proj-1", "v2", 200) + if err := os.RemoveAll(filepath.Join(newest, "backup", ".git")); err != nil { + t.Fatal(err) + } + + recoverAndLog(t, dir) + + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("restored a.txt = %q err %v, want the newest USABLE %q", got, err, "v1") + } + if _, err := os.Stat(older); !os.IsNotExist(err) { + t.Errorf("the restored copy's directory should be gone, got %v", err) + } + if _, err := os.Stat(filepath.Join(parkedStaging(newest), "backup", "a.txt")); err != nil { + t.Errorf("the unusable copy must be parked, not deleted: %v", err) + } +} + +// Unusable and unreadable are different facts. A durable "this copy will never +// pass the predicate" is a selection decision; a filesystem error is not a fact +// about the copy at all, and acting on it turns a fault into a delete. +func TestRecoverBundleDirStopsOnAnUnreadableCandidate(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root ignores the directory permissions this test relies on") + } + if runtime.GOOS == "windows" { + t.Skip("POSIX directory permissions") + } + for _, tc := range []struct { + name string + // break makes one stat under the staging dir fail with EACCES without + // making the marker unreadable: a candidate whose marker cannot be read + // lands in the unowned branch, where every mutation below passes. + breakIt func(t *testing.T, staging string) + }{ + { + name: "the set-aside copy cannot be stat'd", + breakIt: func(t *testing.T, staging string) { + locked := filepath.Join(staging, "locked") + if err := os.Rename(filepath.Join(staging, "backup"), filepath.Join(staging, "locked-backup")); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(locked, 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(filepath.Join(staging, "locked-backup"), filepath.Join(locked, "backup")); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(locked, "backup"), filepath.Join(staging, "backup")); err != nil { + t.Fatal(err) + } + if err := os.Chmod(locked, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(locked, 0o700) }) + }, + }, + { + name: "the usability probe cannot run", + breakIt: func(t *testing.T, staging string) { + backup := filepath.Join(staging, "backup") + if err := os.Chmod(backup, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(backup, 0o700) }) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + older := stageBackup(t, dir, "", "proj-1", "v1", 100) + newest := stageBackup(t, dir, "", "proj-1", "v2", 200) + tc.breakIt(t, newest) + + logs := recoverAndLog(t, dir) + + if _, err := os.Stat(filepath.Join(dir, "proj-1")); !os.IsNotExist(err) { + t.Errorf("recovery must stop for a destination it cannot read, got %v", err) + } + if _, err := os.Lstat(newest); err != nil { + t.Errorf("the unreadable copy must be retained: %v", err) + } + if _, err := os.Stat(filepath.Join(older, "backup", "a.txt")); err != nil { + t.Errorf("every copy for that destination must be retained: %v", err) + } + if !logged(logs, newest) { + t.Errorf("the unreadable copy should be reported, got %v", logs) + } + }) + } +} + +// The reap is licensed by ownership plus the destination's lock, never by a +// prefix and a clock. A live extract sits between its marker write and its +// set-aside with exactly this shape, and the lock is the only thing that tells +// the two apart. +func TestRecoverBundleDirReapsOwnedScratchOnlyUnderTheLock(t *testing.T) { + t.Run("another process holds the destination", func(t *testing.T) { + dir := t.TempDir() + scratch := stageScratch(t, dir, "proj-1", 100) + lockDir := filepath.Join(dir, lockDirName) + if err := os.MkdirAll(lockDir, 0o700); err != nil { + t.Fatal(err) + } + held, err := lockutil.TryAcquireFileLockAt(dir, filepath.Join(lockDir, "proj-1.lock")) + if err != nil { + t.Fatal(err) + } + + logs := recoverAndLog(t, dir) + if _, err := os.Stat(scratch); err != nil { + t.Errorf("a destination another process owns must be left alone: %v", err) + } + if !logged(logs, "proj-1") { + t.Errorf("the skipped destination should be reported, got %v", logs) + } + + if err := held.Release(); err != nil { + t.Fatal(err) + } + recoverAndLog(t, dir) + if _, err := os.Stat(scratch); !os.IsNotExist(err) { + t.Errorf("owned scratch should be reaped once the lock is free, got %v", err) + } + }) + + t.Run("a goroutine in this process holds the destination", func(t *testing.T) { + dir := t.TempDir() + scratch := stageScratch(t, dir, "proj-1", 100) + release := lockExtract(filepath.Join(dir, "proj-1")) + defer release() + + done := make(chan []string, 1) + go func() { done <- recoverAndLog(t, dir) }() + select { + case logs := <-done: + if !logged(logs, "proj-1") { + t.Errorf("the skipped destination should be reported, got %v", logs) + } + case <-time.After(time.Second): + t.Fatal("recovery blocked on a destination a live extract holds; it must skip it") + } + if _, err := os.Stat(scratch); err != nil { + t.Errorf("a destination a live extract holds must be left alone: %v", err) + } + }) +} + +// A directory with no marker names no destination, so no lock excludes the live +// allocation that may be sitting inside it right now. It is retained and named, +// and the cost of that is one empty directory per crash between the mkdir and +// the marker write. +func TestRecoverBundleDirRetainsAnUnmarkedStagingDir(t *testing.T) { + dir := t.TempDir() + unmarked := filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, 3, stagingSeqSuffix)) + if err := os.MkdirAll(unmarked, 0o700); err != nil { + t.Fatal(err) + } + // The v0.8.0 shape: os.MkdirTemp's decimal suffix, which carries no marker + // and may hold a partial clone. + legacy := filepath.Join(dir, stagingPrefix+"1234567890") + if err := os.MkdirAll(filepath.Join(legacy, "repo"), 0o700); err != nil { + t.Fatal(err) + } + + for pass := 1; pass <= 2; pass++ { + logs := recoverAndLog(t, dir) + for _, path := range []string{unmarked, legacy} { + if _, err := os.Stat(path); err != nil { + t.Errorf("pass %d: %s must be retained: %v", pass, path, err) + } + if !logged(logs, path) { + t.Errorf("pass %d: %s should be reported, got %v", pass, path, logs) + } + } + } +} + +// No decision reads a clock. The old pass reaped on mtime age, which makes the +// outcome depend on a forward clock jump or a suspended host rather than on what +// is on disk. +func TestRecoverBundleDirUsesNoClock(t *testing.T) { + for _, tc := range []struct { + name string + when time.Time + }{ + {name: "fresh", when: time.Now()}, + {name: "a year old", when: time.Now().Add(-365 * 24 * time.Hour)}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + scratch := stageScratch(t, dir, "proj-1", 100) + kept := stageBackup(t, dir, "", "proj-1", "v1", 200) + plantUsableDest(t, dir, "proj-1", "live") + for _, path := range []string{scratch, kept} { + if err := os.Chtimes(path, tc.when, tc.when); err != nil { + t.Fatal(err) + } + } + + recoverAndLog(t, dir) + + if _, err := os.Stat(scratch); !os.IsNotExist(err) { + t.Errorf("owned scratch is reaped whatever its age, got %v", err) + } + if _, err := os.Stat(filepath.Join(parkedStaging(kept), "backup", "a.txt")); err != nil { + t.Errorf("an uncommitted copy is kept whatever its age: %v", err) + } + }) + } +} + +// A destination that exists and cannot serve is a stuck state: the old pass saw +// "dest exists" and deleted the copy that could have replaced it. The husk moves +// only once a candidate has been selected, so a destination nothing can replace +// is never taken apart. +func TestRecoverBundleDirSetsAnUnusableDestinationAside(t *testing.T) { + dir := t.TempDir() + // Inside an enclosing checkout, which is what makes the predicate's shape + // falsifiable: git rev-parse --is-inside-work-tree discovers upward and + // answers true for a husk with no .git of its own, so a predicate that + // shells out reads this destination as usable and parks the copy that could + // have replaced it. The structural check does not discover upward. + if out, err := exec.Command("git", "init", filepath.Dir(dir)).CombinedOutput(); err != nil { + t.Fatalf("git init: %v: %s", err, out) + } + dest := plantUnusableDest(t, dir, "proj-1", "partial") + candidate := stageBackup(t, dir, "", "proj-1", "v1", 100) + + logs := recoverAndLog(t, dir) + + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("the usable copy should be live at the destination: got %q err %v", got, err) + } + if _, err := os.Stat(candidate); !os.IsNotExist(err) { + t.Errorf("the restored copy's directory should be gone, got %v", err) + } + if !keptHuskSurvives(t, dir, "partial") { + t.Error("the husk must be kept under the Kept prefix, not deleted") + } + if !logged(logs, candidate) || !logged(logs, dest) { + t.Errorf("both the restore and the set-aside should be reported, got %v", logs) + } +} + +// A legacy work tree can carry the exact generated name AND a file called txn at +// its root, so the .git veto sits ahead of the marker read and ahead of every +// delete. +func TestRecoverBundleDirNeverTrustsAMarkerInsideAWorkTree(t *testing.T) { + dir := t.TempDir() + tree := filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, 5, stagingSeqSuffix)) + if err := os.MkdirAll(filepath.Join(tree, ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tree, "a.txt"), []byte("someones repo"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeMarker(tree, txnMarker{Kind: txnKindBundleExtract, Dest: "proj-1", Seq: 5}); err != nil { + t.Fatal(err) + } + + logs := recoverAndLog(t, dir) + + if got, err := os.ReadFile(filepath.Join(tree, "a.txt")); err != nil || string(got) != "someones repo" { + t.Errorf("a published work tree must be left exactly as it is: got %q err %v", got, err) + } + if !logged(logs, "work tree") { + t.Errorf("a work tree under a reserved name should be reported, got %v", logs) + } +} + +// The scan runs before the lock is held, so everything it saw could have changed +// by the time the lock is taken. Classifying from the scan is a read of state +// nobody was holding. +func TestRecoverBundleDirRevalidatesUnderTheLock(t *testing.T) { + dir := t.TempDir() + scratch := stageScratch(t, dir, "proj-1", 100) + marker := filepath.Join(scratch, stagingMarkerFile) + + // The marker is there for the scan and gone by the time the lock is held, + // which is what a concurrent hand between the two produces. Under the lock + // the directory is unowned, and an unowned directory is never deleted. + real := stagingFS + var reads atomic.Int64 + stagingFS.readFile = func(name string) ([]byte, error) { + out, err := real.readFile(name) + if name == marker && reads.Add(1) == 1 { + if rmErr := os.Remove(name); rmErr != nil { + t.Error(rmErr) + } + } + return out, err + } + t.Cleanup(func() { stagingFS = real }) + + logs := recoverAndLog(t, dir) + + if _, err := os.Stat(scratch); err != nil { + t.Errorf("a directory that is unowned under the lock must be retained: %v", err) + } + if !logged(logs, scratch) { + t.Errorf("the retained directory should be reported, got %v", logs) + } +} + +// With no uncommitted copy to prefer, a committed one is still the last usable +// copy of the tree, so an absent destination gets it back. The commit flag +// licenses a delete only when a usable destination is there to supersede it. +func TestRecoverBundleDirRestoresACommittedCopyWhoseDestIsGone(t *testing.T) { + dir := t.TempDir() + committed := stageBackup(t, dir, "", "proj-1", "v1", 100) + markCommitted(t, committed) + + recoverAndLog(t, dir) + + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("a committed copy beside an absent destination should be restored: got %q err %v", got, err) + } + if _, err := os.Stat(committed); !os.IsNotExist(err) { + t.Errorf("the restored copy's directory should be gone, got %v", err) + } +} + +// The husk moves only after a candidate is selected. With nothing to put in its +// place, taking it apart would leave the destination absent and the operator +// with strictly less than they started with. +func TestRecoverBundleDirLeavesAnUnusableDestinationWithNoCandidateAlone(t *testing.T) { + dir := t.TempDir() + dest := plantUnusableDest(t, dir, "proj-1", "partial") + unusable := stageBackup(t, dir, "", "proj-1", "v1", 100) + if err := os.RemoveAll(filepath.Join(unusable, "backup", ".git")); err != nil { + t.Fatal(err) + } + unowned := filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, 200, stagingSeqSuffix)) + if err := os.MkdirAll(unowned, 0o700); err != nil { + t.Fatal(err) + } + + for pass := 1; pass <= 2; pass++ { + logs := recoverAndLog(t, dir) + got, err := os.ReadFile(filepath.Join(dest, "husk.txt")) + if err != nil || string(got) != "partial" { + t.Fatalf("pass %d: the destination must be left exactly as planted: got %q err %v", pass, got, err) + } + if _, err := os.Stat(filepath.Join(dest, "a.txt")); !os.IsNotExist(err) { + t.Errorf("pass %d: nothing may be installed over the husk: %v", pass, err) + } + if _, err := os.Stat(filepath.Join(parkedStaging(unusable), "backup", "a.txt")); err != nil { + t.Errorf("pass %d: the unusable copy must be kept: %v", pass, err) + } + if _, err := os.Stat(unowned); err != nil { + t.Errorf("pass %d: the unowned directory must stay in place: %v", pass, err) + } + if pass == 1 && (!logged(logs, unusable) || !logged(logs, unowned) || !logged(logs, "proj-1")) { + t.Errorf("pass %d: all three should be reported, got %v", pass, logs) + } + } +} + +// A failed restore after the husk was set aside must leave the destination +// exactly as it was found. Anything else is recovery that made the state worse +// than the crash did. +func TestRecoverBundleDirPutsTheHuskBackWhenTheRestoreFails(t *testing.T) { + dir := t.TempDir() + dest := plantUnusableDest(t, dir, "proj-1", "partial") + candidate := stageBackup(t, dir, "", "proj-1", "v1", 100) + + for pass := 1; pass <= 2; pass++ { + var logs []string + func() { + saved := stagingFS + defer func() { stagingFS = saved }() + injectFault(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(candidate, "backup") + }, errors.New("injected restore failure")) + logs = recoverAndLog(t, dir) + }() + + got, err := os.ReadFile(filepath.Join(dest, "husk.txt")) + if err != nil || string(got) != "partial" { + t.Fatalf("pass %d: the husk must be back at the destination: got %q err %v", pass, got, err) + } + if _, err := os.Stat(filepath.Join(candidate, "backup", "a.txt")); err != nil { + t.Errorf("pass %d: the candidate must be retained at its staging name: %v", pass, err) + } + if keptHuskSurvives(t, dir, "partial") { + t.Errorf("pass %d: no Kept entry may exist while the restore has not landed", pass) + } + if !logged(logs, candidate) { + t.Errorf("pass %d: the failed restore should be reported, got %v", pass, logs) + } + } + + recoverAndLog(t, dir) + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("once the fault is gone the candidate should be restored: got %q err %v", got, err) + } + if !keptHuskSurvives(t, dir, "partial") { + t.Error("the husk should be parked once the restore lands") + } +} + +// The reap boundary of a live extract: between its marker write and its +// set-aside a staging dir is owned and holds nothing, which is exactly the shape +// the reap deletes. Only the destination's lock separates that from a crash, so +// a recovering pass must find the destination held and leave the whole thing +// alone. blockStep gates every call to a step, and the marker write is a rename +// too, so this gates the set-aside by its arguments instead. +func TestRecoverBundleDirLeavesALiveExtractsScratchAlone(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + inSetAside := make(chan struct{}) + gate := make(chan struct{}) + real := stagingFS + var once sync.Once + stagingFS.rename = func(from, to string) error { + if filepath.Base(to) == "backup" { + once.Do(func() { close(inSetAside) }) + <-gate + } + return real.rename(from, to) + } + t.Cleanup(func() { stagingFS = real }) + + var wg sync.WaitGroup + var extractErr error + wg.Add(1) + go func() { + defer wg.Done() + extractErr = extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil) + }() + + <-inSetAside + staging := soleStaging(t, dir) + if _, err := os.Stat(filepath.Join(staging, "backup")); !os.IsNotExist(err) { + t.Fatalf("the fixture must be an owned staging dir with no set-aside copy yet, got %v", err) + } + recoverBundleDir(dir, nil) + if _, err := os.Stat(filepath.Join(staging, stagingMarkerFile)); err != nil { + t.Errorf("a live extract's scratch must survive a recovering pass: %v", err) + } + close(gate) + wg.Wait() + + if extractErr != nil { + t.Errorf("recovery interfered with a live extract: %v", extractErr) + } + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("a.txt = %q err %v, want the published %q", got, err, "v1") + } +} From 9af200c28d4ae3bb396222ae8eeaa7d57cfb69cf Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:53:24 -0500 Subject: [PATCH 26/34] fix(dictation): apply the caller's own usability test to every candidate The absent-destination branch selected a holder on whether its install directory existed, so a higher-sequence partial copy beat an older complete one: recovery published it, EnsureLocalEngine rejected it, and an offline user was left with a failed download while a working engine sat in the holder next to it. A failed restore of the preferred copy fell through to an older one, which the next pass then read as proof the newer copy had been superseded. Every candidate is now judged by the predicate its caller already uses, and selection prefers the newest usable copy no publish has superseded, falling back only to one that was published over. When the preferred copy cannot be restored, recovery keeps it, reports, and stops for that destination rather than installing an older one that a later pass would misread. A destination that exists but is unusable no longer wedges the install: it is set aside after a replacement is selected and moved back if the restore fails. Copies recovery keeps move under the kept prefix it never enumerates, so a later successful install cannot reclaim them. --- internal/dictation/download.go | 307 +++++++++-- internal/dictation/download_test.go | 765 +++++++++++++++++++++++++--- 2 files changed, 941 insertions(+), 131 deletions(-) diff --git a/internal/dictation/download.go b/internal/dictation/download.go index 7ef3b4bad..f2c5966b7 100644 --- a/internal/dictation/download.go +++ b/internal/dictation/download.go @@ -1173,17 +1173,27 @@ func ownedHoldersBeside(destDir string) (owned []holderCandidate, unowned []stri // else claims, and a park that would have to clear something first is a park // onto a copy that is not ours to remove. func parkKeptHolder(holder string) error { + kept, ok := keptHolderPath(holder) + if !ok { + return fmt.Errorf("%q is not a holder name", filepath.Base(holder)) + } + return fsutil.RenameWithRetry(holder, kept, holderFS.rename) +} + +// keptHolderPath is the Kept name a holder parks under: the same sequence, the +// other prefix. Recovery names it in its report before the rename as well as +// after, so the operator is told the same path whether the park lands or not. +func keptHolderPath(holder string) (string, bool) { dir, name := filepath.Split(holder) cut := strings.LastIndex(name, holderSuffix) if cut < 0 { - return fmt.Errorf("%q is not a holder name", name) + return "", false } destDir := filepath.Join(dir, name[:cut]) if _, ok := holderStamp(destDir, holder); !ok { - return fmt.Errorf("%q is not a holder name", name) + return "", false } - kept := filepath.Join(dir, name[:cut]+keptSuffix+name[cut+len(holderSuffix):]) - return fsutil.RenameWithRetry(holder, kept, holderFS.rename) + return filepath.Join(dir, name[:cut]+keptSuffix+name[cut+len(holderSuffix):]), true } // nextHolderSeq is the number a new holder should claim: one past the highest @@ -1243,74 +1253,263 @@ func createSequencedHolder(destDir string, n int64) (string, error) { return "", fmt.Errorf("could not claim a holder name for %s after %d attempts", filepath.Base(destDir), holderSeqAttempts) } -// restoreInterruptedPromotion puts back an install that promoteStagedDir set -// aside but never replaced, which is what a process stop between its two renames -// leaves behind: destDir absent and the only usable copy in a .previous-* holder -// nothing else looks at. Anything already at destDir wins, and the check for it -// is explicit rather than leaning on os.Rename refusing an existing directory. -// Best effort by design, since the caller can still download a fresh engine. -// published reports whether destDir holds an install this caller can actually -// use. Recovery needs it because "there is something at destDir" and "a -// promotion published there" are different claims, and only the second one -// makes a holder beside it superseded. +// holderState is one owned holder after classification. Recovery decides what +// every candidate IS before it moves anything: a ruling made while the disk is +// half-read is a ruling made on a state that no longer exists by the time it is +// acted on, which is how a copy that was the last usable one gets deleted. +type holderState struct { + holderCandidate + // empty means the marker is there and the set-aside content is not, so the + // holder provably holds no copy of any install. + empty bool + // committed means the publish this copy was set aside for actually landed. + // It is the only evidence that licenses removing a copy. + committed bool + // usable is the CALLER's predicate applied to the copy itself. Recovery has + // no opinion about what an install looks like; the consumer does. + usable bool +} + +// classifyHolder answers what one owned holder is, or reports that it cannot be +// read. Unusable and unreadable are deliberately different answers: the first is +// durable and the copy is skipped and kept, the second is a filesystem fault, +// and turning a fault into a classification is how a transient error becomes a +// permanent ruling about a copy nobody can get back. +func classifyHolder(c holderCandidate, published func(string) bool) (holderState, error) { + st := holderState{holderCandidate: c} + install := filepath.Join(c.path, "install") + // ReadDir rather than Stat: a copy whose contents cannot be listed is one + // whose usability cannot be decided either, and the predicate below would + // answer "unusable" for it without ever having seen it. + if _, err := holderFS.readDir(install); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return holderState{}, err + } + st.empty = true + return st, nil + } + if _, err := holderFS.stat(filepath.Join(c.path, committedFile)); err == nil { + st.committed = true + } else if !errors.Is(err, fs.ErrNotExist) { + return holderState{}, err + } + st.usable = published(install) + return st, nil +} + +// retainHolder gets one copy recovery will not restore out of the scan without +// getting it off the disk. An owned holder that provably holds nothing is +// removed; everything else moves under the Kept prefix, where the next pass does +// not look and the operator can still find it by name. A park that fails leaves +// the copy exactly where it is: the next pass reaches the same branch and tries +// again, and nothing here ever turns a retain into a delete. +func retainHolder(st holderState, report func(string)) { + if st.empty { + if err := holderFS.removeAll(st.path); err != nil { + report(fmt.Sprintf("Could not remove the empty holder %s (%v)", st.path, err)) + } + return + } + kept, ok := keptHolderPath(st.path) + if !ok { + report(fmt.Sprintf("Keeping a copy of this install in %s", st.path)) + return + } + if err := parkKeptHolder(st.path); err != nil { + report(fmt.Sprintf("Keeping a copy of this install in %s; it could not be moved to %s (%v)", st.path, kept, err)) + return + } + report(fmt.Sprintf("Keeping a copy of this install in %s", kept)) +} + +// setAsideUnusableDest moves a destination that exists but holds no usable +// install into a fresh sequenced holder, so a usable copy can be restored over +// it. Fresh rather than the candidate's own holder: nesting one set-aside copy +// inside another is how a copy stops being findable by its own name. Only ever +// called once a candidate has been selected, so a destination nothing can +// replace is never taken apart. +func setAsideUnusableDest(destDir string) (string, error) { + seq, err := nextHolderSeq(destDir) + if err != nil { + return "", err + } + holder, err := createSequencedHolder(destDir, seq) + if err != nil { + return "", err + } + claimed, ok := holderStamp(destDir, holder) + if !ok { + _ = holderFS.removeAll(holder) + return "", fmt.Errorf("holder name %q is not one recovery can order", filepath.Base(holder)) + } + // Same order the write path uses: the marker goes in before the first + // destructive rename, so a stop in this window leaves a holder recovery can + // prove is its own rather than one nothing on disk claims. + if err := writeHolderMarker(holder, txnMarker{Kind: holderMarkerKind, Dest: filepath.Base(destDir), Seq: claimed}); err != nil { + _ = holderFS.removeAll(holder) + return "", err + } + if err := fsutil.RenameWithRetry(destDir, filepath.Join(holder, "install"), holderFS.rename); err != nil { + // Owned and holding nothing, which is the one shape that costs nothing + // to remove. Leaving it would accumulate scratch on every failed pass. + _ = holderFS.removeAll(holder) + return "", err + } + return holder, nil +} + +// restoreInterruptedPromotion reconciles one destination with every copy of it +// beside it. A process stopped between promoteStagedDir's two renames leaves the +// only usable install in a .previous-* holder nothing else looks at, and an +// offline caller has no download to fall back on; a failed cleanup leaves a copy +// that IS superseded; and recovery itself can leave a destination holding a tree +// the caller cannot use. So every candidate is classified before anything moves, +// and the ruling is a function of on-disk state alone: a second pass with no +// memory of the first reaches the same answer. +// +// published reports whether a directory holds an install this caller can +// actually use. It is the whole basis of every decision here, because "there is +// something at destDir" and "a promotion published there" are different claims, +// and only the second makes a copy beside it superseded. func restoreInterruptedPromotion(txn *destTxn, destDir string, published func(string) bool, report func(string)) { + if report == nil { + report = func(string) {} + } // Recovery moves and deletes whole installs. Without destDir's Install lock // it can restore a holder a promotion in another process is parked on and // remove it, leaving that promotion's rollback nothing to put back. if !txn.holds(destDir) { - if report != nil { - report(fmt.Sprintf("Skipping recovery of %s: no install lock is held for it", filepath.Base(destDir))) - } + report(fmt.Sprintf("Skipping recovery of %s: no install lock is held for it", filepath.Base(destDir))) return } - if _, err := holderFS.lstat(destDir); err == nil { - // destDir is live. A holder is only ever filled by renaming destDir - // aside, so a destDir holding a USABLE install means a later promotion - // published over every holder beside it. Those are superseded copies of - // whole installs, and promoteStagedDir's cleanup is the only thing that - // removes them: when it fails the copy is stranded for good, and each - // later replacement strands another. - // - // Merely non-empty is not that evidence. An empty husk, or a half - // populated directory left by something outside this transaction, can - // sit at destDir while the only usable copy is in the holder; reaping on - // either would delete exactly what the caller is about to need, and an - // offline caller has no download to fall back on. So the holder only - // loses to a destination that is genuinely usable. - // - // A removal that fails is left for the next call, which reaches this - // same branch and tries again. - if published != nil && published(destDir) { - owned, _ := ownedHoldersBeside(destDir) - for _, candidate := range owned { - _ = holderFS.removeAll(candidate.path) - } - } + // Every ruling below rests on the predicate. With none there is no evidence + // for any of them, and acting anyway would delete or publish a copy on the + // strength of a directory merely existing. + if published == nil { + report(fmt.Sprintf("Skipping recovery of %s: no usability check was supplied", filepath.Base(destDir))) return } // Only holders this code can prove it wrote for THIS destination. A sibling // that merely collides with the prefix is not a copy of this install and is - // never restored from or removed on its account. - owned, _ := ownedHoldersBeside(destDir) + // never restored from, removed, or reported on its account. + owned, unowned := ownedHoldersBeside(destDir) + for _, path := range unowned { + report(fmt.Sprintf("Keeping %s: nothing in it identifies it as a copy of %s", path, filepath.Base(destDir))) + } + states := make([]holderState, 0, len(owned)) + for _, candidate := range owned { + st, err := classifyHolder(candidate, published) + if err != nil { + // One unreadable candidate stops the whole destination. Ruling on + // the rest would mean choosing between copies while one of them is + // unread, and the copy nobody could read may be the newest. + report(fmt.Sprintf("Stopping recovery of %s: %s could not be read (%v)", filepath.Base(destDir), candidate.path, err)) + return + } + states = append(states, st) + } // Newest first: the sequence in the name, which the marker agrees with, is - // the only record of which copy was live last. - slices.SortStableFunc(owned, func(a, b holderCandidate) int { + // the only record of which copy was live last. No decision here reads a + // clock, so a backward clock cannot reorder them. + slices.SortStableFunc(states, func(a, b holderState) int { return cmp.Compare(b.seq, a.seq) }) - for _, candidate := range owned { - holder := candidate.path - install := filepath.Join(holder, "install") - if _, err := holderFS.stat(install); err != nil { - continue + + destPresent := false + if _, err := holderFS.lstat(destDir); err == nil { + destPresent = true + } else if !errors.Is(err, fs.ErrNotExist) { + report(fmt.Sprintf("Stopping recovery of %s: it could not be read (%v)", filepath.Base(destDir), err)) + return + } + + if destPresent && published(destDir) { + // The destination holds a real install, so every copy beside it was set + // aside by some earlier transaction. Only the ones carrying the commit + // flag are provably superseded by it; the rest may still be the last + // usable copy of something and are kept. + for _, st := range states { + if st.committed && !st.empty { + if err := holderFS.removeAll(st.path); err != nil { + report(fmt.Sprintf("Could not remove the superseded copy in %s (%v)", st.path, err)) + } + continue + } + retainHolder(st, report) } - if err := holderFS.rename(install, destDir); err != nil { - continue + return + } + + // The destination is absent or holds nothing this caller can use, so a + // usable copy beside it is what should be live. Uncommitted first: it was + // set aside by the transaction that never finished, which makes it the most + // recent state anything can prove. + selected := -1 + for i, st := range states { + if !st.empty && !st.committed && st.usable { + selected = i + break + } + } + if selected < 0 { + for i, st := range states { + if !st.empty && st.committed && st.usable { + selected = i + break + } + } + } + if selected < 0 { + if destPresent { + // Nothing beside it can replace it, so taking it apart would leave + // the caller with no destination at all instead of an unusable one. + report(fmt.Sprintf("%s holds no usable install and no copy beside it can replace it; it is left as it is", destDir)) + } + for _, st := range states { + retainHolder(st, report) } - // Only the holder this install came out of is removed; an older one is - // left for a human, never deleted on a guess about which is current. - _ = holderFS.removeAll(holder) return } + + // Classify, select, THEN set aside. A husk moved before a candidate is + // chosen is a destination taken apart for a restore that may never happen, + // and a failed restore would strand it under a prefix recovery never reads. + husk := "" + if destPresent { + var err error + husk, err = setAsideUnusableDest(destDir) + if err != nil { + report(fmt.Sprintf("Leaving %s as it is: it could not be set aside (%v)", destDir, err)) + return + } + } + winner := states[selected] + if err := fsutil.RenameWithRetry(filepath.Join(winner.path, "install"), destDir, holderFS.rename); err != nil { + // No fallback to an older copy: installing one would put a tree at the + // destination that the next pass reads as evidence the newer copy was + // superseded, which is exactly how provenance is lost. + if husk != "" { + if backErr := fsutil.RenameWithRetry(filepath.Join(husk, "install"), destDir, holderFS.rename); backErr != nil { + report(fmt.Sprintf("%s could not be put back and is kept in %s (%v)", destDir, husk, backErr)) + } else if rmErr := holderFS.removeAll(husk); rmErr != nil { + report(fmt.Sprintf("Could not remove the empty holder %s (%v)", husk, rmErr)) + } + } + report(fmt.Sprintf("Keeping the copy of %s in %s: it could not be moved back into place (%v)", filepath.Base(destDir), winner.path, err)) + return + } + // The winner's holder is owned and now holds nothing. + if err := holderFS.removeAll(winner.path); err != nil { + report(fmt.Sprintf("Could not remove the empty holder %s (%v)", winner.path, err)) + } + if husk != "" { + retainHolder(holderState{holderCandidate: holderCandidate{path: husk}}, report) + } + for i, st := range states { + if i != selected { + retainHolder(st, report) + } + } } // promoteStagedDir moves stageDir into place at destDir. os.Rename refuses to diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index 0a601b735..f99b8901d 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -385,7 +385,7 @@ func TestPromoteStagedDirKeepsTheSetAsideCopyWhenRestoreAlsoFails(t *testing.T) func TestRestoreInterruptedPromotionPutsTheInstallBack(t *testing.T) { root := t.TempDir() dest := filepath.Join(root, "engine-1.2.3-linux-x64") - holder := plantHolder(t, dest, 1, "kept") + holder := plantHolder(t, dest, 1, "kept", false) restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) @@ -402,63 +402,122 @@ func TestRestoreInterruptedPromotionPutsTheInstallBack(t *testing.T) { } // A promotion that published its install but could not remove the holder leaves -// a complete copy of the OLD install beside a live one. Nothing else ever -// removes it, and every later replacement adds another, so recovery reaps it. -// The holder is filled by renaming destDir aside, so a destDir that holds -// something means a later promotion published over this holder. -func TestRestoreInterruptedPromotionReapsAHolderSupersededByALiveInstall(t *testing.T) { +// a complete copy of the OLD install beside a live one, and the commit flag +// inside it is the evidence that a publish actually landed over it. Only that +// flag licenses the delete: a copy with no flag may still be the last usable +// one, so it is parked rather than reaped, and a destination that merely EXISTS +// proves nothing about either. +func TestRestoreInterruptedPromotionDeletesOnlyCommittedHolders(t *testing.T) { root := t.TempDir() dest := filepath.Join(root, "engine-1.2.3-linux-x64") - if err := os.MkdirAll(dest, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dest, "engine"), []byte("new"), 0o644); err != nil { - t.Fatal(err) - } - stranded := plantHolder(t, dest, 100, "old") - older := plantHolder(t, dest, 50, "older") + stagedTree(t, dest, "new") + superseded := plantHolder(t, dest, 100, "old", true) + uncommitted := plantHolder(t, dest, 50, "older", false) + // One handle for both passes: the lock is per open file description, so a + // second acquire in this process contends with the first rather than nests. + txn := lockFor(t, dest) - restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) // The live install is never touched. This is the assertion that matters most. got, err := os.ReadFile(filepath.Join(dest, "engine")) if err != nil || string(got) != "new" { t.Fatalf("the live install must be left alone: got %q err %v", got, err) } - for _, holder := range []string{stranded, older} { - if _, err := os.Stat(holder); !os.IsNotExist(err) { - t.Errorf("a holder superseded by the live install should be reaped, got %v", err) - } + if _, err := os.Stat(superseded); !os.IsNotExist(err) { + t.Errorf("a copy proven superseded by its commit flag should be reaped, got %v", err) + } + parked := keptName(t, dest, uncommitted) + if _, err := os.Stat(uncommitted); !os.IsNotExist(err) { + t.Errorf("the uncommitted copy should have left the scanned prefix, got %v", err) + } + kept, err := os.ReadFile(filepath.Join(parked, "install", "engine")) + if err != nil || string(kept) != "older" { + t.Fatalf("the uncommitted copy must survive the park intact: %q err %v", kept, err) + } + assertReports(t, reported, parked) + + // No memory: the parked copy is under a prefix the scan does not read, so a + // second pass has nothing left to decide. + reported = nil + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + if _, err := os.Stat(filepath.Join(parked, "install", "engine")); err != nil { + t.Errorf("the second pass must leave the parked copy alone: %v", err) + } + if len(reported) != 0 { + t.Errorf("a pass with nothing beside the destination should report nothing, got %v", reported) + } +} + +// A copy parked by recovery is permanent: a later install that publishes over +// the destination sets the CURRENT install aside, never a Kept backup, so it can +// never write the commit flag that would license removing one. Nothing but the +// operator takes a Kept backup off disk. +func TestRestoreInterruptedPromotionKeepsAParkedCopyAfterALaterInstall(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "new") + uncommitted := plantHolder(t, dest, 50, "older", false) + txn := lockFor(t, dest) + + restoreInterruptedPromotion(txn, dest, testPublished, nil) + parked := keptName(t, dest, uncommitted) + if _, err := os.Stat(filepath.Join(parked, "install", "engine")); err != nil { + t.Fatalf("seeding a parked copy: %v", err) + } + + // A real later install over the same destination. + stage := stagedTree(t, filepath.Join(root, "stage"), "newest") + if err := promoteStagedDir(txn, stage, dest, "engine", nil); err != nil { + t.Fatalf("the later install failed: %v", err) + } + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + + // Silence is the assertion that the Kept prefix is outside the scan: a pass + // that enumerated it would have to rule on the copy, and every ruling it + // could reach (park, delete, restore) says so in the report. + if len(reported) != 0 { + t.Errorf("recovery must not enumerate the Kept prefix, it reported %v", reported) + } + kept, err := os.ReadFile(filepath.Join(parked, "install", "engine")) + if err != nil || string(kept) != "older" { + t.Errorf("a Kept backup must survive a later successful install: %q err %v", kept, err) + } + live, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(live) != "newest" { + t.Errorf("the later install should be live: %q err %v", live, err) } } // A destDir that is merely NOT EMPTY is no evidence a promotion published there. -// An empty husk and a half-populated one are the same thing to recovery, and -// reaping on either would delete the copy the user still needs, so the holder -// only loses to a destination that holds a genuinely usable install. -func TestRestoreInterruptedPromotionKeepsAHolderWhenDestIsNotAUsableInstall(t *testing.T) { +// An empty husk and a half-populated one are the same thing to recovery, and a +// copy beside one may be the last usable install there is. So the destination is +// not what wins: the usable copy is restored over the husk, and the husk itself +// is set aside rather than deleted, which is how an offline caller stops being +// stuck beside an install it cannot use. +func TestRestoreInterruptedPromotionReplacesADestThatIsNotAUsableInstall(t *testing.T) { for _, tc := range []struct { - name string - seed func(t *testing.T, dest string) - usable func(string) bool + name string + seed func(t *testing.T, dest string) + usable func(string) bool + content string }{ { - name: "empty husk", - seed: func(t *testing.T, dest string) {}, - usable: func(dir string) bool { bin, _ := resolveEnginePaths(dir, false); return fileExists(bin) }, + name: "empty husk", + seed: func(t *testing.T, dest string) {}, + usable: func(dir string) bool { bin, _ := resolveEnginePaths(dir, false); return fileExists(bin) }, + content: "bin/sherpa-onnx-offline", }, { name: "non-empty but no engine binary", seed: func(t *testing.T, dest string) { t.Helper() - if err := os.MkdirAll(filepath.Join(dest, "bin"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dest, "bin", "README"), []byte("partial"), 0o644); err != nil { - t.Fatal(err) - } + plantUnusableDest(t, dest) }, - usable: func(dir string) bool { bin, _ := resolveEnginePaths(dir, false); return fileExists(bin) }, + usable: func(dir string) bool { bin, _ := resolveEnginePaths(dir, false); return fileExists(bin) }, + content: "bin/sherpa-onnx-offline", }, { name: "model dir without tokens.txt", @@ -468,7 +527,8 @@ func TestRestoreInterruptedPromotionKeepsAHolderWhenDestIsNotAUsableInstall(t *t t.Fatal(err) } }, - usable: dirHasModel, + usable: dirHasModel, + content: "tokens.txt", }, } { t.Run(tc.name, func(t *testing.T) { @@ -478,26 +538,46 @@ func TestRestoreInterruptedPromotionKeepsAHolderWhenDestIsNotAUsableInstall(t *t t.Fatal(err) } tc.seed(t, dest) - holder := plantHolder(t, dest, 100, "the only copy") + // The copy has to pass the case's own predicate, which is the whole + // point: recovery applies the CALLER's predicate to a candidate, not + // a structural guess about what an install looks like. + holder := plantHolder(t, dest, 100, "the only copy", false) + install := filepath.Join(holder, "install") + if err := os.MkdirAll(filepath.Dir(filepath.Join(install, tc.content)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(install, tc.content), []byte("x"), 0o755); err != nil { + t.Fatal(err) + } restoreInterruptedPromotion(lockFor(t, dest), dest, tc.usable, nil) - got, err := os.ReadFile(filepath.Join(holder, "install", "engine")) + if !tc.usable(dest) { + t.Errorf("the usable copy should be live at %s", dest) + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) if err != nil || string(got) != "the only copy" { - t.Errorf("a dest that is not a usable install must not cost the holder its copy: got %q err %v", got, err) + t.Errorf("the destination should hold the copy: got %q err %v", got, err) + } + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("the restored copy's holder should be cleared, got %v", err) } }) } } -// A holder is a leftover, never a replacement for whatever is already at destDir, -// empty or not. os.Rename refuses an existing directory either way, so this -// pins the behavior rather than one implementation of it. What happens to the -// holder afterwards differs by case and is asserted below. -func TestRestoreInterruptedPromotionLeavesAnExistingDestAlone(t *testing.T) { - for _, tc := range []struct{ name, live string }{ - {"empty dest", ""}, - {"populated dest", "live"}, +// A holder never replaces a destination that holds a USABLE install: that +// install is the published one and the copy beside it is the leftover. Against +// an unusable destination the ruling inverts, and the husk moves aside rather +// than being deleted. +func TestRestoreInterruptedPromotionKeepsAUsableDestAndReplacesAnUnusableOne(t *testing.T) { + for _, tc := range []struct { + name string + live string + committed bool + }{ + {name: "empty dest", live: ""}, + {name: "populated dest", live: "live", committed: true}, } { t.Run(tc.name, func(t *testing.T) { root := t.TempDir() @@ -510,27 +590,29 @@ func TestRestoreInterruptedPromotionLeavesAnExistingDestAlone(t *testing.T) { t.Fatal(err) } } - install := filepath.Join(plantHolder(t, dest, 1, "stale"), "install") + holder := plantHolder(t, dest, 1, "stale", tc.committed) restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) got, err := os.ReadFile(filepath.Join(dest, "engine")) if tc.live == "" { - if err == nil { - t.Fatalf("an existing dest was replaced by a stale holder: engine = %q", got) + // The husk was unusable, so the copy beside it wins and the husk + // is set aside under a Kept name rather than dropped. + if err != nil || string(got) != "stale" { + t.Fatalf("the usable copy should have replaced the husk: %q err %v", got, err) } - } else if err != nil || string(got) != tc.live { - t.Fatalf("engine = %q, err %v, want the live %q", got, err, tc.live) + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("the restored copy's holder should be cleared, got %v", err) + } + return } - // A live dest supersedes the holder and reaps it; an empty husk is - // no such evidence and the holder stays. Either way the assertion - // above stands: dest is never replaced by a holder. - _, err = os.Stat(install) - if tc.live == "" && err != nil { - t.Errorf("an empty dest must leave the holder intact: %v", err) + if err != nil || string(got) != tc.live { + t.Fatalf("engine = %q, err %v, want the live %q", got, err, tc.live) } - if tc.live != "" && !os.IsNotExist(err) { - t.Errorf("a live dest should reap the holder it superseded, got %v", err) + // Its commit flag proves the live install published over it, which + // is the only thing that licenses the delete. + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("a live dest should reap the copy it provably superseded, got %v", err) } }) } @@ -613,8 +695,10 @@ func testPublished(dir string) bool { // plantHolder writes an install into a holder named and marked the way // promoteStagedDir writes one, so recovery sees the same shape it does in // production: the name alone is not ownership, and a fixture without the marker -// would test a directory recovery is supposed to leave alone. -func plantHolder(t *testing.T, destDir string, seq int64, content string) string { +// would test a directory recovery is supposed to leave alone. committed plants +// the flag the publish writes, which is the only evidence that licenses removing +// the copy, so a fixture has to say which of the two it is. +func plantHolder(t *testing.T, destDir string, seq int64, content string, committed bool) string { t.Helper() holder := fmt.Sprintf("%s%s%020d%s", destDir, holderSuffix, seq, holderSeqSuffix) install := filepath.Join(holder, "install") @@ -627,17 +711,80 @@ func plantHolder(t *testing.T, destDir string, seq int64, content string) string if err := writeHolderMarker(holder, txnMarker{Kind: holderMarkerKind, Dest: filepath.Base(destDir), Seq: seq}); err != nil { t.Fatal(err) } + if committed { + if err := writeCommitFlag(holder); err != nil { + t.Fatal(err) + } + } return holder } +// keptName is the name a parked holder takes: the same sequence under the Kept +// prefix, which is what makes a retained copy findable by the operator and +// invisible to the next scan. +func keptName(t *testing.T, destDir, holder string) string { + t.Helper() + seq, ok := holderStamp(destDir, holder) + if !ok { + t.Fatalf("holder name %q carries no sequence", filepath.Base(holder)) + } + return fmt.Sprintf("%s%s%020d%s", destDir, keptSuffix, seq, holderSeqSuffix) +} + +// plantUnowned creates a directory carrying the exact holder name grammar and no +// marker: the shape recovery must retain in place rather than restore from or +// reap, since nothing on disk attributes it. +func plantUnowned(t *testing.T, destDir string, seq int64, content string) string { + t.Helper() + holder := fmt.Sprintf("%s%s%020d%s", destDir, holderSuffix, seq, holderSeqSuffix) + install := filepath.Join(holder, "install") + if err := os.MkdirAll(install, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(install, "engine"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return holder +} + +// plantUnusableDest leaves a directory at destDir that testPublished rejects: +// present, non-empty, and holding no install. It is the husk the review's P2 +// chain ends at, and the state recovery has to be able to leave. +func plantUnusableDest(t *testing.T, destDir string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(destDir, "bin"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(destDir, "bin", "README"), []byte("partial"), 0o644); err != nil { + t.Fatal(err) + } +} + +// reporterFor collects the recovery report so a retain assertion can check the +// copy was named. A copy recovery keeps and never names is one an operator +// cannot find. +func reporterFor(reported *[]string) func(string) { + return func(m string) { *reported = append(*reported, m) } +} + +// assertReports fails unless some report line mentions each wanted fragment. +func assertReports(t *testing.T, reported []string, want ...string) { + t.Helper() + for _, fragment := range want { + if !slices.ContainsFunc(reported, func(m string) bool { return strings.Contains(m, fragment) }) { + t.Errorf("the report should name %q, got %v", fragment, reported) + } + } +} + // A cleanup that could not finish leaves an old holder behind; a later // interrupted promotion adds a second one. Recovery has to put back the newer // install, and Glob's lexical order is no evidence of which that is. func TestRestoreInterruptedPromotionPrefersTheNewestHolder(t *testing.T) { root := t.TempDir() dest := filepath.Join(root, "engine-1.2.3-linux-x64") - stale := plantHolder(t, dest, 100, "stale") - current := plantHolder(t, dest, 200, "current") + stale := plantHolder(t, dest, 100, "stale", false) + current := plantHolder(t, dest, 200, "current", false) restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) @@ -648,9 +795,13 @@ func TestRestoreInterruptedPromotionPrefersTheNewestHolder(t *testing.T) { if _, err := os.Stat(current); !os.IsNotExist(err) { t.Errorf("the restored holder should be cleared, got %v", err) } - // The loser is left for a human rather than deleted on a guess. - if _, err := os.Stat(filepath.Join(stale, "install", "engine")); err != nil { - t.Errorf("the older holder must be left intact: %v", err) + // The loser is kept rather than deleted on a guess, and it moves under the + // Kept prefix so the next pass has nothing left to rule on. + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Errorf("the older holder should have left the scanned prefix, got %v", err) + } + if _, err := os.Stat(filepath.Join(keptName(t, dest, stale), "install", "engine")); err != nil { + t.Errorf("the older holder must be kept intact under the Kept prefix: %v", err) } } @@ -789,7 +940,7 @@ func TestRestoreInterruptedPromotionFindsHoldersUnderAnAwkwardPath(t *testing.T) t.Run(dirName, func(t *testing.T) { root := filepath.Join(t.TempDir(), dirName) dest := filepath.Join(root, "engine-1.2.3-linux-x64") - install := filepath.Join(plantHolder(t, dest, 100, "kept"), "install") + install := filepath.Join(plantHolder(t, dest, 100, "kept", false), "install") if _, err := os.Stat(install); err != nil { t.Fatal(err) } @@ -819,7 +970,7 @@ func TestRestoreInterruptedPromotionPrefersTheRealNewerInstallOverAFutureStamped if err := os.WriteFile(filepath.Join(dest, "engine"), []byte("new"), 0o644); err != nil { t.Fatal(err) } - stale := plantHolder(t, dest, farFuture, "stale") + stale := plantHolder(t, dest, farFuture, "stale", false) // The real transaction sets "new" aside and never publishes. txn := lockFor(t, dest) @@ -831,8 +982,8 @@ func TestRestoreInterruptedPromotionPrefersTheRealNewerInstallOverAFutureStamped if err != nil || string(got) != "new" { t.Errorf("restored %q (err %v), want the install that was live last, %q", got, err, "new") } - // Whichever holder won, the one that lost is left for a human. - if _, err := os.Stat(filepath.Join(stale, "install", "engine")); err != nil { + // Whichever holder won, the one that lost is kept, under the Kept prefix. + if _, err := os.Stat(filepath.Join(keptName(t, dest, stale), "install", "engine")); err != nil { t.Errorf("a holder that lost the ordering must be kept, not deleted: %v", err) } } @@ -1047,7 +1198,7 @@ func TestRestoreInterruptedPromotionRestoresOnlyFromAnOwnedHolder(t *testing.T) t.Fatal(err) } } - owned := plantHolder(t, dest, 100, "ours") + owned := plantHolder(t, dest, 100, "ours", false) restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) @@ -1073,7 +1224,7 @@ func TestRestoreInterruptedPromotionRestoresOnlyFromAnOwnedHolder(t *testing.T) func TestRestoreInterruptedPromotionSkipsAHolderWithNoInstall(t *testing.T) { root := t.TempDir() dest := filepath.Join(root, "engine-1.2.3-linux-x64") - usable := plantHolder(t, dest, 100, "kept") + usable := plantHolder(t, dest, 100, "kept", false) // Newer AND owned, so ordering reaches it first and it is a real candidate, // but it holds nothing: exactly what a stop between the allocation and the // set-aside rename leaves. @@ -1094,6 +1245,11 @@ func TestRestoreInterruptedPromotionSkipsAHolderWithNoInstall(t *testing.T) { if _, err := os.Stat(usable); !os.IsNotExist(err) { t.Errorf("the restored holder should be cleared, got %v", err) } + // A holder that is provably this code's and provably holds nothing costs + // nothing to remove, and leaving it is what accumulates scratch forever. + if _, err := os.Stat(empty); !os.IsNotExist(err) { + t.Errorf("an owned holder with no install should be removed, got %v", err) + } } // ---- filesystem seam ------------------------------------------------------- @@ -1506,7 +1662,7 @@ func TestPromoteStagedDirRefusesWithoutADestinationLock(t *testing.T) { func TestRestoreInterruptedPromotionRefusesWithoutADestinationLock(t *testing.T) { root := t.TempDir() dest := filepath.Join(root, "engine-1.2.3-linux-x64") - holder := plantHolder(t, dest, 100, "the only copy") + holder := plantHolder(t, dest, 100, "the only copy", false) var reported []string restoreInterruptedPromotion(nil, dest, testPublished, func(s string) { reported = append(reported, s) }) @@ -1923,7 +2079,7 @@ func makeDir(t *testing.T, path string) { func TestParkKeptHolderMovesTheCopyUnderTheKeptPrefix(t *testing.T) { root := t.TempDir() dest := filepath.Join(root, "engine-1.2.3-linux-x64") - holder := plantHolder(t, dest, 7, "kept") + holder := plantHolder(t, dest, 7, "kept", false) if err := parkKeptHolder(holder); err != nil { t.Fatalf("park: %v", err) @@ -1949,7 +2105,7 @@ func TestParkKeptHolderMovesTheCopyUnderTheKeptPrefix(t *testing.T) { func TestParkKeptHolderDoesNotClobber(t *testing.T) { root := t.TempDir() dest := filepath.Join(root, "engine-1.2.3-linux-x64") - holder := plantHolder(t, dest, 7, "new") + holder := plantHolder(t, dest, 7, "new", false) occupied := fmt.Sprintf("%s%s%020d%s", dest, keptSuffix, 7, holderSeqSuffix) makeDir(t, filepath.Join(occupied, "install")) if err := os.WriteFile(filepath.Join(occupied, "install", "engine"), []byte("older"), 0o644); err != nil { @@ -2170,3 +2326,458 @@ func findNamed(t *testing.T, root, name string) []string { } return found } + +// ---- recovery reconciliation ---------------------------------------------- + +// Which copy is NEWEST and which copy is USABLE are different questions, and the +// review's P2 is what happens when the code answers only the first: a stop +// between the holder's creation and the set-aside leaves a newer holder whose +// install directory is there and empty, and restoring from it publishes nothing +// over a destination that has nothing either. The caller's predicate is what +// decides, and it has to reach every candidate, not just the winner. +func TestRestoreInterruptedPromotionAppliesThePredicateToEveryHolder(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + usable := plantHolder(t, dest, 1, "the only copy", false) + // Newer, owned, and holding an install directory with nothing in it: the + // exact shape a Stat-only check reads as the copy to restore. + newer := plantHolder(t, dest, 2, "", false) + if err := os.Remove(filepath.Join(newer, "install", "engine")); err != nil { + t.Fatal(err) + } + + var reported []string + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, reporterFor(&reported)) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "the only copy" { + t.Fatalf("recovery published a copy its own caller cannot use: %q err %v", got, err) + } + if _, err := os.Stat(usable); !os.IsNotExist(err) { + t.Errorf("the restored copy's holder should be cleared, got %v", err) + } + parked := keptName(t, dest, newer) + if _, err := os.Stat(parked); err != nil { + t.Errorf("the unusable copy should be kept under the Kept prefix: %v", err) + } + assertReports(t, reported, parked) +} + +// Falling back to an older copy after the newest one could not be restored is +// what manufactures the provenance loss: an older tree lands at the destination +// and the next pass reads it as evidence that the newer copy was superseded. So +// a failed restore stops the destination, keeps every copy, and says so. +func TestRestoreInterruptedPromotionStopsWhenTheNewestRestoreFails(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + older := plantHolder(t, dest, 1, "older", false) + newest := plantHolder(t, dest, 2, "newest", false) + txn := lockFor(t, dest) + + // The fault belongs to the two failing passes only, so the third pass runs + // against a real filesystem and shows the retry the report promises. + func() { + saved := holderFS + defer func() { holderFS = saved }() + injectFault(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(newest, "install") + }, errors.New("injected restore failure")) + + for pass := 1; pass <= 2; pass++ { + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Fatalf("pass %d: an older copy was installed after the newest failed: %v", pass, err) + } + for _, holder := range []string{older, newest} { + if _, err := os.Stat(filepath.Join(holder, "install", "engine")); err != nil { + t.Errorf("pass %d: every copy must be kept where it is: %v", pass, err) + } + } + assertReports(t, reported, newest) + } + }() + + restoreInterruptedPromotion(txn, dest, testPublished, nil) + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "newest" { + t.Fatalf("the pass after the fault cleared should restore the newest copy: %q err %v", got, err) + } + if _, err := os.Stat(filepath.Join(keptName(t, dest, older), "install", "engine")); err != nil { + t.Errorf("the older copy must be kept: %v", err) + } +} + +// The review's P2 chain end to end: recovery leaves a partial tree at the +// destination, the caller's predicate rejects it, and an offline start then has +// no engine while a usable copy sits beside it. The exit is to set the husk +// aside and restore the copy, which costs one Kept backup and leaves the user +// with a working install instead of a permanent stuck state. +func TestRestoreInterruptedPromotionExitsTheStuckState(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + plantUnusableDest(t, dest) + holder := plantHolder(t, dest, 1, "the only copy", false) + txn := lockFor(t, dest) + + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "the only copy" { + t.Fatalf("the usable copy should be live at the destination: %q err %v", got, err) + } + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("the restored copy's holder should be cleared, got %v", err) + } + // The husk moves into a FRESH sequenced holder, above the one the copy came + // out of, so holders never nest and the operator can name it. + husk := filepath.Join(keptName(t, dest, fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 2, holderSeqSuffix)), "install") + if _, err := os.Stat(filepath.Join(husk, "bin", "README")); err != nil { + t.Errorf("the husk must be kept, not deleted: %v", err) + } + assertReports(t, reported, filepath.Dir(husk)) + + // No memory: the destination is usable now and the husk is under the Kept + // prefix, so there is nothing left to rule on. + reported = nil + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + if got, err := os.ReadFile(filepath.Join(dest, "engine")); err != nil || string(got) != "the only copy" { + t.Errorf("the second pass changed the live install: %q err %v", got, err) + } + if len(reported) != 0 { + t.Errorf("the second pass should have nothing to report, got %v", reported) + } + + // A later real install sets the now-usable destination aside as an ordinary + // holder and the Kept husk is still where recovery put it. + stage := stagedTree(t, filepath.Join(root, "stage"), "newest") + if err := promoteStagedDir(txn, stage, dest, "engine", nil); err != nil { + t.Fatalf("the later install failed: %v", err) + } + if _, err := os.Stat(filepath.Join(husk, "bin", "README")); err != nil { + t.Errorf("a later install must not disturb a Kept backup: %v", err) + } +} + +// An unusable destination with nothing usable beside it is not a state recovery +// can improve, and moving the husk out of the way anyway would leave the caller +// with no destination at all. So the destination is left exactly as found and +// only the report changes. +func TestRestoreInterruptedPromotionLeavesAnUnusableDestinationWithNoCandidateAlone(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + plantUnusableDest(t, dest) + // Owned, uncommitted, and holding an install nothing can use. + unusable := plantHolder(t, dest, 1, "", false) + if err := os.Remove(filepath.Join(unusable, "install", "engine")); err != nil { + t.Fatal(err) + } + unowned := plantUnowned(t, dest, 2, "not ours") + txn := lockFor(t, dest) + + for pass := 1; pass <= 2; pass++ { + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + + if _, err := os.Stat(filepath.Join(dest, "bin", "README")); err != nil { + t.Fatalf("pass %d: the destination must be left exactly as found: %v", pass, err) + } + if testPublished(dest) { + t.Fatalf("pass %d: nothing was restored, so the destination cannot have become usable", pass) + } + parked := keptName(t, dest, unusable) + if _, err := os.Stat(filepath.Join(parked, "install")); err != nil { + t.Errorf("pass %d: the unusable copy should be kept under the Kept prefix: %v", pass, err) + } + if _, err := os.Stat(filepath.Join(unowned, "install", "engine")); err != nil { + t.Errorf("pass %d: an unowned directory is never moved: %v", pass, err) + } + if pass == 1 { + assertReports(t, reported, parked, unowned, dest) + } else { + assertReports(t, reported, unowned, dest) + } + } +} + +// The husk moves only after a candidate is chosen, and it moves BACK when the +// restore fails. Parking it first would strand the destination's own contents +// under a prefix recovery never enumerates, which is the one outcome worse than +// the stuck state this branch exists to exit. +func TestRestoreInterruptedPromotionPutsTheHuskBackWhenTheRestoreFails(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + plantUnusableDest(t, dest) + holder := plantHolder(t, dest, 1, "the only copy", false) + txn := lockFor(t, dest) + + func() { + saved := holderFS + defer func() { holderFS = saved }() + injectFault(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(holder, "install") + }, errors.New("injected restore failure")) + + for pass := 1; pass <= 2; pass++ { + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + + if _, err := os.Stat(filepath.Join(dest, "bin", "README")); err != nil { + t.Fatalf("pass %d: the destination must be left exactly as found: %v", pass, err) + } + if _, err := os.Stat(filepath.Join(holder, "install", "engine")); err != nil { + t.Errorf("pass %d: the copy must stay at its own name: %v", pass, err) + } + kept, err := filepath.Glob(dest + keptSuffix + "*") + if err != nil { + t.Fatal(err) + } + if len(kept) != 0 { + t.Errorf("pass %d: a failed restore must leave no Kept backup, got %v", pass, kept) + } + assertReports(t, reported, holder) + } + }() + + restoreInterruptedPromotion(txn, dest, testPublished, nil) + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "the only copy" { + t.Fatalf("the pass after the fault cleared should restore the copy: %q err %v", got, err) + } + kept, err := filepath.Glob(dest + keptSuffix + "*") + if err != nil { + t.Fatal(err) + } + if len(kept) != 1 { + t.Fatalf("the husk should be kept exactly once, got %v", kept) + } + if _, err := os.Stat(filepath.Join(kept[0], "install", "bin", "README")); err != nil { + t.Errorf("the husk must be kept intact: %v", err) + } +} + +// A committed copy is only provably superseded by a destination that is present +// AND usable. With no such destination it is the last usable copy there is, so +// it is the fallback selection rather than something to reclaim. +func TestRestoreInterruptedPromotionRestoresACommittedHolderOverAnUnusableDestination(t *testing.T) { + for _, tc := range []struct { + name string + seedDest func(t *testing.T, dest string) + wantHusk bool + }{ + { + name: "unusable destination", + seedDest: plantUnusableDest, + wantHusk: true, + }, + { + name: "absent destination", + seedDest: func(t *testing.T, dest string) {}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + tc.seedDest(t, dest) + holder := plantHolder(t, dest, 1, "the only copy", true) + + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "the only copy" { + t.Fatalf("the last usable copy should be live: %q err %v", got, err) + } + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("the restored copy's holder should be cleared, got %v", err) + } + kept, err := filepath.Glob(dest + keptSuffix + "*") + if err != nil { + t.Fatal(err) + } + if tc.wantHusk { + if len(kept) != 1 { + t.Fatalf("the husk should be kept exactly once, got %v", kept) + } + if _, err := os.Stat(filepath.Join(kept[0], "install", "bin", "README")); err != nil { + t.Errorf("the husk must be kept intact: %v", err) + } + return + } + if len(kept) != 0 { + t.Errorf("nothing was set aside, so there is no Kept backup to write: %v", kept) + } + }) + } +} + +// A directory that merely starts like a holder name was never this install's, +// so it is neither acted on nor reported on this install's account. Reporting it +// would be as wrong as moving it: it tells the operator a copy of their install +// is somewhere it is not. +func TestRestoreInterruptedPromotionIgnoresAPrefixCollidingSibling(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "live") + sibling := dest + holderSuffix + "notes" + if err := os.MkdirAll(sibling, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(sibling, "note"), []byte("mine"), 0o644); err != nil { + t.Fatal(err) + } + committed := plantHolder(t, dest, 1, "old", true) + + var reported []string + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, reporterFor(&reported)) + + if got, err := os.ReadFile(filepath.Join(sibling, "note")); err != nil || string(got) != "mine" { + t.Errorf("a prefix-colliding sibling must be left exactly as found: %q err %v", got, err) + } + if _, err := os.Stat(committed); !os.IsNotExist(err) { + t.Errorf("the superseded copy should be reaped, got %v", err) + } + for _, m := range reported { + if strings.Contains(m, sibling) { + t.Errorf("a sibling that is not this install's must not be reported on its account: %q", m) + } + } +} + +// Unusable and unreadable are different answers. Unusable is durable and the +// candidate is skipped and kept; unreadable is a filesystem fault, and deciding +// anything on it would turn a transient error into a permanent ruling. So a +// candidate that cannot be read stops the destination with everything intact. +func TestRestoreInterruptedPromotionStopsOnAnUnreadableHolder(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root ignores the directory permissions this test relies on") + } + if runtime.GOOS == "windows" { + t.Skip("POSIX directory permissions") + } + for _, tc := range []struct { + name string + // seed makes the holder's install unreadable without touching the holder + // itself: chmod the holder and the MARKER stops being readable too, the + // directory is classified unowned, and the branch under test is never + // reached. + seed func(t *testing.T, holder string) + }{ + { + name: "install is a symlink through an unreadable directory", + seed: func(t *testing.T, holder string) { + t.Helper() + locked := filepath.Join(holder, "locked") + if err := os.Rename(filepath.Join(holder, "install"), locked); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(locked, "inner"), filepath.Join(holder, "install")); err != nil { + t.Fatal(err) + } + if err := os.Chmod(locked, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(locked, 0o700) }) + }, + }, + { + name: "install itself cannot be read", + seed: func(t *testing.T, holder string) { + t.Helper() + install := filepath.Join(holder, "install") + if err := os.Chmod(install, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(install, 0o700) }) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + holder := plantHolder(t, dest, 1, "the only copy", false) + tc.seed(t, holder) + + var reported []string + // The delete is recorded rather than inferred from the disk: a + // holder whose own contents cannot be read cannot be removed + // either, so a pass that classified it as empty and ASKED for the + // delete leaves exactly the same directory behind as one that + // never touched it. + removed := recordRemoveAll(t) + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, reporterFor(&reported)) + + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Errorf("nothing readable was found, so nothing may be published: %v", err) + } + if _, err := os.Lstat(holder); err != nil { + t.Errorf("a copy that could not be read must be left where it is: %v", err) + } + if slices.Contains(*removed, holder) { + t.Errorf("a copy that could not be read must never be handed to a delete: %v", *removed) + } + assertNoOtherEntries(t, root, filepath.Base(holder), installLockDir) + assertReports(t, reported, holder) + }) + } +} + +// A directory carrying the exact generated name with no marker behind it is one +// this code cannot claim. Restoring from it would publish a stranger's tree as +// the user's install, and deleting it would destroy something that was never +// ours, so it is kept where it is and named. +func TestRestoreInterruptedPromotionRetainsUnownedHolders(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + unowned := plantUnowned(t, dest, 1, "not ours") + txn := lockFor(t, dest) + + for pass := 1; pass <= 2; pass++ { + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Errorf("pass %d: an unowned directory must never be restored: %v", pass, err) + } + if got, err := os.ReadFile(filepath.Join(unowned, "install", "engine")); err != nil || string(got) != "not ours" { + t.Errorf("pass %d: an unowned directory must be left exactly as found: %q err %v", pass, got, err) + } + assertReports(t, reported, unowned) + } +} + +// The predicate is what tells a published install from a directory that merely +// exists. Without one there is no such evidence, and every ruling recovery could +// make rests on it, so a caller that supplies none gets no mutation at all. +func TestRestoreInterruptedPromotionNilPredicateNeverDeletesOrRestores(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "live") + committed := plantHolder(t, dest, 1, "old", true) + + restoreInterruptedPromotion(lockFor(t, dest), dest, nil, nil) + + if got, err := os.ReadFile(filepath.Join(dest, "engine")); err != nil || string(got) != "live" { + t.Errorf("the destination must be left alone: %q err %v", got, err) + } + if got, err := os.ReadFile(filepath.Join(committed, "install", "engine")); err != nil || string(got) != "old" { + t.Errorf("with no predicate nothing is proven superseded: %q err %v", got, err) + } + assertNoOtherEntries(t, root, filepath.Base(dest), filepath.Base(committed), installLockDir) +} + +// recordRemoveAll collects every path recovery asks the seam to delete, and +// passes each call through. It is how a test tells "the delete was refused by +// the filesystem" from "the delete was never asked for", which the directory +// left on disk cannot show. +func recordRemoveAll(t *testing.T) *[]string { + t.Helper() + var paths []string + real := holderFS + t.Cleanup(func() { holderFS = real }) + holderFS.removeAll = func(path string) error { + paths = append(paths, path) + return real.removeAll(path) + } + return &paths +} From 062c88dd533ed8d38a76b11a26e9aa8fc259ef96 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:53:24 -0500 Subject: [PATCH 27/34] test(daemon/remote,dictation): run the crash states as a table Each row drives the real writer to a crash state with an injected fault, then runs recovery twice and asserts both halves of the outcome: which tree is live, and what happened to every other copy. Driving the writer rather than planting directories means the fixture is whatever the code actually leaves behind. The second pass is asserted because recovery keeps no memory, so a copy kept on one pass must not read as superseded on the next. A companion table renames unrelated directories into every shape the grammar accepts and plants markers that disagree with their own name, kind or destination, so a directory can never be owned by looking like one. --- internal/daemon/remote/bundle_matrix_test.go | 395 ++++++++++++++++++ internal/dictation/download_matrix_test.go | 413 +++++++++++++++++++ 2 files changed, 808 insertions(+) create mode 100644 internal/daemon/remote/bundle_matrix_test.go create mode 100644 internal/dictation/download_matrix_test.go diff --git a/internal/daemon/remote/bundle_matrix_test.go b/internal/daemon/remote/bundle_matrix_test.go new file mode 100644 index 000000000..947f00bc2 --- /dev/null +++ b/internal/daemon/remote/bundle_matrix_test.go @@ -0,0 +1,395 @@ +package remote + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +// The crash rows of the recovery plan's Acceptance Examples, as code. Each row +// drives the REAL extract into the on-disk state a stop at one step leaves, then +// runs recovery twice and asserts both halves the table asserts: which tree is +// live, and where every other copy of a tree ended up. Nothing here plants a +// staging dir by hand, so the fixture is whatever the write path actually +// writes; a hand-built one would only prove recovery agrees with the test's idea +// of the write path. +// +// A second pass is asserted because recovery keeps no memory: a copy retained on +// pass one must not be reclassified on pass two, and a copy deleted on pass one +// must not come back. + +// copyDisposition is where one copy of a tree ends up. The five values are the +// only terminal states the table uses, so a row that means "retained" has to say +// which kind of retention, and a delete can never be written down as anything +// else. +type copyDisposition int + +const ( + // copyDeleted: gone from disk, under neither prefix. + copyDeleted copyDisposition = iota + // copyAtStaging: still under the name the extract allocated for it. + copyAtStaging + // copyAtKept: moved under the Kept prefix, where the scan does not look. + copyAtKept + // copyInPlaceUnowned: still at its name and carrying no marker, so nothing + // on disk attributes it to this code. + copyInPlaceUnowned + // copyAtDest: restored, so the copy's content is the live tree and its + // directory is gone. + copyAtDest +) + +// bundleCrashRow is one row of the table. Each of these rows leaves exactly one +// staged copy, and the assertion compares the WHOLE set of copies on disk +// against that one, so a row can never pass while a second copy nobody expected +// sits beside it. +type bundleCrashRow struct { + id string + // arrange runs the real extract with faults injected to stop it at one step. + // The faults are unwound before recovery runs, so recovery sees a crashed + // tree and not a failing filesystem. + arrange func(t *testing.T, dir, dest string) + wantLive string + wantCopy copyDisposition + // report1 and report2 assert the copy is named in the recovery report on + // that pass. A copy recovery retains but never names is one an operator + // cannot find, which the table counts as a failure of the row. + report1 bool + report2 bool +} + +// arrangeInterruptedSwap stops the extract between its two swap renames: the +// prior tree is aside in backup, the publish failed, and the restore failed too, +// so the destination is absent and that backup is the only copy of it. S4 and S7 +// differ in how the writer got here and are identical on disk, which is why they +// share one arrangement. +func arrangeInterruptedSwap(t *testing.T, dir, dest string) { + t.Helper() + injected := errors.New("injected swap failure") + injectFault(t, "rename", func(args ...string) bool { + from := filepath.Base(args[0]) + return from == "repo" || from == "backup" + }, injected) + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil); err == nil { + t.Fatal("an extract whose publish and restore both fail must report an error") + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatalf("the interrupted swap should leave %s absent, got %v", dest, err) + } +} + +func bundleCrashRows() []bundleCrashRow { + return []bundleCrashRow{ + { + // A stop between the directory and its marker leaves a directory + // nothing on disk attributes. Deleting it would need proof this code + // wrote it, and the marker is that proof, so it is retained and + // named rather than reaped on the strength of its name. + id: "S1", + arrange: func(t *testing.T, dir, dest string) { + injectFault(t, "createTemp", func(args ...string) bool { + return args[1] == stagingMarkerFile+"-*" + }, errors.New("injected marker temp failure")) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil); err == nil { + t.Fatal("an extract whose marker cannot be written must fail") + } + }, + wantLive: "v0", + wantCopy: copyInPlaceUnowned, + report1: true, + report2: true, + }, + { + // The marker landed and the clone did not, so the directory is owned + // and holds no copy of any tree. Nothing can be lost by removing it, + // and leaving it is what accumulates scratch forever. + id: "S2", + arrange: func(t *testing.T, dir, dest string) { + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := extractBundle(context.Background(), filepath.Join(dir, "missing.bundle"), dest, nil); err == nil { + t.Fatal("cloning a bundle that does not exist must fail") + } + }, + wantLive: "v0", + wantCopy: copyDeleted, + }, + { + // The prior tree is aside and the destination is absent, so the + // staged copy is the only copy of it and has to come back. + id: "S4", + arrange: arrangeInterruptedSwap, + wantLive: "v0", + wantCopy: copyAtDest, + }, + { + // The publish landed and the flag did not. Without the flag there is + // no evidence the copy in backup was superseded, so it may not be + // deleted; parking it is what keeps it out of the next pass's way + // without dropping the last copy of a tree. + id: "S5", + arrange: func(t *testing.T, dir, dest string) { + injectFault(t, "create", func(args ...string) bool { + return filepath.Base(args[0]) == committedFile + }, errors.New("injected commit flag failure")) + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil); err != nil { + t.Fatalf("a committed publish must be reported as success: %v", err) + } + }, + wantLive: "v1", + wantCopy: copyAtKept, + report1: true, + }, + { + // The publish failed and the restore put the tree back, so the + // directory holds no copy of anything: owned and empty. + id: "S6", + arrange: func(t *testing.T, dir, dest string) { + injected := errors.New("injected publish failure") + injectFault(t, "rename", func(args ...string) bool { + return filepath.Base(args[0]) == "repo" + }, injected) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil); !errors.Is(err, injected) { + t.Fatalf("extract = %v, want the injected publish failure", err) + } + }, + wantLive: "v0", + wantCopy: copyDeleted, + }, + { + // The writer's own retain branch: publish failed, restore failed, the + // copy was kept and named in the error. On disk this is S4, and + // recovery must not tell them apart, because nothing on disk does. + id: "S7", + arrange: arrangeInterruptedSwap, + wantLive: "v0", + wantCopy: copyAtDest, + }, + { + // The flag is there, so the copy in backup is provably superseded by + // the tree now live at the destination. This is the one and only + // shape that licenses deleting a copy of a tree. + id: "S8", + arrange: func(t *testing.T, dir, dest string) { + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil); err != nil { + t.Fatalf("a cleanup failure must be logged, not returned: %v", err) + } + }, + wantLive: "v1", + wantCopy: copyDeleted, + }, + } +} + +func TestBundleCrashMatrix(t *testing.T) { + for _, row := range bundleCrashRows() { + t.Run(row.id, func(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + // The faults belong to the crash, not to recovery: restoring the + // seam here is what makes the next lines a recovery over a real + // filesystem rather than over a failing one. + func() { + saved := stagingFS + defer func() { stagingFS = saved }() + row.arrange(t, dir, dest) + }() + staging := soleStaging(t, dir) + + for pass, wantReport := range []bool{row.report1, row.report2} { + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { + logged = append(logged, fmt.Sprintf(format, args...)) + }) + assertBundleTerminalState(t, dir, dest, staging, row.wantCopy, row.wantLive, pass+1) + if wantReport { + assertCopyReported(t, logged, stagingSeqDigitsOf(t, staging), pass+1) + } + } + }) + } +} + +// assertBundleTerminalState asserts both halves of a row: the live tree, and the +// full set of copies still on disk under either prefix. The set is compared +// whole rather than one path at a time, so an extra copy nobody expected fails +// the row instead of going unnoticed. +func assertBundleTerminalState(t *testing.T, dir, dest, staging string, want copyDisposition, wantLive string, pass int) { + t.Helper() + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + switch { + case wantLive == "": + if err == nil { + t.Errorf("pass %d: the destination should be absent, it holds %q", pass, got) + } + case err != nil || string(got) != wantLive: + t.Errorf("pass %d: dest a.txt = %q (err %v), want %q", pass, got, err, wantLive) + } + + var wantPaths []string + switch want { + case copyAtStaging, copyInPlaceUnowned: + wantPaths = []string{staging} + case copyAtKept: + wantPaths = []string{parkedStaging(staging)} + } + assertCopySet(t, dir, []string{stagingPrefix, keptPrefix}, wantPaths, pass) + + switch want { + case copyInPlaceUnowned: + // Retained is not the whole claim: the reason it is retained is that + // nothing on disk attributes it, and a row that stopped checking that + // would pass over a copy recovery could have proved was its own. + if _, err := readMarker(staging); !errors.Is(err, errMarkerMissing) { + t.Errorf("pass %d: %s should carry no marker, readMarker = %v", pass, staging, err) + } + case copyAtDest: + if _, err := os.Stat(dest); err != nil { + t.Errorf("pass %d: the copy should have been restored to the destination: %v", pass, err) + } + } +} + +// stagingSeqDigitsOf is the sequence a copy carries in its name. It survives a +// park, so a report assertion keyed on it holds whether the copy is still under +// the staging prefix or has moved under the Kept one. +func stagingSeqDigitsOf(t *testing.T, staging string) string { + t.Helper() + base := filepath.Base(staging) + digits := strings.TrimSuffix(strings.TrimPrefix(base, stagingPrefix), stagingSeqSuffix) + if len(digits) != stagingSeqDigits { + t.Fatalf("staging name %q carries no sequence", base) + } + return digits +} + +// A copy recovery keeps and never names is one an operator cannot find, so the +// report is half of every retain row rather than a nicety. +func assertCopyReported(t *testing.T, logged []string, seq string, pass int) { + t.Helper() + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, seq) }) { + t.Errorf("pass %d: the retained copy (sequence %s) should be named in the report, got %v", pass, seq, logged) + } +} + +// assertCopySet compares every directory under the given prefixes against the +// paths the row expects, resolving both sides through EvalSymlinks because the +// temp root is a symlink on macOS and a path built from t.TempDir would then +// never equal one read back out of the directory. +func assertCopySet(t *testing.T, parent string, prefixes, want []string, pass int) { + t.Helper() + entries, err := os.ReadDir(parent) + if err != nil { + t.Fatal(err) + } + var got []string + for _, entry := range entries { + if !entry.IsDir() { + continue + } + for _, prefix := range prefixes { + if strings.HasPrefix(entry.Name(), prefix) { + got = append(got, resolvedPath(t, filepath.Join(parent, entry.Name()))) + break + } + } + } + wantResolved := make([]string, 0, len(want)) + for _, path := range want { + wantResolved = append(wantResolved, resolvedPath(t, path)) + } + slices.Sort(got) + slices.Sort(wantResolved) + if !slices.Equal(got, wantResolved) { + t.Errorf("pass %d: copies on disk = %v, want %v", pass, got, wantResolved) + } +} + +// resolvedPath resolves symlinks so the two sides of a path comparison are the +// same path. A path that does not exist cannot be resolved, and its cleaned form +// is what the comparison then reports. +func resolvedPath(t *testing.T, path string) string { + t.Helper() + if resolved, err := filepath.EvalSymlinks(path); err == nil { + return resolved + } + return filepath.Clean(path) +} + +// Ownership is a name plus a marker that agrees with it, and neither half alone. +// A directory that only carries the generated name is a legacy work tree or a +// sibling someone renamed, and recovery restoring from one, or reaping one, is +// the whole class of defect the marker exists to close. Each case here is a +// single check dropped: the marker, its kind, its sequence, its destination. +func TestOwnershipCannotBeForgedBySiblingNames(t *testing.T) { + const forgedSeq = 7 + for _, tc := range []struct { + name string + marker *txnMarker + }{ + {name: "no marker at all"}, + {name: "another site's kind", marker: &txnMarker{Kind: "dictation-promote", Dest: "proj-1", Seq: forgedSeq}}, + {name: "sequence disagrees with the name", marker: &txnMarker{Kind: txnKindBundleExtract, Dest: "proj-1", Seq: forgedSeq + 1}}, + {name: "destination is not a link id", marker: &txnMarker{Kind: txnKindBundleExtract, Dest: ".hidden", Seq: forgedSeq}}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + // Built outside and renamed in, so nothing about it was ever written + // by this code: the name is the only thing it shares with a staging + // dir, and it is shaped like a restorable one. + unrelated := filepath.Join(t.TempDir(), "unrelated") + if err := os.MkdirAll(filepath.Join(unrelated, "backup", ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(unrelated, "backup", "a.txt"), []byte("forged"), 0o644); err != nil { + t.Fatal(err) + } + forged := filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, forgedSeq, stagingSeqSuffix)) + if err := os.Rename(unrelated, forged); err != nil { + t.Fatal(err) + } + if tc.marker != nil { + if err := writeMarker(forged, *tc.marker); err != nil { + t.Fatal(err) + } + } + + for pass := 1; pass <= 2; pass++ { + recoverBundleDir(dir, nil) + got, err := os.ReadFile(filepath.Join(forged, "backup", "a.txt")) + if err != nil || string(got) != "forged" { + t.Fatalf("pass %d: the forged directory must be left exactly as it was, got %q (err %v)", pass, got, err) + } + // A restore, a park, or a reap all show up here: any of them + // either creates a destination or moves the directory. + assertNoOtherEntries(t, dir, filepath.Base(forged), lockDirName) + } + }) + } +} + +// assertNoOtherEntries fails when anything but the named entries is in dir. It +// is how "nothing was owned" is asserted without naming every way ownership +// could have been acted on: a restore, a park, and a reap all change this set. +func assertNoOtherEntries(t *testing.T, dir string, allowed ...string) { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !slices.Contains(allowed, entry.Name()) { + t.Errorf("%s should hold only %v, found %s", dir, allowed, entry.Name()) + } + } +} diff --git a/internal/dictation/download_matrix_test.go b/internal/dictation/download_matrix_test.go new file mode 100644 index 000000000..db76086d7 --- /dev/null +++ b/internal/dictation/download_matrix_test.go @@ -0,0 +1,413 @@ +package dictation + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +// The crash rows of the recovery plan's Acceptance Examples, as code. Each row +// drives the REAL promotion into the on-disk state a stop at one step leaves, +// then runs recovery twice and asserts both halves the table asserts: which +// install is live, and where every other copy of an install ended up. Nothing +// here plants a holder by hand, so the fixture is whatever the write path +// actually writes. +// +// A second pass is asserted because recovery keeps no memory: a copy retained on +// pass one must not be reclassified on pass two, and a copy deleted on pass one +// must not come back. + +// copyDisposition is where one copy of an install ends up. The five values are +// the only terminal states the table uses, so a row that means "retained" has to +// say which kind of retention, and a delete can never be written down as +// anything else. +type copyDisposition int + +const ( + // copyDeleted: gone from disk, under neither prefix. + copyDeleted copyDisposition = iota + // copyAtHolder: still under the name the promotion allocated for it. + copyAtHolder + // copyAtKept: moved under the Kept prefix, where the scan does not look. + copyAtKept + // copyInPlaceUnowned: still at its name and carrying no marker, so nothing + // on disk attributes it to this code. + copyInPlaceUnowned + // copyAtDest: restored, so the copy's content is the live install and its + // holder is gone. + copyAtDest +) + +// dictationCrashRow is one row of the table. Each of these rows leaves exactly +// one holder, and the assertion compares the WHOLE set of copies beside the +// destination against that one, so a row can never pass while a second copy +// nobody expected sits beside it. +type dictationCrashRow struct { + id string + // arrange runs the real promotion with faults injected to stop it at one + // step. The faults are unwound before recovery runs, so recovery sees a + // crashed install and not a failing filesystem. + arrange func(t *testing.T, txn *destTxn, stage, dest string) + wantLive string + wantCopy copyDisposition + // report1 and report2 assert the copy is named in the recovery report on + // that pass. A copy recovery retains but never names is one an operator + // cannot find, which the table counts as a failure of the row. + report1 bool + report2 bool +} + +// arrangeInterruptedPromotion stops the promotion between its two renames: the +// previous install is aside in the holder, the publish failed, and the restore +// failed too, so the destination is absent and the holder has the only copy of +// it. D2 and D5 differ in how the writer got here and are identical on disk, +// which is why they share one arrangement. +func arrangeInterruptedPromotion(t *testing.T, txn *destTxn, stage, dest string) { + t.Helper() + injectFault(t, "rename", func(args ...string) bool { + return filepath.Base(args[1]) == filepath.Base(dest) + }, errors.New("injected publish and restore failure")) + if err := promoteStagedDir(txn, stage, dest, "engine", nil); err == nil { + t.Fatal("a promotion whose publish and restore both fail must report an error") + } + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Fatalf("the interrupted promotion should leave %s absent, got %v", dest, err) + } +} + +func dictationCrashRows() []dictationCrashRow { + return []dictationCrashRow{ + { + // A stop between the holder and its marker leaves a directory + // nothing on disk attributes. Deleting it would need proof this code + // wrote it, and the marker is that proof, so it is retained and + // named rather than reaped on the strength of its name. + id: "D1", + arrange: func(t *testing.T, txn *destTxn, stage, dest string) { + injectFault(t, "createTemp", func(args ...string) bool { + return args[1] == holderMarkerFile+"-*" + }, errors.New("injected marker temp failure")) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := promoteStagedDir(txn, stage, dest, "engine", nil); err == nil { + t.Fatal("a promotion whose marker cannot be written must fail") + } + }, + wantLive: "old", + wantCopy: copyInPlaceUnowned, + report1: true, + report2: true, + }, + { + // The marker landed and the set-aside did not, so the holder is + // owned and holds no copy of any install. Nothing can be lost by + // removing it, and leaving it is what accumulates scratch forever. + id: "D1b", + arrange: func(t *testing.T, txn *destTxn, stage, dest string) { + injected := errors.New("injected set-aside failure") + injectFault(t, "rename", func(args ...string) bool { + return filepath.Base(args[1]) == "install" + }, injected) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := promoteStagedDir(txn, stage, dest, "engine", nil); !errors.Is(err, injected) { + t.Fatalf("promote = %v, want the injected set-aside failure", err) + } + }, + wantLive: "old", + wantCopy: copyDeleted, + }, + { + // The previous install is aside and the destination is absent, so + // the holder has the only copy of it and it has to come back. An + // offline caller has no download to fall back on. + id: "D2", + arrange: arrangeInterruptedPromotion, + wantLive: "old", + wantCopy: copyAtDest, + }, + { + // The publish landed and the flag did not. Without the flag there is + // no evidence the copy in the holder was superseded, so it may not + // be deleted; parking it is what keeps it out of the next pass's way + // without dropping the last copy of an install. + id: "D3", + arrange: func(t *testing.T, txn *destTxn, stage, dest string) { + injectFault(t, "create", func(args ...string) bool { + return filepath.Base(args[0]) == committedFile + }, errors.New("injected commit flag failure")) + if err := promoteStagedDir(txn, stage, dest, "engine", nil); err != nil { + t.Fatalf("a published install must be reported as success: %v", err) + } + }, + wantLive: "new", + wantCopy: copyAtKept, + report1: true, + }, + { + // The publish failed and the restore put the install back, so the + // holder holds no copy of anything: owned and empty. + id: "D4", + arrange: func(t *testing.T, txn *destTxn, stage, dest string) { + injected := errors.New("injected publish failure") + injectFault(t, "rename", func(args ...string) bool { + return filepath.Base(args[0]) == filepath.Base(stage) + }, injected) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := promoteStagedDir(txn, stage, dest, "engine", nil); !errors.Is(err, injected) { + t.Fatalf("promote = %v, want the injected publish failure", err) + } + }, + wantLive: "old", + wantCopy: copyDeleted, + }, + { + // The writer's own retain branch: publish failed, restore failed, + // the copy was kept and named in the error. On disk this is D2, and + // recovery must not tell them apart, because nothing on disk does. + id: "D5", + arrange: arrangeInterruptedPromotion, + wantLive: "old", + wantCopy: copyAtDest, + }, + { + // The flag is there, so the copy in the holder is provably + // superseded by the install now live at the destination. This is the + // one and only shape that licenses deleting a copy of an install. + id: "D6", + arrange: func(t *testing.T, txn *destTxn, stage, dest string) { + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := promoteStagedDir(txn, stage, dest, "engine", nil); err != nil { + t.Fatalf("a cleanup failure must not fail the install: %v", err) + } + }, + wantLive: "new", + wantCopy: copyDeleted, + }, + } +} + +func TestDictationCrashMatrix(t *testing.T) { + for _, row := range dictationCrashRows() { + t.Run(row.id, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + stagedTree(t, dest, "old") + // One handle for the whole row: the lock is per open file + // description, so a second acquire in this process would contend + // with the first rather than nest. + txn := lockFor(t, dest) + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + // The faults belong to the crash, not to recovery: restoring the + // seam here is what makes the next lines a recovery over a real + // filesystem rather than over a failing one. + func() { + saved := holderFS + defer func() { holderFS = saved }() + row.arrange(t, txn, stage, dest) + }() + holder := soleHolder(t, dest) + + for pass, wantReport := range []bool{row.report1, row.report2} { + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, func(m string) { + reported = append(reported, m) + }) + assertDictationTerminalState(t, dest, holder, row.wantCopy, row.wantLive, pass+1) + if wantReport { + assertCopyReported(t, reported, holderSeqDigitsOf(t, dest, holder), pass+1) + } + } + }) + } +} + +// assertDictationTerminalState asserts both halves of a row: the live install, +// and the full set of copies still beside the destination under either prefix. +// The set is compared whole rather than one path at a time, so an extra copy +// nobody expected fails the row instead of going unnoticed. +func assertDictationTerminalState(t *testing.T, dest, holder string, want copyDisposition, wantLive string, pass int) { + t.Helper() + got, err := os.ReadFile(filepath.Join(dest, "engine")) + switch { + case wantLive == "": + if err == nil { + t.Errorf("pass %d: the destination should be absent, it holds %q", pass, got) + } + case err != nil || string(got) != wantLive: + t.Errorf("pass %d: dest engine = %q (err %v), want %q", pass, got, err, wantLive) + } + + var wantPaths []string + switch want { + case copyAtHolder, copyInPlaceUnowned: + wantPaths = []string{holder} + case copyAtKept: + wantPaths = []string{keptHolderName(t, dest, holder)} + } + base := filepath.Base(dest) + assertCopySet(t, filepath.Dir(dest), []string{base + holderSuffix, base + keptSuffix}, wantPaths, pass) + + switch want { + case copyInPlaceUnowned: + // Retained is not the whole claim: the reason it is retained is that + // nothing on disk attributes it, and a row that stopped checking that + // would pass over a copy recovery could have proved was its own. + if _, err := readHolderMarker(holder); !errors.Is(err, errMarkerMissing) { + t.Errorf("pass %d: %s should carry no marker, readHolderMarker = %v", pass, holder, err) + } + case copyAtDest: + if _, err := os.Stat(dest); err != nil { + t.Errorf("pass %d: the copy should have been restored to the destination: %v", pass, err) + } + } +} + +// keptHolderName is the name a parked copy takes, derived the way parkKeptHolder +// derives it: the same sequence under the Kept prefix. +func keptHolderName(t *testing.T, dest, holder string) string { + t.Helper() + name := filepath.Base(holder) + cut := strings.LastIndex(name, holderSuffix) + if cut < 0 { + t.Fatalf("holder name %q carries no holder suffix", name) + } + return filepath.Join(filepath.Dir(holder), name[:cut]+keptSuffix+name[cut+len(holderSuffix):]) +} + +// holderSeqDigitsOf is the sequence a copy carries in its name. It survives a +// park, so a report assertion keyed on it holds whether the copy is still under +// the holder prefix or has moved under the Kept one. +func holderSeqDigitsOf(t *testing.T, dest, holder string) string { + t.Helper() + seq, ok := holderStamp(dest, holder) + if !ok { + t.Fatalf("holder name %q carries no sequence", filepath.Base(holder)) + } + return fmt.Sprintf("%020d", seq) +} + +// A copy recovery keeps and never names is one an operator cannot find, so the +// report is half of every retain row rather than a nicety. +func assertCopyReported(t *testing.T, reported []string, seq string, pass int) { + t.Helper() + if !slices.ContainsFunc(reported, func(m string) bool { return strings.Contains(m, seq) }) { + t.Errorf("pass %d: the retained copy (sequence %s) should be named in the report, got %v", pass, seq, reported) + } +} + +// assertCopySet compares every directory under the given prefixes against the +// paths the row expects, resolving both sides through EvalSymlinks because the +// temp root is a symlink on macOS and a path built from t.TempDir would then +// never equal one read back out of the directory. +func assertCopySet(t *testing.T, parent string, prefixes, want []string, pass int) { + t.Helper() + entries, err := os.ReadDir(parent) + if err != nil { + t.Fatal(err) + } + var got []string + for _, entry := range entries { + if !entry.IsDir() { + continue + } + for _, prefix := range prefixes { + if strings.HasPrefix(entry.Name(), prefix) { + got = append(got, resolvedPath(t, filepath.Join(parent, entry.Name()))) + break + } + } + } + wantResolved := make([]string, 0, len(want)) + for _, path := range want { + wantResolved = append(wantResolved, resolvedPath(t, path)) + } + slices.Sort(got) + slices.Sort(wantResolved) + if !slices.Equal(got, wantResolved) { + t.Errorf("pass %d: copies on disk = %v, want %v", pass, got, wantResolved) + } +} + +// resolvedPath resolves symlinks so the two sides of a path comparison are the +// same path. A path that does not exist cannot be resolved, and its cleaned form +// is what the comparison then reports. +func resolvedPath(t *testing.T, path string) string { + t.Helper() + if resolved, err := filepath.EvalSymlinks(path); err == nil { + return resolved + } + return filepath.Clean(path) +} + +// Ownership is a name plus a marker that agrees with it, and neither half alone. +// A directory that only carries the generated name is a sibling someone renamed +// or an unrelated install, and recovery restoring from one, or reaping one, is +// the whole class of defect the marker exists to close. Each case here is a +// single check dropped: the marker, its kind, its sequence, its destination. +func TestOwnershipCannotBeForgedBySiblingNames(t *testing.T) { + const forgedSeq = 7 + for _, tc := range []struct { + name string + marker *txnMarker + }{ + {name: "no marker at all"}, + {name: "another site's kind", marker: &txnMarker{Kind: "bundle-extract", Dest: "engine-dir", Seq: forgedSeq}}, + {name: "sequence disagrees with the name", marker: &txnMarker{Kind: holderMarkerKind, Dest: "engine-dir", Seq: forgedSeq + 1}}, + {name: "destination is another install", marker: &txnMarker{Kind: holderMarkerKind, Dest: "model-dir", Seq: forgedSeq}}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + // Built outside and renamed in, so nothing about it was ever written + // by this code: the name is the only thing it shares with a holder, + // and it is shaped like a restorable one. + unrelated := filepath.Join(t.TempDir(), "unrelated") + if err := os.MkdirAll(filepath.Join(unrelated, "install"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(unrelated, "install", "engine"), []byte("forged"), 0o644); err != nil { + t.Fatal(err) + } + forged := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, forgedSeq, holderSeqSuffix) + if err := os.Rename(unrelated, forged); err != nil { + t.Fatal(err) + } + if tc.marker != nil { + if err := writeHolderMarker(forged, *tc.marker); err != nil { + t.Fatal(err) + } + } + txn := lockFor(t, dest) + + for pass := 1; pass <= 2; pass++ { + restoreInterruptedPromotion(txn, dest, testPublished, nil) + got, err := os.ReadFile(filepath.Join(forged, "install", "engine")) + if err != nil || string(got) != "forged" { + t.Fatalf("pass %d: the forged directory must be left exactly as it was, got %q (err %v)", pass, got, err) + } + // A restore, a park, or a reap all show up here: any of them + // either creates a destination or moves the directory. + assertNoOtherEntries(t, root, filepath.Base(forged), installLockDir) + } + }) + } +} + +// assertNoOtherEntries fails when anything but the named entries is in dir. It +// is how "nothing was owned" is asserted without naming every way ownership +// could have been acted on: a restore, a park, and a reap all change this set. +func assertNoOtherEntries(t *testing.T, dir string, allowed ...string) { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !slices.Contains(allowed, entry.Name()) { + t.Errorf("%s should hold only %v, found %s", dir, allowed, entry.Name()) + } + } +} From 0750b3fa454a189d4ab47776ec89ad45b5d28821 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:28:07 -0500 Subject: [PATCH 28/34] feat(cli): let an operator see and reclaim the copies recovery keeps Recovery now keeps anything it cannot prove disposable, and the mtime reap that used to reclaim disk is gone, so without this there is no way to get that space back and no way to even see what is holding it. On a shared bridge host that is a full work tree per retained copy, and on a workstation a whole speech engine. kept-backups list names every copy under the kept prefix with its destination, sequence and size, and lists what recovery could not attribute beside them, so residue is visible rather than merely present. It reads the prefix off disk, so a copy retained by a bridge built without a logger is still discoverable even though nothing reported it at the time. Removal is not a general rm. It takes one name, proves ownership the same way recovery does, refuses anything still under the scanned prefix, and takes the destination's Install lock so it cannot race a live install. Nothing is ever removed automatically: the two sites differ in what a wrong call costs, since a bundle tree can be uploaded again and a dictation copy may be the only one that exists offline, and the usage text says so. --- internal/cli/app.go | 2 + internal/cli/kept_backups.go | 216 ++++++++++++++++++++++++ internal/cli/kept_backups_test.go | 177 +++++++++++++++++++ internal/daemon/remote/bundle.go | 128 ++++++++++++++ internal/daemon/remote/bundle_test.go | 234 ++++++++++++++++++++++++++ internal/dictation/download.go | 189 ++++++++++++++++++--- internal/dictation/download_test.go | 220 ++++++++++++++++++++++++ 7 files changed, 1147 insertions(+), 19 deletions(-) create mode 100644 internal/cli/kept_backups.go create mode 100644 internal/cli/kept_backups_test.go diff --git a/internal/cli/app.go b/internal/cli/app.go index 44fa370ff..103c5bb5d 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -497,6 +497,8 @@ func runWithDeps(args []string, stdout io.Writer, stderr io.Writer, deps appDeps return runCron(args[1:], stdout, stderr, deps) case "repo-info", "repoinfo": return runRepoInfo(args[1:], stdout, stderr, deps) + case "kept-backups": + return runKeptBackups(args[1:], stdout, stderr, deps) case "serve": return runServe(args[1:], stdout, stderr, deps) case "acp": diff --git a/internal/cli/kept_backups.go b/internal/cli/kept_backups.go new file mode 100644 index 000000000..e68a3b025 --- /dev/null +++ b/internal/cli/kept_backups.go @@ -0,0 +1,216 @@ +package cli + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/Gitlawb/zero/internal/daemon/remote" + "github.com/Gitlawb/zero/internal/dictation" + "github.com/Gitlawb/zero/internal/redaction" +) + +// runKeptBackups implements `zero kept-backups`, the only way a copy recovery +// retained ever leaves the disk. Recovery moves a copy it will not restore and +// cannot prove superseded under a Kept prefix its own scan never enumerates, and +// nothing reclaims one on its own, so without this command retention is one-way. +// +// zero kept-backups list [--bundle-dir ] list what is retained +// zero kept-backups remove [--bundle-dir ] remove one by name +func runKeptBackups(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) int { + rest, bundleDir, err := splitBundleDirFlag(args) + if err != nil { + if _, werr := fmt.Fprintf(stderr, "zero kept-backups: %s\n\n", err); werr != nil { + return exitCrash + } + writeKeptBackupsUsage(stderr) + return exitUsage + } + if len(rest) == 0 { + writeKeptBackupsUsage(stderr) + return exitUsage + } + switch rest[0] { + case "list", "ls": + return keptBackupsList(rest[1:], bundleDir, stdout, stderr, deps) + case "remove", "rm": + return keptBackupsRemove(rest[1:], bundleDir, stdout, stderr, deps) + case "-h", "--help", "help": + // Explicit help is a success path, matching the other subcommands: usage + // to stdout, exit 0. Only the error paths below write it to stderr. + writeKeptBackupsUsage(stdout) + return exitSuccess + default: + if _, err := fmt.Fprintf(stderr, "zero kept-backups: unknown subcommand %q\n\n", rest[0]); err != nil { + return exitCrash + } + writeKeptBackupsUsage(stderr) + return exitUsage + } +} + +// splitBundleDirFlag pulls --bundle-dir out of the argument list wherever it +// appears, so it can sit before or after the subcommand and its name argument. +// The daemon has no config key for the bundle dir; it is a serve-remote flag, so +// the operator has to be able to name the same directory here. +func splitBundleDirFlag(args []string) (rest []string, bundleDir string, err error) { + for i := 0; i < len(args); i++ { + arg := args[i] + switch { + case arg == "--bundle-dir": + if i+1 >= len(args) { + return nil, "", errors.New("--bundle-dir needs a directory") + } + bundleDir = args[i+1] + i++ + case len(arg) > len("--bundle-dir=") && arg[:len("--bundle-dir=")] == "--bundle-dir=": + bundleDir = arg[len("--bundle-dir="):] + default: + rest = append(rest, arg) + } + } + return rest, bundleDir, nil +} + +// sttKeptRoot is where the dictation installs live: the same tree the rest of the +// TUI downloads into, derived from userConfigPath rather than the default config +// dir so an overridden config root does not leave this command reading a +// directory nothing writes to. +func sttKeptRoot(deps appDeps) (string, error) { + path, err := deps.userConfigPath() + if err != nil { + return "", err + } + if path == "" { + return "", errors.New("no user config path, so the dictation install root cannot be resolved") + } + return filepath.Join(filepath.Dir(path), "stt"), nil +} + +func keptBackupsList(args []string, bundleDir string, stdout io.Writer, stderr io.Writer, deps appDeps) int { + if len(args) > 0 { + if _, err := fmt.Fprintf(stderr, "zero kept-backups list: unexpected argument %q\n", args[0]); err != nil { + return exitCrash + } + return exitUsage + } + root, err := sttKeptRoot(deps) + if err != nil { + return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash) + } + sttBackups, err := dictation.ListKeptBackups(root) + if err != nil && !os.IsNotExist(err) { + return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash) + } + found := writeKeptBackups(stdout, "stt", sttKeptLines(sttBackups)) + if bundleDir != "" { + bundleBackups, err := remote.ListKeptBackups(bundleDir) + if err != nil && !os.IsNotExist(err) { + return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash) + } + found = writeKeptBackups(stdout, "bundle", bundleKeptLines(bundleBackups)) || found + } + if !found { + if _, err := fmt.Fprintln(stdout, "No kept backups."); err != nil { + return exitCrash + } + } + return exitSuccess +} + +// keptBackupLine is one retained copy as this command prints it. The two sites +// return their own KeptBackup types, and flattening them here is what keeps the +// output one format rather than two that drift. +type keptBackupLine struct { + name string + dest string + seq int64 + bytes int64 + owned bool +} + +func sttKeptLines(backups []dictation.KeptBackup) []keptBackupLine { + lines := make([]keptBackupLine, 0, len(backups)) + for _, b := range backups { + lines = append(lines, keptBackupLine{name: filepath.Base(b.Path), dest: b.Dest, seq: b.Seq, bytes: b.Bytes, owned: b.Owned}) + } + return lines +} + +func bundleKeptLines(backups []remote.KeptBackup) []keptBackupLine { + lines := make([]keptBackupLine, 0, len(backups)) + for _, b := range backups { + lines = append(lines, keptBackupLine{name: filepath.Base(b.Path), dest: b.Dest, seq: b.Seq, bytes: b.Bytes, owned: b.Owned}) + } + return lines +} + +// writeKeptBackups prints one line per retained copy. The name comes first after +// the site because it is exactly what `remove` takes; the destination is the +// install or link the copy was set aside for, and an entry nothing on disk +// attributes says so instead of borrowing a destination from its own name. +func writeKeptBackups(stdout io.Writer, site string, lines []keptBackupLine) bool { + for _, line := range lines { + dest := line.dest + if dest == "" { + dest = "-" + } + suffix := "" + if !line.owned { + suffix = " unowned" + } + if _, err := fmt.Fprintf(stdout, "%s %s dest=%s seq=%d bytes=%d%s\n", site, line.name, dest, line.seq, line.bytes, suffix); err != nil { + break + } + } + return len(lines) > 0 +} + +func keptBackupsRemove(args []string, bundleDir string, stdout io.Writer, stderr io.Writer, deps appDeps) int { + if len(args) != 1 { + if _, err := fmt.Fprintln(stderr, "usage: zero kept-backups remove [--bundle-dir ]"); err != nil { + return exitCrash + } + return exitUsage + } + name := args[0] + root := bundleDir + remove := func() error { return remote.RemoveKeptBackup(bundleDir, name) } + if bundleDir == "" { + sttRoot, err := sttKeptRoot(deps) + if err != nil { + return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash) + } + root = sttRoot + remove = func() error { return dictation.RemoveKeptBackup(sttRoot, name) } + } + if err := remove(); err != nil { + return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash) + } + if _, err := fmt.Fprintf(stdout, "Removed %s from %s\n", name, root); err != nil { + return exitCrash + } + return exitSuccess +} + +func writeKeptBackupsUsage(w io.Writer) { + _, _ = fmt.Fprint(w, `Usage: + zero kept-backups list [--bundle-dir ] List retained copies + zero kept-backups remove [--bundle-dir ] Remove one by name + +Recovery never deletes a copy it cannot prove was superseded; it moves that copy +under a kept- name and leaves it there. Nothing reclaims one on its own, so this +command is how retained copies leave the disk. + +Without --bundle-dir both subcommands work on the dictation install root; with +it they work on that daemon bundle dir instead. Weigh the two differently: a +dictation kept backup is the only offline copy of an engine or a model, while a +bundle kept backup is a work tree the client that sent it can upload again. + +Entries marked unowned carry the kept- name with nothing on disk attributing +them. They are reported so they can be found, and remove refuses them; check +what they hold and remove those by hand. +`) +} diff --git a/internal/cli/kept_backups_test.go b/internal/cli/kept_backups_test.go new file mode 100644 index 000000000..450beacfc --- /dev/null +++ b/internal/cli/kept_backups_test.go @@ -0,0 +1,177 @@ +package cli + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// plantKeptTxn writes the transaction marker both install sites read: the kind +// that site writes, the destination the copy was set aside for, and the sequence +// its directory name carries. Written here by hand rather than through either +// package, so the command is exercised against the on-disk format an operator +// actually has, not against a helper that could drift with it. +func plantKeptTxn(t *testing.T, dir, kind, dest string, seq int64) { + t.Helper() + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + payload, err := json.Marshal(map[string]any{"kind": kind, "dest": dest, "seq": seq}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "txn"), payload, 0o600); err != nil { + t.Fatal(err) + } +} + +// plantSTTKept writes a Kept backup of a dictation install: the engine copy an +// operator has to be able to find, because it is the only offline one. +func plantSTTKept(t *testing.T, root, dest string, seq int64, content string) string { + t.Helper() + name := fmt.Sprintf("%s.kept-%020d-seq", dest, seq) + dir := filepath.Join(root, name) + plantKeptTxn(t, dir, "dictation-promote", dest, seq) + if err := os.MkdirAll(filepath.Join(dir, "install"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "install", "engine"), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return name +} + +// plantBundleKept writes a Kept backup of a bundle extract: a tree the client +// that uploaded it can send again, which is why the two sites cost differently. +func plantBundleKept(t *testing.T, dir, linkID string, seq int64, content string) string { + t.Helper() + name := fmt.Sprintf(".kept-%020d-seq", seq) + path := filepath.Join(dir, name) + plantKeptTxn(t, path, "bundle-extract", linkID, seq) + if err := os.MkdirAll(filepath.Join(path, "backup"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "backup", "a.txt"), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return name +} + +func keptDeps(userConfigPath string) appDeps { + return appDeps{userConfigPath: func() (string, error) { return userConfigPath, nil }} +} + +// The listing is the only way a retained copy is found again: recovery never +// enumerates the Kept prefix and nothing reclaims one on its own. It has to name +// the site, because the same command reads two of them and the cost of keeping a +// copy is not the same at both. +func TestKeptBackupsListPrintsBothSites(t *testing.T) { + userConfigPath := filepath.Join(t.TempDir(), "zero", "config.json") + sttRoot := filepath.Join(filepath.Dir(userConfigPath), "stt") + bundleDir := t.TempDir() + + sttName := plantSTTKept(t, sttRoot, "engine-a", 1, "engine-bytes") + bundleName := plantBundleKept(t, bundleDir, "proj-1", 7, "tree") + // Kept grammar, nothing attributing it: recovery's residue, which the + // operator has to see beside the real backups to reclaim it by hand. + unownedName := fmt.Sprintf(".kept-%020d-seq", 8) + if err := os.MkdirAll(filepath.Join(bundleDir, unownedName), 0o700); err != nil { + t.Fatal(err) + } + + var stdout, stderr bytes.Buffer + if code := runWithDeps([]string{"kept-backups", "list", "--bundle-dir", bundleDir}, &stdout, &stderr, keptDeps(userConfigPath)); code != 0 { + t.Fatalf("exit = %d, stderr = %s", code, stderr.String()) + } + + lines := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(stdout.String()), "\n") { + fields := strings.Fields(line) + if len(fields) > 1 { + lines[fields[1]] = line + } + } + stt, ok := lines[sttName] + if !ok { + t.Fatalf("the dictation kept backup is missing from:\n%s", stdout.String()) + } + for _, want := range []string{"stt", "dest=engine-a", "seq=1", "bytes="} { + if !strings.Contains(stt, want) { + t.Errorf("dictation line %q is missing %q", stt, want) + } + } + if strings.Contains(stt, "bytes=0") { + t.Errorf("dictation line %q reports no bytes for a copy that holds some", stt) + } + bundle, ok := lines[bundleName] + if !ok { + t.Fatalf("the bundle kept backup is missing from:\n%s", stdout.String()) + } + for _, want := range []string{"bundle", "dest=proj-1", "seq=7", "bytes="} { + if !strings.Contains(bundle, want) { + t.Errorf("bundle line %q is missing %q", bundle, want) + } + } + unowned, ok := lines[unownedName] + if !ok { + t.Fatalf("the unowned entry is missing from:\n%s", stdout.String()) + } + if !strings.Contains(unowned, "unowned") { + t.Errorf("unowned line %q does not say so", unowned) + } +} + +// Which site a removal lands on is decided by --bundle-dir alone. Getting that +// wrong deletes a copy at one site while the operator watches the other, so the +// name has to resolve at the site the flag names and nowhere else. +func TestKeptBackupsRemoveNamesTheSite(t *testing.T) { + userConfigPath := filepath.Join(t.TempDir(), "zero", "config.json") + sttRoot := filepath.Join(filepath.Dir(userConfigPath), "stt") + bundleDir := t.TempDir() + sttName := plantSTTKept(t, sttRoot, "engine-a", 1, "engine-bytes") + bundleName := plantBundleKept(t, bundleDir, "proj-1", 7, "tree") + + var stdout, stderr bytes.Buffer + if code := runWithDeps([]string{"kept-backups", "remove"}, &stdout, &stderr, keptDeps(userConfigPath)); code != exitUsage { + t.Fatalf("remove with no name: exit = %d, want %d (stderr %s)", code, exitUsage, stderr.String()) + } + + stdout.Reset() + stderr.Reset() + if code := runWithDeps([]string{"kept-backups", "remove", sttName}, &stdout, &stderr, keptDeps(userConfigPath)); code != 0 { + t.Fatalf("remove at the dictation root: exit = %d, stderr = %s", code, stderr.String()) + } + if _, err := os.Stat(filepath.Join(sttRoot, sttName)); !os.IsNotExist(err) { + t.Errorf("the dictation kept backup should be gone, got %v", err) + } + if _, err := os.Stat(filepath.Join(bundleDir, bundleName)); err != nil { + t.Errorf("a removal with no --bundle-dir must not touch the bundle site: %v", err) + } + + stdout.Reset() + stderr.Reset() + if code := runWithDeps([]string{"kept-backups", "remove", bundleName, "--bundle-dir", bundleDir}, &stdout, &stderr, keptDeps(userConfigPath)); code != 0 { + t.Fatalf("remove at the bundle dir: exit = %d, stderr = %s", code, stderr.String()) + } + if _, err := os.Stat(filepath.Join(bundleDir, bundleName)); !os.IsNotExist(err) { + t.Errorf("the bundle kept backup should be gone, got %v", err) + } +} + +// The two sites cost differently and only the usage text says so, so a run that +// stops printing it leaves the operator deciding blind. +func TestKeptBackupsUsageNamesTheCostOfEachSite(t *testing.T) { + var stdout, stderr bytes.Buffer + if code := runWithDeps([]string{"kept-backups", "-h"}, &stdout, &stderr, keptDeps("")); code != exitSuccess { + t.Fatalf("exit = %d, stderr = %s", code, stderr.String()) + } + for _, want := range []string{"only offline copy", "upload"} { + if !strings.Contains(stdout.String(), want) { + t.Errorf("usage is missing %q:\n%s", want, stdout.String()) + } + } +} diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index 6a1a5aae4..8ac98c681 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -927,6 +927,134 @@ func parkKeptBackup(dir, staging, id string, logf func(string, ...any)) { } } +// keptStamp reads the sequence out of a Kept backup name. It is the same +// grammar the scanned prefix uses with the other prefix in front, and it is +// deliberately the ONLY prefix this file's operator commands accept: a directory +// under stagingPrefix may be the staging dir of an extract running right now, +// and recovery has a disposition for it either way. +func keptStamp(name string) (int64, bool) { + rest, ok := strings.CutPrefix(name, keptPrefix) + if !ok { + return 0, false + } + return stagingStamp(stagingPrefix + rest) +} + +// KeptBackup is one copy recovery moved under the Kept prefix rather than +// deleted, as an operator sees it. Recovery never enumerates that prefix, so +// this listing is the only way a retained copy is found again; nothing reclaims +// one on its own. Owned false is a directory carrying the Kept name that nothing +// on disk attributes, listed beside the real backups so recovery's residue is +// visible rather than silent until a disk fills. +type KeptBackup struct { + Path string + Dest string + Seq int64 + Bytes int64 + Owned bool +} + +// ListKeptBackups reports every Kept backup in a bundle dir. The destination +// comes off the marker and never off the name: a bundle Kept name carries no +// link id at all, so a name-derived destination would be an invention. +func ListKeptBackups(dir string) ([]KeptBackup, error) { + entries, err := stagingFS.readDir(dir) + if err != nil { + return nil, err + } + var out []KeptBackup + for _, entry := range entries { + if !entry.IsDir() { + continue + } + seq, ok := keptStamp(entry.Name()) + if !ok { + continue + } + path := filepath.Join(dir, entry.Name()) + backup := KeptBackup{Path: path, Seq: seq, Bytes: dirBytes(path)} + // The same proof recovery requires before it touches a directory: the + // work-tree veto, then a marker agreeing with the name and naming a link + // the write path would have accepted. + if id, ok := attributeStagingDir(dir, path, seq, discardLog); ok { + backup.Dest = id + backup.Owned = true + } + out = append(out, backup) + } + slices.SortFunc(out, func(a, b KeptBackup) int { return cmp.Compare(a.Path, b.Path) }) + return out, nil +} + +// RemoveKeptBackup deletes the one Kept backup an operator named. It is not an +// rm: the name has to be a single component under dir, pass the Kept grammar, +// hold no work tree at its root, and carry a marker whose kind, link, and +// sequence all agree, and the link's extract locks have to be free. Anything +// else is refused and left for the operator to remove by hand, because this is +// the only command that deletes a Kept backup and a wrong answer here is the +// last copy of a tree. +func RemoveKeptBackup(dir, name string) error { + if name == "" || name == "." || name == ".." || name != filepath.Base(name) { + // Ahead of every filesystem call: a name carrying a separator joins to a + // path outside dir, which would make this a remote rm. + return fmt.Errorf("remote: %q is not the name of an entry in %s", name, dir) + } + seq, ok := keptStamp(name) + if !ok { + return fmt.Errorf("remote: %s is not a kept backup; recovery may still act on it, so remove it by hand", name) + } + path := filepath.Join(dir, name) + id, ok := attributeStagingDir(dir, path, seq, discardLog) + if !ok { + return fmt.Errorf("remote: nothing on disk attributes %s to a link; remove it by hand", name) + } + dest := filepath.Join(dir, id) + unlock, held := tryLockExtract(dest) + if held { + return fmt.Errorf("remote: an extract in this process holds %s; try again once it finishes", id) + } + defer unlock() + unlockFile, heldFile, err := tryLockExtractFile(dir, dest) + if err != nil { + return err + } + if heldFile { + return fmt.Errorf("remote: another process holds %s; try again once it finishes", id) + } + defer unlockFile() + // Re-read under the lock. The attribution above ran with nothing excluding a + // live extract, and a directory that stopped being this link's in between is + // one that must not be deleted on the strength of the earlier reading. + if again, ok := attributeStagingDir(dir, path, seq, discardLog); !ok || again != id { + return fmt.Errorf("remote: %s is no longer attributable to %s; leaving it in place", name, id) + } + return stagingFS.removeAll(path) +} + +// dirBytes sums the regular files under path. Unreadable entries are skipped +// rather than failing the listing: a size an operator cannot see is a worse +// answer than a size that is short, and the entry itself is still reported. +func dirBytes(path string) int64 { + var total int64 + _ = filepath.WalkDir(path, func(_ string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return nil + } + total += info.Size() + return nil + }) + return total +} + +// discardLog is the reporter the operator commands hand the recovery helpers +// they reuse. Those helpers narrate every refusal for the recovery pass's log; +// here the refusal comes back as an error instead. +func discardLog(string, ...any) {} + // txnMarker is what a staging dir carries to prove which transaction created it. // Seq repeats the number in the directory's own name: a marker that disagrees // with its name proves nothing about who wrote either, so the pair is what makes diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index ee2aa9403..59b85780c 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -2926,3 +2926,237 @@ func TestRecoverBundleDirLeavesALiveExtractsScratchAlone(t *testing.T) { t.Fatalf("a.txt = %q err %v, want the published %q", got, err, "v1") } } + +// ---- the reclaim surface --------------------------------------------------- + +// plantKeptBackup writes what recovery leaves under the Kept prefix: a staged +// copy, named and marked for linkID, moved out of the prefix the scan +// enumerates. Going through stageBackup and the real park name is what keeps the +// fixture from testing a shape production never writes. +func plantKeptBackup(t *testing.T, dir, linkID, content string, seq int64) string { + t.Helper() + staging := stageBackup(t, dir, "", linkID, content, seq) + kept := filepath.Join(dir, keptPrefix+strings.TrimPrefix(filepath.Base(staging), stagingPrefix)) + if err := os.Rename(staging, kept); err != nil { + t.Fatal(err) + } + return kept +} + +// keptBytes is what the listing should report for a backup plantKeptBackup +// wrote: the marker and the one file in the copy. Summed from the two names the +// fixture created rather than by walking, so the assertion is not the +// implementation restated. +func keptBytes(t *testing.T, kept, content string) int64 { + t.Helper() + info, err := os.Stat(filepath.Join(kept, stagingMarkerFile)) + if err != nil { + t.Fatal(err) + } + return info.Size() + int64(len(content)) +} + +func findKept(t *testing.T, list []KeptBackup, path string) KeptBackup { + t.Helper() + for _, b := range list { + if b.Path == path { + return b + } + } + t.Fatalf("%s is missing from the listing %v", path, list) + return KeptBackup{} +} + +// A Kept backup an operator cannot find is one that can never be reclaimed, and +// this listing is the only thing that finds them: recovery deliberately never +// enumerates the prefix. The destination comes off the marker, because the +// bundle site's Kept name carries no link id at all, and anything that merely +// carries the name is reported unowned so recovery's own residue is visible +// beside the real backups. +func TestListKeptBackupsReportsDestSeqAndSize(t *testing.T) { + dir := t.TempDir() + first := plantKeptBackup(t, dir, "proj-1", "one", 1) + second := plantKeptBackup(t, dir, "proj-2", "second-copy", 2) + + // Kept grammar, nothing attributing it: the empty directory a crash between + // Mkdir and the marker write leaves, parked by a later pass. + bare := filepath.Join(dir, fmt.Sprintf("%s%020d%s", keptPrefix, 3, stagingSeqSuffix)) + if err := os.Mkdir(bare, 0o700); err != nil { + t.Fatal(err) + } + // A marker that contradicts the name it sits in proves nothing about who + // wrote either of them. + skewed := plantKeptBackup(t, dir, "proj-3", "skew", 4) + if err := writeMarker(skewed, txnMarker{Kind: txnKindBundleExtract, Dest: "proj-3", Seq: 99}); err != nil { + t.Fatal(err) + } + // A sibling that fails the grammar is not a Kept backup and is not listed. + if err := os.Mkdir(filepath.Join(dir, keptPrefix+"notanumber-seq"), 0o700); err != nil { + t.Fatal(err) + } + + list, err := ListKeptBackups(dir) + if err != nil { + t.Fatalf("ListKeptBackups: %v", err) + } + if len(list) != 4 { + t.Fatalf("listed %d kept backups, want 4: %v", len(list), list) + } + + got := findKept(t, list, first) + if !got.Owned || got.Dest != "proj-1" || got.Seq != 1 || got.Bytes != keptBytes(t, first, "one") { + t.Errorf("first = %+v, want owned proj-1 seq 1 bytes %d", got, keptBytes(t, first, "one")) + } + got = findKept(t, list, second) + if !got.Owned || got.Dest != "proj-2" || got.Seq != 2 || got.Bytes != keptBytes(t, second, "second-copy") { + t.Errorf("second = %+v, want owned proj-2 seq 2 bytes %d", got, keptBytes(t, second, "second-copy")) + } + for _, path := range []string{bare, skewed} { + got := findKept(t, list, path) + if got.Owned { + t.Errorf("%s is attributed by nothing on disk and must be listed unowned, got %+v", path, got) + } + // The destination has to be empty too: naming one the marker does not + // support tells the operator this copy belongs to a link it may not. + if got.Dest != "" { + t.Errorf("%s: unowned entries carry no destination, got %q", path, got.Dest) + } + } +} + +// Removal carries the same ownership proof recovery's own deletes carry. It is +// the one thing on this machine that deletes a Kept backup, so an entry nothing +// attributes has to survive it: the operator is told to remove that one by hand. +func TestRemoveKeptBackupRefusesAnUnownedEntry(t *testing.T) { + t.Run("no marker", func(t *testing.T) { + dir := t.TempDir() + kept := plantKeptBackup(t, dir, "proj-1", "one", 1) + if err := os.Remove(filepath.Join(kept, stagingMarkerFile)); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err == nil { + t.Fatal("a directory with no marker must not be removed") + } + if _, err := os.Stat(filepath.Join(kept, "backup", "a.txt")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) + t.Run("marker disagrees with the name", func(t *testing.T) { + dir := t.TempDir() + kept := plantKeptBackup(t, dir, "proj-1", "one", 1) + if err := writeMarker(kept, txnMarker{Kind: txnKindBundleExtract, Dest: "proj-1", Seq: 42}); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err == nil { + t.Fatal("a marker for another sequence must not license a removal") + } + if _, err := os.Stat(filepath.Join(kept, "backup", "a.txt")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) + // Link ids beginning with '.' used to be accepted, so a published work tree + // can sit under the exact Kept name and carry a file called txn at its root. + // The work-tree veto has to run ahead of the marker read, or that file + // licenses removing a user's checkout. + t.Run("work tree beside a valid-looking marker", func(t *testing.T) { + dir := t.TempDir() + kept := plantKeptBackup(t, dir, "proj-1", "one", 1) + if err := os.MkdirAll(filepath.Join(kept, ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err == nil { + t.Fatal("a work tree must not be removed whatever its marker says") + } + if _, err := os.Stat(filepath.Join(kept, ".git")); err != nil { + t.Errorf("the work tree must be left intact: %v", err) + } + }) +} + +// The lock is what separates a Kept backup from one a live extract is about to +// restore from: on disk the two are the same directory, and removing one +// mid-transaction takes the copy the transaction rolls back to. +func TestRemoveKeptBackupRefusesWhileTheDestinationIsLocked(t *testing.T) { + t.Run("in process", func(t *testing.T) { + dir := t.TempDir() + kept := plantKeptBackup(t, dir, "proj-1", "one", 1) + unlock := lockExtract(filepath.Join(dir, "proj-1")) + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err == nil { + unlock() + t.Fatal("a removal must not run while an extract in this process holds the link") + } + if _, err := os.Stat(kept); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + unlock() + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err != nil { + t.Fatalf("the same removal must go through once the lock is free: %v", err) + } + if _, err := os.Stat(kept); !os.IsNotExist(err) { + t.Errorf("the copy should be gone, got %v", err) + } + }) + t.Run("another process", func(t *testing.T) { + dir := t.TempDir() + kept := plantKeptBackup(t, dir, "proj-1", "one", 1) + lockDir := filepath.Join(dir, lockDirName) + if err := os.MkdirAll(lockDir, 0o700); err != nil { + t.Fatal(err) + } + held, err := lockutil.TryAcquireFileLockAt(dir, filepath.Join(lockDir, "proj-1.lock")) + if err != nil { + t.Fatalf("take the lock as the other daemon would: %v", err) + } + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err == nil { + _ = held.Release() + t.Fatal("a removal must not run while another process holds the link") + } + if _, err := os.Stat(kept); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + if err := held.Release(); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err != nil { + t.Fatalf("the same removal must go through once the lock is free: %v", err) + } + }) +} + +// The command takes a name, never a path. A separator in the argument would join +// to a directory outside the bundle dir, which turns an operator's reclaim into +// a remote rm, so the name is refused before any filesystem call happens at all. +func TestRemoveKeptBackupRejectsANameThatIsNotABaseName(t *testing.T) { + dir := t.TempDir() + injectFault(t, "removeAll", func(args ...string) bool { + t.Errorf("a rejected name must not reach the filesystem, got removeAll(%v)", args) + return false + }, nil) + injectFault(t, "stat", func(args ...string) bool { + t.Errorf("a rejected name must not reach the filesystem, got stat(%v)", args) + return false + }, nil) + injectFault(t, "readFile", func(args ...string) bool { + t.Errorf("a rejected name must not reach the filesystem, got readFile(%v)", args) + return false + }, nil) + for _, name := range []string{"../x", "a/b", filepath.Join(dir, ".kept-x"), ".", "..", ""} { + if err := RemoveKeptBackup(dir, name); err == nil { + t.Errorf("RemoveKeptBackup(%q) = nil, want a refusal", name) + } + } +} + +// The scanned prefix is recovery's, not the operator's: a directory under it may +// be the staging dir of an extract running right now, and the next pass has a +// disposition for it either way. Only the Kept prefix is this command's to touch. +func TestRemoveKeptBackupNeverTouchesTheScannedPrefix(t *testing.T) { + dir := t.TempDir() + staging := stageBackup(t, dir, "", "proj-1", "one", 1) + if err := RemoveKeptBackup(dir, filepath.Base(staging)); err == nil { + t.Fatal("a directory under the scanned prefix must not be removed by name") + } + if _, err := os.Stat(filepath.Join(staging, "backup", "a.txt")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } +} diff --git a/internal/dictation/download.go b/internal/dictation/download.go index f2c5966b7..01ded877a 100644 --- a/internal/dictation/download.go +++ b/internal/dictation/download.go @@ -899,29 +899,15 @@ func isInstallDestName(name string) bool { // out. The budget expiring is errInstallInProgress, which the caller answers by // re-checking the destination rather than by failing. func lockDestination(ctx context.Context, destRoot, dest string) (*destTxn, error) { - if !isInstallDestName(dest) { - return nil, fmt.Errorf("dictation download: %q is not an install destination name", dest) - } - // destRoot is otherwise created lazily by the download this lock covers, and - // lockutil opens the root with O_DIRECTORY, so on a first run there would be - // no directory to take the lock in. - if err := os.MkdirAll(destRoot, 0o755); err != nil { - return nil, err - } - lockDir := filepath.Join(destRoot, installLockDir) - if err := holderFS.mkdir(lockDir, 0o700); err != nil && !errors.Is(err, fs.ErrExist) { - return nil, err - } - lockPath := filepath.Join(lockDir, dest+".lock") deadline := time.Now().Add(installLockWait) for { - lock, err := lockutil.TryAcquireFileLockAt(destRoot, lockPath) - if err == nil { - return &destTxn{root: destRoot, dest: dest, lock: lock}, nil - } - if !errors.Is(err, lockutil.ErrLockHeld) { + txn, held, err := tryLockDestination(destRoot, dest) + if err != nil { return nil, err } + if !held { + return txn, nil + } // Waiting is the normal case: the holder is almost always another // process installing this very component, and failing on the first // contended try would turn that into a user-visible install failure. @@ -936,6 +922,36 @@ func lockDestination(ctx context.Context, destRoot, dest string) (*destTxn, erro } } +// tryLockDestination takes one destination's Install lock without waiting. Held +// is not an error: a caller either waits the holder out, as lockDestination +// does, or leaves that destination alone, as the operator's reclaim does. An +// operator command cannot use the waiting form, because the wait budget covers a +// download and stalling a terminal for two minutes to report a refusal is not a +// refusal anyone reads. +func tryLockDestination(destRoot, dest string) (txn *destTxn, held bool, err error) { + if !isInstallDestName(dest) { + return nil, false, fmt.Errorf("dictation download: %q is not an install destination name", dest) + } + // destRoot is otherwise created lazily by the download this lock covers, and + // lockutil opens the root with O_DIRECTORY, so on a first run there would be + // no directory to take the lock in. + if err := os.MkdirAll(destRoot, 0o755); err != nil { + return nil, false, err + } + lockDir := filepath.Join(destRoot, installLockDir) + if err := holderFS.mkdir(lockDir, 0o700); err != nil && !errors.Is(err, fs.ErrExist) { + return nil, false, err + } + lock, err := lockutil.TryAcquireFileLockAt(destRoot, filepath.Join(lockDir, dest+".lock")) + if err != nil { + if errors.Is(err, lockutil.ErrLockHeld) { + return nil, true, nil + } + return nil, false, err + } + return &destTxn{root: destRoot, dest: dest, lock: lock}, false, nil +} + // withDestinationLock runs fn under dest's Install lock. A wait that runs out is // not by itself a failure: the likeliest reason is that the other process // finished this exact install, so usable re-checks the destination before the @@ -1196,6 +1212,141 @@ func keptHolderPath(holder string) (string, bool) { return filepath.Join(dir, name[:cut]+keptSuffix+name[cut+len(holderSuffix):]), true } +// keptHolderStamp splits a Kept backup name into the install it was set aside +// for and its sequence. The install's own name is the leading part, so the +// grammar alone does NOT bound the name to one path component: "../x.kept-N-seq" +// parses as destination "../x". Callers check both halves are base names before +// joining either of them to anything. +func keptHolderStamp(name string) (base string, seq int64, ok bool) { + cut := strings.LastIndex(name, keptSuffix) + if cut <= 0 { + return "", 0, false + } + base = name[:cut] + seq, ok = holderStamp(base, name) + if !ok { + return "", 0, false + } + return base, seq, true +} + +// KeptBackup is one copy recovery moved under the Kept prefix rather than +// deleted, as an operator sees it. Recovery never enumerates that prefix and +// nothing reclaims one on its own, so this listing is the only way a retained +// copy is found again. Owned false is a directory carrying the Kept name that +// nothing on disk attributes, listed beside the real backups so recovery's +// residue is visible rather than silent until a disk fills. +type KeptBackup struct { + Path string + Dest string + Seq int64 + Bytes int64 + Owned bool +} + +// ListKeptBackups reports every Kept backup under an install root. The +// destination is reported only when the marker inside the copy agrees with the +// name it sits in: a name is what anything that can write in this directory +// says, and a copy attributed on its name alone is one an operator would remove +// believing it belonged to an install it never came from. +func ListKeptBackups(destRoot string) ([]KeptBackup, error) { + entries, err := holderFS.readDir(destRoot) + if err != nil { + return nil, err + } + var out []KeptBackup + for _, entry := range entries { + if !entry.IsDir() { + continue + } + base, seq, ok := keptHolderStamp(entry.Name()) + if !ok { + continue + } + path := filepath.Join(destRoot, entry.Name()) + backup := KeptBackup{Path: path, Seq: seq, Bytes: dirBytes(path)} + if ownedKeptHolder(path, base, seq) == nil { + backup.Dest = base + backup.Owned = true + } + out = append(out, backup) + } + slices.SortFunc(out, func(a, b KeptBackup) int { return cmp.Compare(a.Path, b.Path) }) + return out, nil +} + +// ownedKeptHolder is the whole ownership proof for a Kept backup: a marker this +// site wrote, for this destination, carrying the sequence its name carries. It +// is the same agreement ownedHoldersBeside requires before recovery touches a +// holder, applied here so an operator's delete is licensed by no less. +func ownedKeptHolder(path, base string, seq int64) error { + m, err := readHolderMarker(path) + if err != nil { + return err + } + if m.Kind != holderMarkerKind || m.Seq != seq || m.Dest != base || !isInstallDestName(m.Dest) { + return fmt.Errorf("the transaction marker in %s does not match its name", path) + } + return nil +} + +// RemoveKeptBackup deletes the one Kept backup an operator named. It is not an +// rm: the name has to be a single component under destRoot, pass the Kept +// grammar, and carry a marker whose kind, destination, and sequence all agree, +// and the destination's Install lock has to be free. Anything else is refused +// and left for the operator to remove by hand, because a Kept backup here is +// often the only offline copy of an engine or a model. +func RemoveKeptBackup(destRoot, name string) error { + if !isInstallDestName(name) { + // Ahead of every filesystem call: the Kept grammar accepts a leading + // install name with a separator in it, so without this the join reaches + // outside destRoot. + return fmt.Errorf("dictation download: %q is not the name of an entry in %s", name, destRoot) + } + base, seq, ok := keptHolderStamp(name) + if !ok || !isInstallDestName(base) { + return fmt.Errorf("dictation download: %s is not a kept backup; recovery may still act on it, so remove it by hand", name) + } + path := filepath.Join(destRoot, name) + if err := ownedKeptHolder(path, base, seq); err != nil { + return fmt.Errorf("dictation download: nothing on disk attributes %s to an install (%w); remove it by hand", name, err) + } + txn, held, err := tryLockDestination(destRoot, base) + if err != nil { + return err + } + if held { + return fmt.Errorf("dictation download: %s is being installed by another process; try again once it finishes", base) + } + defer txn.release() + // Re-read under the lock. The check above ran with nothing excluding a live + // promotion, and a directory that stopped being this install's in between is + // one that must not be deleted on the strength of the earlier reading. + if err := ownedKeptHolder(path, base, seq); err != nil { + return fmt.Errorf("dictation download: %s is no longer attributable to %s (%w); leaving it in place", name, base, err) + } + return holderFS.removeAll(path) +} + +// dirBytes sums the regular files under path. Unreadable entries are skipped +// rather than failing the listing: a size an operator cannot see is a worse +// answer than a size that is short, and the entry itself is still reported. +func dirBytes(path string) int64 { + var total int64 + _ = filepath.WalkDir(path, func(_ string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return nil + } + total += info.Size() + return nil + }) + return total +} + // nextHolderSeq is the number a new holder should claim: one past the highest // already set aside for this install, counting BOTH the scanned prefix and the // Kept one. Recovery parks a holder by renaming it under .kept-, and a sequence diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index f99b8901d..39c73790d 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -2781,3 +2781,223 @@ func recordRemoveAll(t *testing.T) *[]string { } return &paths } + +// ---- the reclaim surface --------------------------------------------------- + +// plantKeptHolder writes what recovery leaves under the Kept prefix: a holder, +// named and marked the way promoteStagedDir writes one, renamed to the Kept name +// recovery parks it under. Going through the real helpers is what keeps the +// fixture from testing a shape production never writes. +func plantKeptHolder(t *testing.T, destDir string, seq int64, content string) string { + t.Helper() + holder := plantHolder(t, destDir, seq, content, false) + kept := keptName(t, destDir, holder) + if err := os.Rename(holder, kept); err != nil { + t.Fatal(err) + } + return kept +} + +// keptHolderBytes is what the listing should report for a backup +// plantKeptHolder wrote: the marker and the one file in the copy. Summed from +// the two names the fixture created rather than by walking, so the assertion is +// not the implementation restated. +func keptHolderBytes(t *testing.T, kept, content string) int64 { + t.Helper() + info, err := os.Stat(filepath.Join(kept, holderMarkerFile)) + if err != nil { + t.Fatal(err) + } + return info.Size() + int64(len(content)) +} + +func findKept(t *testing.T, list []KeptBackup, path string) KeptBackup { + t.Helper() + for _, b := range list { + if b.Path == path { + return b + } + } + t.Fatalf("%s is missing from the listing %v", path, list) + return KeptBackup{} +} + +// A Kept backup an operator cannot find is one that can never be reclaimed, and +// this listing is the only thing that finds them: recovery deliberately never +// enumerates the prefix. Here the copy is the only offline copy of an engine or +// a model, so a directory nothing attributes is reported unowned rather than +// dropped: the operator has to be able to see recovery's residue. +func TestListKeptBackupsReportsDestSeqAndSize(t *testing.T) { + root := t.TempDir() + engine := filepath.Join(root, "engine-a") + model := filepath.Join(root, "model-b") + first := plantKeptHolder(t, engine, 1, "engine-bytes") + second := plantKeptHolder(t, model, 2, "model") + + // Kept grammar, nothing attributing it: the shape a crash between the mkdir + // and the marker write leaves, parked by a later pass. + bare := fmt.Sprintf("%s%s%020d%s", engine, keptSuffix, 3, holderSeqSuffix) + if err := os.Mkdir(bare, 0o755); err != nil { + t.Fatal(err) + } + // A marker naming another destination is evidence this copy belongs to a + // transaction that is not the one the name claims. + skewed := plantKeptHolder(t, engine, 4, "skew") + if err := writeHolderMarker(skewed, txnMarker{Kind: holderMarkerKind, Dest: "engine-z", Seq: 4}); err != nil { + t.Fatal(err) + } + // A sibling that fails the grammar was never a holder and is not listed. + if err := os.Mkdir(engine+keptSuffix+"notanumber"+holderSeqSuffix, 0o755); err != nil { + t.Fatal(err) + } + + list, err := ListKeptBackups(root) + if err != nil { + t.Fatalf("ListKeptBackups: %v", err) + } + if len(list) != 4 { + t.Fatalf("listed %d kept backups, want 4: %v", len(list), list) + } + + got := findKept(t, list, first) + if !got.Owned || got.Dest != "engine-a" || got.Seq != 1 || got.Bytes != keptHolderBytes(t, first, "engine-bytes") { + t.Errorf("first = %+v, want owned engine-a seq 1 bytes %d", got, keptHolderBytes(t, first, "engine-bytes")) + } + got = findKept(t, list, second) + if !got.Owned || got.Dest != "model-b" || got.Seq != 2 || got.Bytes != keptHolderBytes(t, second, "model") { + t.Errorf("second = %+v, want owned model-b seq 2 bytes %d", got, keptHolderBytes(t, second, "model")) + } + for _, path := range []string{bare, skewed} { + got := findKept(t, list, path) + if got.Owned { + t.Errorf("%s is attributed by nothing on disk and must be listed unowned, got %+v", path, got) + } + // The destination has to be empty too: naming one off the directory's + // own name tells the operator this copy belongs to an install the + // marker does not support. + if got.Dest != "" { + t.Errorf("%s: unowned entries carry no destination, got %q", path, got.Dest) + } + } +} + +// Removal carries the same ownership proof recovery's own deletes carry. This is +// the one thing that deletes a Kept backup here, and the copy it deletes may be +// the only offline copy of an engine, so an entry nothing attributes survives it +// and the operator is told to remove that one by hand. +func TestRemoveKeptBackupRefusesAnUnownedEntry(t *testing.T) { + t.Run("no marker", func(t *testing.T) { + root := t.TempDir() + kept := plantKeptHolder(t, filepath.Join(root, "engine-a"), 1, "engine") + if err := os.Remove(filepath.Join(kept, holderMarkerFile)); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(root, filepath.Base(kept)); err == nil { + t.Fatal("a directory with no marker must not be removed") + } + if _, err := os.Stat(filepath.Join(kept, "install", "engine")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) + t.Run("marker disagrees with the sequence", func(t *testing.T) { + root := t.TempDir() + kept := plantKeptHolder(t, filepath.Join(root, "engine-a"), 1, "engine") + if err := writeHolderMarker(kept, txnMarker{Kind: holderMarkerKind, Dest: "engine-a", Seq: 42}); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(root, filepath.Base(kept)); err == nil { + t.Fatal("a marker for another sequence must not license a removal") + } + if _, err := os.Stat(filepath.Join(kept, "install", "engine")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) + t.Run("marker disagrees with the destination", func(t *testing.T) { + root := t.TempDir() + kept := plantKeptHolder(t, filepath.Join(root, "engine-a"), 1, "engine") + if err := writeHolderMarker(kept, txnMarker{Kind: holderMarkerKind, Dest: "engine-z", Seq: 1}); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(root, filepath.Base(kept)); err == nil { + t.Fatal("a marker for another destination must not license a removal") + } + if _, err := os.Stat(filepath.Join(kept, "install", "engine")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) +} + +// The lock is what separates a Kept backup from a copy a live install is about +// to roll back to: on disk the two are the same directory, and removing one +// mid-promotion takes the only copy of the install. +func TestRemoveKeptBackupRefusesWhileTheDestinationIsLocked(t *testing.T) { + root := t.TempDir() + kept := plantKeptHolder(t, filepath.Join(root, "engine-a"), 1, "engine") + lockDir := filepath.Join(root, installLockDir) + if err := os.MkdirAll(lockDir, 0o700); err != nil { + t.Fatal(err) + } + held, err := lockutil.TryAcquireFileLockAt(root, filepath.Join(lockDir, "engine-a.lock")) + if err != nil { + t.Fatalf("take the lock as the other install would: %v", err) + } + if err := RemoveKeptBackup(root, filepath.Base(kept)); err == nil { + _ = held.Release() + t.Fatal("a removal must not run while another process holds the destination") + } + if _, err := os.Stat(filepath.Join(kept, "install", "engine")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + if err := held.Release(); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(root, filepath.Base(kept)); err != nil { + t.Fatalf("the same removal must go through once the lock is free: %v", err) + } + if _, err := os.Stat(kept); !os.IsNotExist(err) { + t.Errorf("the copy should be gone, got %v", err) + } +} + +// The command takes a name, never a path. The Kept grammar here is a +// destination's own name followed by the suffix, so a name carrying a separator +// still passes it: "../x.kept-...-seq" parses as destination "../x". Only the +// base-name check keeps that from joining to a directory outside the root, and +// it has to run before any filesystem call. +func TestRemoveKeptBackupRejectsANameThatIsNotABaseName(t *testing.T) { + root := t.TempDir() + suffix := fmt.Sprintf("%s%020d%s", keptSuffix, 1, holderSeqSuffix) + for _, step := range []string{"removeAll", "stat", "readFile", "mkdir"} { + injectFault(t, step, func(args ...string) bool { + t.Errorf("a rejected name must not reach the filesystem, got %s(%v)", step, args) + return false + }, nil) + } + names := []string{ + "../x" + suffix, + "a/b" + suffix, + filepath.Join(root, "engine-a") + suffix, + "." + suffix, + ".." + suffix, + "", ".", "..", + } + for _, name := range names { + if err := RemoveKeptBackup(root, name); err == nil { + t.Errorf("RemoveKeptBackup(%q) = nil, want a refusal", name) + } + } +} + +// The scanned prefix is recovery's, not the operator's: a holder under it may +// belong to a promotion running right now, and recovery has a disposition for it +// either way. Only the Kept prefix is this command's to touch. +func TestRemoveKeptBackupNeverTouchesTheScannedPrefix(t *testing.T) { + root := t.TempDir() + holder := plantHolder(t, filepath.Join(root, "engine-a"), 1, "engine", false) + if err := RemoveKeptBackup(root, filepath.Base(holder)); err == nil { + t.Fatal("a holder under the scanned prefix must not be removed by name") + } + if _, err := os.Stat(filepath.Join(holder, "install", "engine")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } +} From 053857e73344980b9ec574fe48fb525833c82edc Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:54:50 -0500 Subject: [PATCH 29/34] test(daemon/remote,dictation): fail every recovery step, on either pass or both The crash table proved recovery reaches the right terminal state from each crash. It did not prove recovery stays safe when its own steps fail, which is the case a reviewer cannot check by reading and the one where a retain quietly becomes a delete. Each recovery step now fails on pass one, on pass two, and on both, and every case asserts the same invariant at the seam rather than off the disk: no recursive delete is ever requested for a copy no commit flag supersedes. Asserting the request rather than the survivor matters, because a delete that fails leaves the same directory behind as a delete that was never attempted. Each case then runs a clean pass and reaches the terminal state, so a fault delays recovery rather than corrupting it, and asserts the faulted pass did not finish, so a matcher that never fires cannot make a row pass in silence. The racing rows park a live transaction at the set-aside, the publish and the reap while recovery runs beside it. They assert that neither side destroys the other's copy, never that recovery wins, since a test that demands one interleaving is a flake with a schedule. --- internal/daemon/remote/bundle_matrix_test.go | 814 +++++++++++++++++ internal/dictation/download_matrix_test.go | 888 +++++++++++++++++++ 2 files changed, 1702 insertions(+) diff --git a/internal/daemon/remote/bundle_matrix_test.go b/internal/daemon/remote/bundle_matrix_test.go index 947f00bc2..b892fe6c2 100644 --- a/internal/daemon/remote/bundle_matrix_test.go +++ b/internal/daemon/remote/bundle_matrix_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "slices" "strings" + "sync" "testing" ) @@ -393,3 +394,816 @@ func assertNoOtherEntries(t *testing.T, dir string, allowed ...string) { } } } + +// ---- the regression net: recovery's own steps, and a live extract ---------- + +// deleteWatch records every removeAll recovery asks for, and flags the ones no +// classification licenses: a directory that still holds a set-aside tree with no +// commit flag beside it. It is asserted AT THE SEAM rather than off the disk +// because a fault-injected removeAll leaves the tree exactly where a removeAll +// that was never requested does, so a row reading the surviving files could not +// tell a recovery that did the right thing from one that tried to do the wrong +// thing and was stopped by the injected fault. +type deleteWatch struct { + mu sync.Mutex + all []string + denied []string +} + +// install swaps removeAll for a recorder, restoring the seam in t.Cleanup. The +// order matters: only a watch installed AFTER a fault sees the calls that fault +// is failing, and cleanups unwind in reverse, so the seam still ends the test as +// the real filesystem. +func (w *deleteWatch) install(t *testing.T) { + t.Helper() + real := stagingFS + t.Cleanup(func() { stagingFS = real }) + stagingFS.removeAll = func(path string) error { + w.mu.Lock() + w.all = append(w.all, path) + if holdsUncommittedCopy(path) { + w.denied = append(w.denied, path) + } + w.mu.Unlock() + return real.removeAll(path) + } +} + +func (w *deleteWatch) counts() (all, denied []string) { + w.mu.Lock() + defer w.mu.Unlock() + return slices.Clone(w.all), slices.Clone(w.denied) +} + +// holdsUncommittedCopy reports whether path still holds a copy of a tree that +// nothing proves was published over. Recovery may park one of these forever; it +// may never delete one, whatever step just failed. +func holdsUncommittedCopy(path string) bool { + if _, err := os.Stat(filepath.Join(path, "backup")); err != nil { + return false + } + _, err := os.Stat(filepath.Join(path, committedFile)) + return os.IsNotExist(err) +} + +// The watch is the whole regression net below, so it gets its own proof: a net +// that cannot see the delete it exists to catch would make every row under it +// pass for the wrong reason. The three shapes are the two deletes recovery is +// licensed to make and the one it is not. +func TestDeleteWatchSeesOnlyTheDeleteRecoveryMayNotMake(t *testing.T) { + dir := t.TempDir() + uncommitted := stageBackup(t, dir, "", "proj-1", "v1", 100) + committed := stageBackup(t, dir, "", "proj-1", "v2", 200) + markCommitted(t, committed) + scratch := stageScratch(t, dir, "proj-1", 50) + + var watch deleteWatch + func() { + saved := stagingFS + defer func() { stagingFS = saved }() + watch.install(t) + for _, path := range []string{committed, scratch, uncommitted} { + if err := stagingFS.removeAll(path); err != nil { + t.Fatal(err) + } + } + }() + + all, denied := watch.counts() + if len(all) != 3 { + t.Errorf("the watch should have seen all three deletes, got %v", all) + } + if !slices.Equal(denied, []string{uncommitted}) { + t.Errorf("only the uncommitted copy is a violation, got %v", denied) + } +} + +// recoveryStep is one filesystem step of a recovery pass, with the fault that +// fails it. Each is matched where it is reached rather than by call ordinal, so +// a change in how many times recovery stats a directory cannot silently move +// which call the row is failing. +type recoveryStep struct { + name string + inject func(t *testing.T, newest string) +} + +func bundleRecoverySteps() []recoveryStep { + failed := errors.New("injected recovery failure") + return []recoveryStep{ + { + name: "readDir", + inject: func(t *testing.T, _ string) { injectFault(t, "readDir", nil, failed) }, + }, + { + name: "stat", + inject: func(t *testing.T, _ string) { injectFault(t, "stat", nil, failed) }, + }, + { + name: "marker read", + inject: func(t *testing.T, _ string) { injectFault(t, "readFile", nil, failed) }, + }, + { + name: "restore rename", + inject: func(t *testing.T, newest string) { + injectFault(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(newest, "backup") + }, failed) + }, + }, + { + name: "removeAll", + inject: func(t *testing.T, _ string) { injectFault(t, "removeAll", nil, failed) }, + }, + { + name: "park rename", + inject: func(t *testing.T, _ string) { + injectFault(t, "rename", func(args ...string) bool { + return strings.HasPrefix(filepath.Base(args[1]), keptPrefix) + }, failed) + }, + }, + } +} + +// Every recovery step, failed on pass one, on pass two, and on both. Two claims +// per case, and the first is the one the whole branch exists for: no failure of +// any step ever produces a delete of a copy nothing proves was superseded. The +// second is that a fault only DELAYS recovery: a later pass with the fault gone +// reaches the same terminal state a clean run reaches, so a step that failed +// once has not left the destination in a state recovery can no longer reason +// about. +func TestBundleRecoveryStepFailures(t *testing.T) { + for _, step := range bundleRecoverySteps() { + for _, when := range []struct { + name string + faulty [2]bool + }{ + {name: "pass one", faulty: [2]bool{true, false}}, + {name: "pass two", faulty: [2]bool{false, true}}, + {name: "both passes", faulty: [2]bool{true, true}}, + } { + t.Run(step.name+"/"+when.name, func(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + // Two uncommitted usable copies and one owned scratch dir, with + // the destination gone: the state that drives every step in the + // list through a real decision. One copy is restored, one is + // parked, and the scratch is the only delete on the happy path, + // so a step that fails has something to corrupt. + older := stageBackup(t, dir, "", "proj-1", "v1", 100) + newest := stageBackup(t, dir, "", "proj-1", "v2", 200) + scratch := stageScratch(t, dir, "proj-1", 50) + + var watch deleteWatch + pass := func(faulty bool) { + saved := stagingFS + defer func() { stagingFS = saved }() + if faulty { + step.inject(t, newest) + } + // After the fault, so the watch sees the calls the fault + // is failing rather than only the ones it lets through. + watch.install(t) + recoverBundleDir(dir, discardLog) + } + pass(when.faulty[0]) + if when.faulty[0] && bundleStepTerminal(dir, dest, older, newest, scratch) { + // The row would pass without the code being tested doing + // anything: a fault that changes nothing is a fault that was + // never reached, and every assertion below it is vacuous. + t.Errorf("the injected %s failure left the pass free to finish, so this row proves nothing", step.name) + } + pass(when.faulty[1]) + // The fault is gone, and recovery has to be able to finish from + // wherever the failed passes left the directory. + pass(false) + + all, denied := watch.counts() + if len(denied) > 0 { + t.Errorf("no step failure may produce a delete of an uncommitted copy, got %v", denied) + } + if len(all) == 0 { + t.Fatal("the watch saw no delete at all, so it proves nothing about this pass") + } + assertBundleStepTerminalState(t, dir, dest, older, newest, scratch) + }) + } + } +} + +// bundleStepTerminal reports whether the fixture reached the state a clean run +// reaches. It is a predicate rather than an assertion because it is asked twice +// for opposite reasons: a faulted pass must NOT have got here, and the pass +// after the fault is gone must have. +func bundleStepTerminal(dir, dest, older, newest, scratch string) bool { + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v2" { + return false + } + if _, err := os.Stat(filepath.Join(parkedStaging(older), "backup", "a.txt")); err != nil { + return false + } + for _, gone := range []string{newest, scratch} { + if _, err := os.Stat(gone); !os.IsNotExist(err) { + return false + } + } + return true +} + +// assertBundleStepTerminalState is where the fixture above has to end up once +// nothing is failing: the newest copy live, its own directory gone, the older +// one kept under the Kept prefix with its tree intact, and the scratch dir +// reaped. Content is read back rather than names checked, because a park that +// moved a name and lost the tree passes every name assertion. +func assertBundleStepTerminalState(t *testing.T, dir, dest, older, newest, scratch string) { + t.Helper() + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v2" { + t.Errorf("dest a.txt = %q (err %v), want the newest copy %q", got, err, "v2") + } + parked := parkedStaging(older) + kept, err := os.ReadFile(filepath.Join(parked, "backup", "a.txt")) + if err != nil || string(kept) != "v1" { + t.Errorf("the parked copy's a.txt = %q (err %v), want %q", kept, err, "v1") + } + for _, gone := range []string{newest, scratch} { + if _, err := os.Stat(gone); !os.IsNotExist(err) { + t.Errorf("%s should be gone once recovery finished: %v", gone, err) + } + } + assertCopySet(t, dir, []string{stagingPrefix, keptPrefix}, []string{parked}, 3) +} + +// gateAt parks the write path at one named boundary: it signals when the call +// whose arguments match is reached, holds that call until release, and passes +// every other call through. blockStep gates EVERY call to a step, which for a +// rename would park an extract at the rename inside its marker write rather than +// at the set-aside or the publish, so a matched gate is what puts the writer at +// the boundary a row is actually about. +func gateAt(t *testing.T, match func(from, to string) bool) (reached <-chan struct{}, release func()) { + t.Helper() + real := stagingFS + t.Cleanup(func() { stagingFS = real }) + arrived := make(chan struct{}) + gate := make(chan struct{}) + var arrive, open sync.Once + release = func() { open.Do(func() { close(gate) }) } + t.Cleanup(release) + stagingFS.rename = func(from, to string) error { + if match(from, to) { + arrive.Do(func() { close(arrived) }) + <-gate + } + return real.rename(from, to) + } + return arrived, release +} + +// signalRemoveAll reports when a removeAll of path is reached. It wraps whatever +// is already installed, so layering it over blockStep signals BEFORE the call +// parks rather than after it is released. +func signalRemoveAll(t *testing.T, match func(path string) bool) <-chan struct{} { + t.Helper() + installed := stagingFS + t.Cleanup(func() { stagingFS = installed }) + arrived := make(chan struct{}) + var arrive sync.Once + stagingFS.removeAll = func(p string) error { + if match(p) { + arrive.Do(func() { close(arrived) }) + } + return installed.removeAll(p) + } + return arrived +} + +// A recovery pass and a live extract for one destination, running at the same +// time, at each of the three boundaries where the destination's only copy is in +// a staging dir. Nothing here asserts that recovery wins: the claim is that +// neither side destroys the other's copy, that the extract that was already +// running completes, and that the recovering side either waits or says out loud +// that it skipped. The lock is what makes that true, and this is the only test +// that drives both sides of it at once, under -race, through one package-level +// seam. +func TestBundleRecoveryRacesALivePromotion(t *testing.T) { + for _, tc := range []struct { + name string + // gate parks the extract at this row's boundary and reports when it is + // there. staging is the dir the extract allocated, known only once the + // extract is under way for the reap row, which is why this takes dir. + gate func(t *testing.T, dir, dest string) (reached <-chan struct{}, release func()) + }{ + { + // The prior tree is on its way into the staging dir: for that + // instant the destination still holds it and the staging dir holds + // nothing. + name: "set-aside", + gate: func(t *testing.T, dir, dest string) (<-chan struct{}, func()) { + return gateAt(t, func(from, to string) bool { + return from == dest && filepath.Base(to) == "backup" + }) + }, + }, + { + // The destination is absent and the staging dir holds the only copy + // of the prior tree. A recovering pass that acted here would restore + // that copy over the publish this extract is in the middle of. + name: "publish", + gate: func(t *testing.T, dir, dest string) (<-chan struct{}, func()) { + return gateAt(t, func(from, to string) bool { + return filepath.Base(from) == "repo" && to == dest + }) + }, + }, + { + // The publish landed and the commit flag is written, so the copy in + // the staging dir is superseded and the extract is deleting it. A + // recovering pass that reached the same directory would be deleting + // it too. + name: "reap", + gate: func(t *testing.T, dir, dest string) (<-chan struct{}, func()) { + release := blockStep(t, "removeAll") + // Any staging dir under dir: the extract's reap is the only + // removeAll either side makes on this path, since the seed + // extract finished before the gate was installed. + return signalRemoveAll(t, func(path string) bool { + return filepath.Dir(path) == dir && strings.HasPrefix(filepath.Base(path), stagingPrefix) + }), release + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + // Built before the seam is gated: git writes it through os, but a + // gate installed around a step this still reaches would park the + // fixture rather than the extract under test. + bundle := testBundle(t, "a.txt", "v1") + + var watch deleteWatch + watch.install(t) + reached, release := tc.gate(t, dir, dest) + + var extractErr error + extracted := make(chan struct{}) + go func() { + defer close(extracted) + extractErr = extractBundle(context.Background(), bundle, dest, nil) + }() + <-reached + + var logs []string + recovered := make(chan struct{}) + go func() { + defer close(recovered) + var mu sync.Mutex + recoverBundleDir(dir, func(format string, args ...any) { + mu.Lock() + defer mu.Unlock() + logs = append(logs, fmt.Sprintf(format, args...)) + }) + }() + <-recovered + release() + <-extracted + + if extractErr != nil { + t.Errorf("the extract that was already running must finish: %v", extractErr) + } + // Skipped, not raced: the extract holds the destination's lock from + // before its first rename until after its last delete, and the + // recovering pass is required to say so rather than act. + if !logged(logs, "holds proj-1") { + t.Errorf("the recovering pass should report the destination it skipped, got %v", logs) + } + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v1" { + t.Errorf("dest a.txt = %q (err %v), want the extract's own tree %q", got, err, "v1") + } + if _, denied := watch.counts(); len(denied) > 0 { + t.Errorf("neither side may delete an uncommitted copy, got %v", denied) + } + // The extract cleaned up after itself, and recovery left nothing of + // its own behind. + assertCopySet(t, dir, []string{stagingPrefix, keptPrefix}, nil, 1) + }) + } +} + +// ---- the rows no single crash of the writer can reach ---------------------- + +// The X rows of the Acceptance Examples. They need arrangements the six crash +// shapes cannot produce: a held lock, more than one candidate, a destination +// that exists and cannot serve, a restore that fails, or a legacy directory +// planted by hand. Everything else about them is the crash table's contract: +// the whole set of copies under both prefixes is compared, not one path at a +// time, and every row runs twice because recovery keeps no memory. + +// xFile is the destination after a pass. An empty name means it must be absent, +// which is a real terminal state here and not a missing expectation. +type xFile struct { + name string + content string +} + +// xCopy is one copy and where it must be when the pass ends. final is written +// out rather than derived from a disposition, because these rows retain copies +// under three different names (its own, the Kept one, a freshly allocated one) +// and a derived name would encode the test's guess at the rule the row exists to +// check. An empty final is a delete. +type xCopy struct { + final string + file string + content string +} + +type xWant struct { + live xFile + copies []xCopy + // report is the fragments this pass's report must name. A copy recovery + // keeps and never names is one an operator cannot find. + report []string +} + +type bundleXRow struct { + id string + // arrange plants the state and returns what each pass must produce, plus, + // where the row needs one, the fault or held lock that makes the row's + // situation happen during a given pass. They come back together so both can + // close over the paths the arrangement planted. + arrange func(t *testing.T, dir, dest string) (want func(pass int) xWant, impede func(t *testing.T, pass int) func()) +} + +func stagingNamed(dir string, seq int64) string { + return filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, seq, stagingSeqSuffix)) +} + +func keptNamed(dir string, seq int64) string { + return filepath.Join(dir, fmt.Sprintf("%s%020d%s", keptPrefix, seq, stagingSeqSuffix)) +} + +// bothPasses is the common case: a row whose two passes look identical, because +// pass one reached a terminal state and pass two has nothing left to do. +func bothPasses(w xWant) func(int) xWant { + return func(int) xWant { return w } +} + +// faultDuring installs one fault for the length of a pass and returns the undo. +// The pass-scoped restore is the point: a row that fails a step on pass one only +// needs the seam back before pass two runs. +func faultDuring(t *testing.T, step string, match func(args ...string) bool) func() { + t.Helper() + saved := stagingFS + injectFault(t, step, match, errors.New("injected recovery failure")) + return func() { stagingFS = saved } +} + +func bundleXRows() []bundleXRow { + return []bundleXRow{ + { + // A destination that exists and cannot serve is not a reason to keep + // a usable copy out of it, and the husk is not this code's to delete + // either: it moves into a fresh sequenced directory of its own and + // is parked from there. + id: "X1", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + stageBackup(t, dir, "", "proj-1", "v1", 100) + plantUnusableDest(t, dir, "proj-1", "husk") + return func(pass int) xWant { + w := xWant{ + live: xFile{"a.txt", "v1"}, + copies: []xCopy{{final: keptNamed(dir, 101), file: filepath.Join("backup", "husk.txt"), content: "husk"}}, + } + if pass == 1 { + w.report = []string{"keeping the tree that could not serve"} + } + return w + }, nil + }, + }, + { + // A Kept backup is permanent. A later publish is not evidence about + // it: the commit flag goes into the copy the publishing transaction + // set aside, and a Kept backup is never that copy. + id: "X2", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + plantUsableDest(t, dir, "proj-1", "live") + kept := plantKeptBackup(t, dir, "proj-1", "unproven", 500) + return bothPasses(xWant{ + live: xFile{"a.txt", "live"}, + copies: []xCopy{{final: kept, file: filepath.Join("backup", "a.txt"), content: "unproven"}}, + }), nil + }, + }, + { + // The newest usable copy cannot be put back. Falling through to the + // older one would put a tree at the destination that the next pass + // reads as having published over the newer copy, which is how a + // retained copy turns into a deleted one two passes later. + id: "X3", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + older := stageBackup(t, dir, "", "proj-1", "v1", 100) + newest := stageBackup(t, dir, "", "proj-1", "v2", 200) + want := bothPasses(xWant{ + copies: []xCopy{ + {final: older, file: filepath.Join("backup", "a.txt"), content: "v1"}, + {final: newest, file: filepath.Join("backup", "a.txt"), content: "v2"}, + }, + report: []string{newest}, + }) + return want, func(t *testing.T, pass int) func() { + return faultDuring(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(newest, "backup") + }) + } + }, + }, + { + // Unusable is a selection decision, not a delete: the newest copy is + // skipped for one that can serve and is then kept like any other + // copy nothing proves was superseded. + id: "X4", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + stageBackup(t, dir, "", "proj-1", "v1", 100) + newest := stageBackup(t, dir, "", "proj-1", "v2", 200) + if err := os.RemoveAll(filepath.Join(newest, "backup", ".git")); err != nil { + t.Fatal(err) + } + return func(pass int) xWant { + w := xWant{ + live: xFile{"a.txt", "v1"}, + copies: []xCopy{{final: parkedStaging(newest), file: filepath.Join("backup", "a.txt"), content: "v2"}}, + } + if pass == 1 { + w.report = []string{newest} + } + return w + }, nil + }, + }, + { + // Unreadable is not unusable. A filesystem fault says nothing about + // the copy, so the whole destination stops rather than ruling on the + // copies that could be read while the newest one could not. + id: "X5", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + cand := stageBackup(t, dir, "", "proj-1", "v1", 100) + want := bothPasses(xWant{ + copies: []xCopy{{final: cand, file: filepath.Join("backup", "a.txt"), content: "v1"}}, + report: []string{cand}, + }) + return want, func(t *testing.T, pass int) func() { + return faultDuring(t, "stat", func(args ...string) bool { + return args[0] == filepath.Join(cand, "backup") + }) + } + }, + }, + { + // A destination someone else holds is one recovery knows nothing + // about: on disk a live extract mid-swap and a crashed one are the + // same thing, and the lock is the only thing that tells them apart. + id: "X6", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + cand := stageBackup(t, dir, "", "proj-1", "v1", 100) + want := func(pass int) xWant { + if pass == 1 { + return xWant{ + copies: []xCopy{{final: cand, file: filepath.Join("backup", "a.txt"), content: "v1"}}, + report: []string{"holds proj-1"}, + } + } + return xWant{live: xFile{"a.txt", "v1"}} + } + return want, func(t *testing.T, pass int) func() { + if pass != 1 { + return func() {} + } + return lockExtract(dest) + } + }, + }, + { + // A published work tree can sit under the exact generated name, so + // the marker beside it proves nothing: the work-tree veto runs first + // and the directory is left alone whatever the marker says. + id: "X7", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + legacy := stagingNamed(dir, 300) + if err := os.MkdirAll(filepath.Join(legacy, ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(legacy, "a.txt"), []byte("legacy"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeMarker(legacy, txnMarker{Kind: txnKindBundleExtract, Dest: "proj-1", Seq: 300}); err != nil { + t.Fatal(err) + } + return bothPasses(xWant{ + copies: []xCopy{{final: legacy, file: "a.txt", content: "legacy"}}, + report: []string{"holds a work tree"}, + }), nil + }, + }, + { + // A name that shares the prefix and fails the grammar carries no + // order at all, so it is not a candidate for anything: not restored, + // not parked, not deleted. It is still reported, because at this + // site a dot-prefixed sibling nothing attributes is residue an + // operator has to be able to see. + id: "X8", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + sibling := stageBackup(t, dir, "foo", "proj-1", "sib", 0) + return bothPasses(xWant{ + copies: []xCopy{{final: sibling, file: filepath.Join("backup", "a.txt"), content: "sib"}}, + report: []string{"does not carry a name this code writes"}, + }), nil + }, + }, + { + // The destination went away outside this code and the only copy left + // carries a commit flag. Committed is second in selection, never + // excluded from it: a copy that was superseded by a tree that is no + // longer there is still a copy of something. + id: "X9", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + markCommitted(t, stageBackup(t, dir, "", "proj-1", "v1", 100)) + return bothPasses(xWant{live: xFile{"a.txt", "v1"}}), nil + }, + }, + { + // Two copies, one destination: the newest goes back and the older is + // kept rather than deleted, because nothing proves anything + // published over it either. + id: "X10", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + older := stageBackup(t, dir, "", "proj-1", "v1", 100) + stageBackup(t, dir, "", "proj-1", "v2", 200) + return func(pass int) xWant { + w := xWant{ + live: xFile{"a.txt", "v2"}, + copies: []xCopy{{final: parkedStaging(older), file: filepath.Join("backup", "a.txt"), content: "v1"}}, + } + if pass == 1 { + w.report = []string{older} + } + return w + }, nil + }, + }, + { + // v0.8.0 staged under os.MkdirTemp, whose suffix carries no sequence + // at all, and a hand-planted name can fail the grammar in the other + // direction. Neither is orderable, so both are retained in place and + // reported rather than guessed at. + id: "X11", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + residue := stageBackup(t, dir, "3921749", "proj-1", "resid", 0) + tied := stageBackup(t, dir, "0000000000000000000x"+stagingSeqSuffix, "proj-1", "tied", 0) + return bothPasses(xWant{ + copies: []xCopy{ + {final: residue, file: filepath.Join("backup", "a.txt"), content: "resid"}, + {final: tied, file: filepath.Join("backup", "a.txt"), content: "tied"}, + }, + report: []string{"does not carry a name this code writes"}, + }), nil + }, + }, + { + // Same as X1 with the only candidate committed. The husk still moves + // aside first, and the copy that replaces it is the committed one, + // because there is no uncommitted copy to prefer. + id: "X12", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + markCommitted(t, stageBackup(t, dir, "", "proj-1", "v1", 100)) + plantUnusableDest(t, dir, "proj-1", "husk") + return func(pass int) xWant { + w := xWant{ + live: xFile{"a.txt", "v1"}, + copies: []xCopy{{final: keptNamed(dir, 101), file: filepath.Join("backup", "husk.txt"), content: "husk"}}, + } + if pass == 1 { + w.report = []string{"keeping the tree that could not serve"} + } + return w + }, nil + }, + }, + { + // Nothing beside it can replace the husk, so it is not taken apart: + // an operator with an unusable destination has strictly more than + // one with no destination at all. + id: "X13", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + cand := stageBackup(t, dir, "", "proj-1", "v1", 100) + if err := os.RemoveAll(filepath.Join(cand, "backup", ".git")); err != nil { + t.Fatal(err) + } + plantUnusableDest(t, dir, "proj-1", "husk") + return func(pass int) xWant { + w := xWant{ + live: xFile{"husk.txt", "husk"}, + copies: []xCopy{{final: parkedStaging(cand), file: filepath.Join("backup", "a.txt"), content: "v1"}}, + } + if pass == 1 { + w.report = []string{"no usable staged copy"} + } + return w + }, nil + }, + }, + { + // The husk was already aside when the restore failed, so recovery + // owes the destination its husk back: the row's whole claim is that + // the destination ends the pass exactly as it was found, with the + // candidate still where it was and the set-aside directory, now + // empty, gone. + id: "X14", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + cand := stageBackup(t, dir, "", "proj-1", "v1", 100) + plantUnusableDest(t, dir, "proj-1", "husk") + want := bothPasses(xWant{ + live: xFile{"husk.txt", "husk"}, + copies: []xCopy{{final: cand, file: filepath.Join("backup", "a.txt"), content: "v1"}}, + report: []string{cand}, + }) + return want, func(t *testing.T, pass int) func() { + return faultDuring(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(cand, "backup") && args[1] == dest + }) + } + }, + }, + } +} + +func TestBundleXMatrix(t *testing.T) { + for _, row := range bundleXRows() { + t.Run(row.id, func(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + want, impede := row.arrange(t, dir, dest) + + var watch deleteWatch + for pass := 1; pass <= 2; pass++ { + var logs []string + func() { + saved := stagingFS + defer func() { stagingFS = saved }() + if impede != nil { + defer impede(t, pass)() + } + // After the impediment, so a delete the injected fault + // would have failed is still seen as a delete that was + // asked for. + watch.install(t) + logs = recoverAndLog(t, dir) + }() + assertBundleXState(t, dir, dest, want(pass), logs, pass) + } + if _, denied := watch.counts(); len(denied) > 0 { + t.Errorf("no row may delete a copy nothing proves was superseded, got %v", denied) + } + }) + } +} + +// assertBundleXState asserts a row's whole terminal state: the destination, the +// content of every copy at the name it is supposed to be at, the full set of +// directories under both prefixes, and the report. Content rather than names +// alone, because a park that moved a name and lost the tree passes every name +// assertion there is. +func assertBundleXState(t *testing.T, dir, dest string, want xWant, logs []string, pass int) { + t.Helper() + if want.live.name == "" { + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Errorf("pass %d: %s should be absent: %v", pass, dest, err) + } + } else { + got, err := os.ReadFile(filepath.Join(dest, want.live.name)) + if err != nil || string(got) != want.live.content { + t.Errorf("pass %d: dest %s = %q (err %v), want %q", pass, want.live.name, got, err, want.live.content) + } + } + var paths []string + for _, c := range want.copies { + if c.final == "" { + continue + } + paths = append(paths, c.final) + got, err := os.ReadFile(filepath.Join(c.final, c.file)) + if err != nil || string(got) != c.content { + t.Errorf("pass %d: %s = %q (err %v), want %q", pass, filepath.Join(c.final, c.file), got, err, c.content) + } + } + assertCopySet(t, dir, []string{stagingPrefix, keptPrefix}, paths, pass) + for _, fragment := range want.report { + if !logged(logs, fragment) { + t.Errorf("pass %d: the report should name %q, got %v", pass, fragment, logs) + } + } +} diff --git a/internal/dictation/download_matrix_test.go b/internal/dictation/download_matrix_test.go index db76086d7..536fae9b6 100644 --- a/internal/dictation/download_matrix_test.go +++ b/internal/dictation/download_matrix_test.go @@ -1,13 +1,16 @@ package dictation import ( + "context" "errors" "fmt" "os" "path/filepath" "slices" "strings" + "sync" "testing" + "time" ) // The crash rows of the recovery plan's Acceptance Examples, as code. Each row @@ -411,3 +414,888 @@ func assertNoOtherEntries(t *testing.T, dir string, allowed ...string) { } } } + +// ---- the regression net: recovery's own steps, and a live promotion -------- + +// deleteWatch records every removeAll recovery asks for, and flags the ones no +// classification licenses: a holder that still holds a copy of an install with +// no commit flag beside it. It is asserted AT THE SEAM rather than off the disk +// because a fault-injected removeAll leaves the copy exactly where a removeAll +// that was never requested does, so a row reading the surviving files could not +// tell a recovery that did the right thing from one that tried to do the wrong +// thing and was stopped by the injected fault. +type deleteWatch struct { + mu sync.Mutex + all []string + denied []string +} + +// install swaps removeAll for a recorder, restoring the seam in t.Cleanup. The +// order matters: only a watch installed AFTER a fault sees the calls that fault +// is failing, and cleanups unwind in reverse, so the seam still ends the test as +// the real filesystem. +func (w *deleteWatch) install(t *testing.T) { + t.Helper() + real := holderFS + t.Cleanup(func() { holderFS = real }) + holderFS.removeAll = func(path string) error { + w.mu.Lock() + w.all = append(w.all, path) + if holdsUncommittedCopy(path) { + w.denied = append(w.denied, path) + } + w.mu.Unlock() + return real.removeAll(path) + } +} + +func (w *deleteWatch) counts() (all, denied []string) { + w.mu.Lock() + defer w.mu.Unlock() + return slices.Clone(w.all), slices.Clone(w.denied) +} + +// holdsUncommittedCopy reports whether path still holds a copy of an install +// that nothing proves was published over. Recovery may park one of these +// forever; it may never delete one, whatever step just failed. At this site that +// copy is the only offline copy of an engine or a model. +func holdsUncommittedCopy(path string) bool { + if _, err := os.Stat(filepath.Join(path, "install")); err != nil { + return false + } + _, err := os.Stat(filepath.Join(path, committedFile)) + return os.IsNotExist(err) +} + +// The watch is the whole regression net below, so it gets its own proof: a net +// that cannot see the delete it exists to catch would make every row under it +// pass for the wrong reason. The three shapes are the two deletes recovery is +// licensed to make and the one it is not. +func TestDeleteWatchSeesOnlyTheDeleteRecoveryMayNotMake(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + uncommitted := plantHolder(t, dest, 100, "v1", false) + committed := plantHolder(t, dest, 200, "v2", true) + scratch := plantScratchHolder(t, dest, 50) + + var watch deleteWatch + func() { + saved := holderFS + defer func() { holderFS = saved }() + watch.install(t) + for _, path := range []string{committed, scratch, uncommitted} { + if err := holderFS.removeAll(path); err != nil { + t.Fatal(err) + } + } + }() + + all, denied := watch.counts() + if len(all) != 3 { + t.Errorf("the watch should have seen all three deletes, got %v", all) + } + if !slices.Equal(denied, []string{uncommitted}) { + t.Errorf("only the uncommitted copy is a violation, got %v", denied) + } +} + +// plantScratchHolder plants an owned holder that holds no copy of any install: +// the state a crash between the marker and the set-aside leaves. It is the one +// shape besides a committed copy that recovery may delete. +func plantScratchHolder(t *testing.T, destDir string, seq int64) string { + t.Helper() + holder := holderNamed(destDir, seq) + if err := os.MkdirAll(holder, 0o755); err != nil { + t.Fatal(err) + } + if err := writeHolderMarker(holder, txnMarker{Kind: holderMarkerKind, Dest: filepath.Base(destDir), Seq: seq}); err != nil { + t.Fatal(err) + } + return holder +} + +func holderNamed(destDir string, seq int64) string { + return fmt.Sprintf("%s%s%020d%s", destDir, holderSuffix, seq, holderSeqSuffix) +} + +func keptNamed(destDir string, seq int64) string { + return fmt.Sprintf("%s%s%020d%s", destDir, keptSuffix, seq, holderSeqSuffix) +} + +// recoveryStep is one filesystem step of a recovery pass, with the fault that +// fails it. Each is matched where it is reached rather than by call ordinal, so +// a change in how many times recovery stats a holder cannot silently move which +// call the row is failing. +type recoveryStep struct { + name string + inject func(t *testing.T, newest string) +} + +func dictationRecoverySteps() []recoveryStep { + failed := errors.New("injected recovery failure") + return []recoveryStep{ + { + name: "readDir", + inject: func(t *testing.T, _ string) { injectFault(t, "readDir", nil, failed) }, + }, + { + name: "stat", + inject: func(t *testing.T, _ string) { injectFault(t, "stat", nil, failed) }, + }, + { + name: "marker read", + inject: func(t *testing.T, _ string) { injectFault(t, "readFile", nil, failed) }, + }, + { + name: "restore rename", + inject: func(t *testing.T, newest string) { + injectFault(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(newest, "install") + }, failed) + }, + }, + { + name: "removeAll", + inject: func(t *testing.T, _ string) { injectFault(t, "removeAll", nil, failed) }, + }, + { + name: "park rename", + inject: func(t *testing.T, _ string) { + injectFault(t, "rename", func(args ...string) bool { + return strings.Contains(filepath.Base(args[1]), keptSuffix) + }, failed) + }, + }, + } +} + +// Every recovery step, failed on pass one, on pass two, and on both. Two claims +// per case, and the first is the one the whole branch exists for: no failure of +// any step ever produces a delete of a copy nothing proves was superseded. The +// second is that a fault only DELAYS recovery: a later pass with the fault gone +// reaches the same terminal state a clean run reaches, so a step that failed +// once has not left the destination in a state recovery can no longer reason +// about. +func TestDictationRecoveryStepFailures(t *testing.T) { + for _, step := range dictationRecoverySteps() { + for _, when := range []struct { + name string + faulty [2]bool + }{ + {name: "pass one", faulty: [2]bool{true, false}}, + {name: "pass two", faulty: [2]bool{false, true}}, + {name: "both passes", faulty: [2]bool{true, true}}, + } { + t.Run(step.name+"/"+when.name, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + // Two uncommitted usable copies and one owned holder that holds + // nothing, with the destination gone: the state that drives + // every step in the list through a real decision. One copy is + // restored, one is parked, and the empty holder is the only + // delete on the happy path, so a step that fails has something + // to corrupt. + older := plantHolder(t, dest, 100, "v1", false) + newest := plantHolder(t, dest, 200, "v2", false) + scratch := plantScratchHolder(t, dest, 50) + txn := lockFor(t, dest) + + var watch deleteWatch + pass := func(faulty bool) { + saved := holderFS + defer func() { holderFS = saved }() + if faulty { + step.inject(t, newest) + } + // After the fault, so the watch sees the calls the fault is + // failing rather than only the ones it lets through. + watch.install(t) + restoreInterruptedPromotion(txn, dest, testPublished, nil) + } + pass(when.faulty[0]) + if when.faulty[0] && dictationStepTerminal(dest, older, newest, scratch) { + // The row would pass without the code being tested doing + // anything: a fault that changes nothing is a fault that was + // never reached, and every assertion below it is vacuous. + t.Errorf("the injected %s failure left the pass free to finish, so this row proves nothing", step.name) + } + pass(when.faulty[1]) + // The fault is gone, and recovery has to be able to finish from + // wherever the failed passes left the destination. + pass(false) + + all, denied := watch.counts() + if len(denied) > 0 { + t.Errorf("no step failure may produce a delete of an uncommitted copy, got %v", denied) + } + if len(all) == 0 { + t.Fatal("the watch saw no delete at all, so it proves nothing about this pass") + } + assertDictationStepTerminalState(t, dest, older, newest, scratch) + }) + } + } +} + +// dictationStepTerminal reports whether the fixture reached the state a clean +// run reaches. It is a predicate rather than an assertion because it is asked +// twice for opposite reasons: a faulted pass must NOT have got here, and the +// pass after the fault is gone must have. +func dictationStepTerminal(dest, older, newest, scratch string) bool { + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "v2" { + return false + } + kept, ok := keptHolderPath(older) + if !ok { + return false + } + if _, err := os.Stat(filepath.Join(kept, "install", "engine")); err != nil { + return false + } + for _, gone := range []string{newest, scratch} { + if _, err := os.Stat(gone); !os.IsNotExist(err) { + return false + } + } + return true +} + +// assertDictationStepTerminalState is where the fixture above has to end up once +// nothing is failing: the newest copy live, its own holder gone, the older one +// kept under the Kept prefix with its install intact, and the empty holder +// reaped. Content is read back rather than names checked, because a park that +// moved a name and lost the install passes every name assertion. +func assertDictationStepTerminalState(t *testing.T, dest, older, newest, scratch string) { + t.Helper() + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "v2" { + t.Errorf("dest engine = %q (err %v), want the newest copy %q", got, err, "v2") + } + parked := keptName(t, dest, older) + kept, err := os.ReadFile(filepath.Join(parked, "install", "engine")) + if err != nil || string(kept) != "v1" { + t.Errorf("the parked copy's engine = %q (err %v), want %q", kept, err, "v1") + } + for _, gone := range []string{newest, scratch} { + if _, err := os.Stat(gone); !os.IsNotExist(err) { + t.Errorf("%s should be gone once recovery finished: %v", gone, err) + } + } + base := filepath.Base(dest) + assertCopySet(t, filepath.Dir(dest), []string{base + holderSuffix, base + keptSuffix}, []string{parked}, 3) +} + +// gateAt parks the write path at one named boundary: it signals when the call +// whose arguments match is reached, holds that call until release, and passes +// every other call through. blockStep gates EVERY call to a step, which for a +// rename would park a promotion at the rename inside its marker write rather +// than at the set-aside or the publish, so a matched gate is what puts the +// writer at the boundary a row is actually about. +func gateAt(t *testing.T, match func(from, to string) bool) (reached <-chan struct{}, release func()) { + t.Helper() + real := holderFS + t.Cleanup(func() { holderFS = real }) + arrived := make(chan struct{}) + gate := make(chan struct{}) + var arrive, open sync.Once + release = func() { open.Do(func() { close(gate) }) } + t.Cleanup(release) + holderFS.rename = func(from, to string) error { + if match(from, to) { + arrive.Do(func() { close(arrived) }) + <-gate + } + return real.rename(from, to) + } + return arrived, release +} + +// signalRemoveAll reports when a removeAll of a matching path is reached. It +// wraps whatever is already installed, so layering it over blockStep signals +// BEFORE the call parks rather than after it is released. +func signalRemoveAll(t *testing.T, match func(path string) bool) <-chan struct{} { + t.Helper() + installed := holderFS + t.Cleanup(func() { holderFS = installed }) + arrived := make(chan struct{}) + var arrive sync.Once + holderFS.removeAll = func(path string) error { + if match(path) { + arrive.Do(func() { close(arrived) }) + } + return installed.removeAll(path) + } + return arrived +} + +// A recovery pass and a live promotion for one destination, running at the same +// time, at each of the three boundaries where the destination's only copy is in +// a holder. Nothing here asserts that recovery wins: the claim is that neither +// side destroys the other's copy, that the promotion that was already running +// completes, and that the recovering side either waits the lock out or is told +// the install is in progress. The Install lock is what makes that true, and this +// is the only test that drives both sides of it at once, under -race, through +// one package-level seam. +func TestDictationRecoveryRacesALivePromotion(t *testing.T) { + for _, tc := range []struct { + name string + gate func(t *testing.T, root, dest string) (reached <-chan struct{}, release func()) + }{ + { + // The previous install is on its way into the holder: for that + // instant the destination still holds it and the holder is empty. + name: "set-aside", + gate: func(t *testing.T, root, dest string) (<-chan struct{}, func()) { + return gateAt(t, func(from, to string) bool { + return from == dest && filepath.Base(to) == "install" + }) + }, + }, + { + // The destination is absent and the holder has the only copy of the + // previous install. A recovering pass that acted here would restore + // that copy over the publish this promotion is in the middle of. + name: "publish", + gate: func(t *testing.T, root, dest string) (<-chan struct{}, func()) { + return gateAt(t, func(from, to string) bool { + return to == dest && filepath.Base(from) == "stage" + }) + }, + }, + { + // The publish landed and the commit flag is written, so the copy in + // the holder is superseded and the promotion is deleting it. A + // recovering pass that reached the same holder would be deleting it + // too. + name: "reap", + gate: func(t *testing.T, root, dest string) (<-chan struct{}, func()) { + release := blockStep(t, "removeAll") + base := filepath.Base(dest) + return signalRemoveAll(t, func(path string) bool { + return strings.HasPrefix(filepath.Base(path), base+holderSuffix) + }), release + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + txn := lockFor(t, dest) + + var watch deleteWatch + watch.install(t) + reached, release := tc.gate(t, root, dest) + + var promoteErr error + promoted := make(chan struct{}) + go func() { + defer close(promoted) + promoteErr = promoteStagedDir(txn, stage, dest, "engine", nil) + // The recovering side is waiting on this lock, and the only + // thing that ends its wait is the promotion letting go of it. + txn.release() + }() + <-reached + + var recovered bool + var lockErr error + var reports []string + done := make(chan struct{}) + go func() { + defer close(done) + lockErr = withDestinationLock(context.Background(), root, filepath.Base(dest), testPublished, func(own *destTxn) error { + recovered = true + restoreInterruptedPromotion(own, dest, testPublished, reporterFor(&reports)) + return nil + }) + }() + release() + <-promoted + <-done + + if promoteErr != nil { + t.Errorf("the promotion that was already running must finish: %v", promoteErr) + } + // Waited, or was told the install is in progress. Either is a pass; + // acting on the destination while the promotion held it is not. + if !recovered && !errors.Is(lockErr, errInstallInProgress) { + t.Errorf("the recovering side should have waited or reported the install in progress, got %v", lockErr) + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "new" { + t.Errorf("dest engine = %q (err %v), want the promotion's own install %q", got, err, "new") + } + if _, denied := watch.counts(); len(denied) > 0 { + t.Errorf("neither side may delete an uncommitted copy, got %v", denied) + } + // The promotion cleaned up after itself, and recovery left nothing + // of its own behind. + base := filepath.Base(dest) + assertCopySet(t, root, []string{base + holderSuffix, base + keptSuffix}, nil, 1) + }) + } +} + +// ---- the rows no single crash of the writer can reach ---------------------- + +// The X rows of the Acceptance Examples. They need arrangements the six crash +// shapes cannot produce: a held lock, more than one candidate, a destination +// that exists and cannot serve, a restore that fails, or a legacy directory +// planted by hand. Everything else about them is the crash table's contract: +// the whole set of copies beside the destination is compared, not one path at a +// time, and every row runs twice because recovery keeps no memory. + +// xFile is the destination after a pass. An empty name means it must be absent, +// which is a real terminal state here and not a missing expectation. +type xFile struct { + name string + content string +} + +// xCopy is one copy and where it must be when the pass ends. final is written +// out rather than derived from a disposition, because these rows retain copies +// under three different names (its own, the Kept one, a freshly allocated one) +// and a derived name would encode the test's guess at the rule the row exists to +// check. An empty final is a delete. +type xCopy struct { + final string + file string + content string +} + +type xWant struct { + live xFile + copies []xCopy + // report is the fragments this pass's report must name. A copy recovery + // keeps and never names is one an operator cannot find. + report []string + // silent is the fragments it must NOT name. A sibling that merely collides + // with the prefix is not a copy of this install, and reporting on it would + // tell an operator their own directory is recovery's residue. + silent []string +} + +type dictationXRow struct { + id string + // arrange plants the state and returns what each pass must produce, plus, + // where the row needs one, the fault that makes the row's situation happen + // during a given pass. They come back together so both can close over the + // paths the arrangement planted. + arrange func(t *testing.T, root, dest string) (want func(pass int) xWant, impede func(t *testing.T, pass int) func()) + // recover overrides how the pass is driven, for the one row whose subject is + // the lock itself rather than what is on disk. + recover func(t *testing.T, root, dest string, pass int) []string +} + +// bothPasses is the common case: a row whose two passes look identical, because +// pass one reached a terminal state and pass two has nothing left to do. +func bothPasses(w xWant) func(int) xWant { + return func(int) xWant { return w } +} + +// faultDuring installs one fault for the length of a pass and returns the undo. +// The pass-scoped restore is the point: a row that fails a step on pass one only +// needs the seam back before pass two runs. +func faultDuring(t *testing.T, step string, match func(args ...string) bool) func() { + t.Helper() + saved := holderFS + injectFault(t, step, match, errors.New("injected recovery failure")) + return func() { holderFS = saved } +} + +// plantSibling creates a directory beside the destination that shares the holder +// prefix and is not a holder: someone else's directory, which recovery has to +// leave alone and not report on. +func plantSibling(t *testing.T, path, content string) string { + t.Helper() + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "notes"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func dictationXRows() []dictationXRow { + return []dictationXRow{ + { + // A destination that exists and cannot serve is not a reason to keep + // a usable copy out of it, and the husk is not this code's to delete + // either: it moves into a fresh sequenced holder of its own and is + // parked from there. + id: "X1", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + plantHolder(t, dest, 100, "v1", false) + plantUnusableDest(t, dest) + kept := keptNamed(dest, 101) + return func(pass int) xWant { + w := xWant{ + live: xFile{"engine", "v1"}, + copies: []xCopy{{final: kept, file: filepath.Join("install", "bin", "README"), content: "partial"}}, + } + if pass == 1 { + w.report = []string{kept} + } + return w + }, nil + }, + }, + { + // A Kept backup is permanent. A later publish is not evidence about + // it: the commit flag goes into the holder the publishing + // transaction set aside, and a Kept backup is never that holder. + id: "X2", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + stagedTree(t, dest, "live") + kept := plantKeptHolder(t, dest, 500, "unproven") + return bothPasses(xWant{ + live: xFile{"engine", "live"}, + copies: []xCopy{{final: kept, file: filepath.Join("install", "engine"), content: "unproven"}}, + }), nil + }, + }, + { + // The newest usable copy cannot be put back. Falling through to the + // older one would put an install at the destination that the next + // pass reads as having published over the newer copy, which is how a + // retained copy turns into a deleted one two passes later. + id: "X3", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + older := plantHolder(t, dest, 100, "v1", false) + newest := plantHolder(t, dest, 200, "v2", false) + want := bothPasses(xWant{ + copies: []xCopy{ + {final: older, file: filepath.Join("install", "engine"), content: "v1"}, + {final: newest, file: filepath.Join("install", "engine"), content: "v2"}, + }, + report: []string{newest}, + }) + return want, func(t *testing.T, pass int) func() { + return faultDuring(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(newest, "install") + }) + } + }, + }, + { + // Unusable is a selection decision, not a delete: the newest copy is + // skipped for one that can serve and is then kept like any other + // copy nothing proves was superseded. + id: "X4", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + plantHolder(t, dest, 100, "v1", false) + newest := plantHolder(t, dest, 200, "v2", false) + if err := os.Rename(filepath.Join(newest, "install", "engine"), filepath.Join(newest, "install", "partial")); err != nil { + t.Fatal(err) + } + kept := keptName(t, dest, newest) + return func(pass int) xWant { + w := xWant{ + live: xFile{"engine", "v1"}, + copies: []xCopy{{final: kept, file: filepath.Join("install", "partial"), content: "v2"}}, + } + if pass == 1 { + w.report = []string{kept} + } + return w + }, nil + }, + }, + { + // Unreadable is not unusable. A filesystem fault says nothing about + // the copy, so the whole destination stops rather than ruling on the + // copies that could be read while the newest one could not. + id: "X5", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + holder := plantHolder(t, dest, 100, "v1", false) + want := bothPasses(xWant{ + copies: []xCopy{{final: holder, file: filepath.Join("install", "engine"), content: "v1"}}, + report: []string{holder}, + }) + return want, func(t *testing.T, pass int) func() { + return faultDuring(t, "stat", func(args ...string) bool { + return args[0] == filepath.Join(holder, committedFile) + }) + } + }, + }, + { + // A destination another process holds is one recovery knows nothing + // about: on disk a live promotion mid-swap and a crashed one are the + // same thing, and the Install lock is the only thing that tells them + // apart. The lock is per open file description, so the handle this + // test holds during pass one is what a second process's hold looks + // like to the recovering side. + id: "X6", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + holder := plantHolder(t, dest, 100, "v1", false) + return func(pass int) xWant { + if pass == 1 { + return xWant{ + copies: []xCopy{{final: holder, file: filepath.Join("install", "engine"), content: "v1"}}, + } + } + return xWant{live: xFile{"engine", "v1"}} + }, nil + }, + recover: recoverAroundAHeldLock, + }, + { + // A name that shares the prefix and fails the grammar carries no + // order at all, so it is not a candidate for anything: not restored, + // not parked, not deleted. At this site it is not reported either, + // because the prefix is a suffix of someone's own directory name and + // telling an operator their notes are recovery's residue is worse + // than saying nothing. + id: "X8", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + sibling := plantSibling(t, dest+holderSuffix+"notes", "mine") + return bothPasses(xWant{ + copies: []xCopy{{final: sibling, file: "notes", content: "mine"}}, + silent: []string{sibling}, + }), nil + }, + }, + { + // The destination went away outside this code and the only copy left + // carries a commit flag. Committed is second in selection, never + // excluded from it: a copy that was superseded by an install that is + // no longer there is still a copy of something. + id: "X9", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + plantHolder(t, dest, 100, "v1", true) + return bothPasses(xWant{live: xFile{"engine", "v1"}}), nil + }, + }, + { + // Two copies, one destination: the newest goes back and the older is + // kept rather than deleted, because nothing proves anything + // published over it either. + id: "X10", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + older := plantHolder(t, dest, 100, "v1", false) + plantHolder(t, dest, 200, "v2", false) + kept := keptName(t, dest, older) + return func(pass int) xWant { + w := xWant{ + live: xFile{"engine", "v2"}, + copies: []xCopy{{final: kept, file: filepath.Join("install", "engine"), content: "v1"}}, + } + if pass == 1 { + w.report = []string{kept} + } + return w + }, nil + }, + }, + { + // v0.8.0 residue: a holder-shaped name with no sequence in it. It + // cannot be ordered against anything, so it is left exactly where it + // is, and like any other unstampable sibling at this site it is not + // reported. + id: "X11", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + residue := plantSibling(t, dest+holderSuffix+"1699999999", "resid") + return bothPasses(xWant{ + copies: []xCopy{{final: residue, file: "notes", content: "resid"}}, + silent: []string{residue}, + }), nil + }, + }, + { + // Same as X1 with the only candidate committed. The husk still moves + // aside first, and the copy that replaces it is the committed one, + // because there is no uncommitted copy to prefer. + id: "X12", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + plantHolder(t, dest, 100, "v1", true) + plantUnusableDest(t, dest) + kept := keptNamed(dest, 101) + return func(pass int) xWant { + w := xWant{ + live: xFile{"engine", "v1"}, + copies: []xCopy{{final: kept, file: filepath.Join("install", "bin", "README"), content: "partial"}}, + } + if pass == 1 { + w.report = []string{kept} + } + return w + }, nil + }, + }, + { + // Nothing beside it can replace the husk, so it is not taken apart: + // an operator with an unusable install directory has strictly more + // than one with no install directory at all. + id: "X13", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + holder := plantHolder(t, dest, 100, "v1", false) + if err := os.Rename(filepath.Join(holder, "install", "engine"), filepath.Join(holder, "install", "partial")); err != nil { + t.Fatal(err) + } + plantUnusableDest(t, dest) + kept := keptName(t, dest, holder) + return func(pass int) xWant { + w := xWant{ + live: xFile{filepath.Join("bin", "README"), "partial"}, + copies: []xCopy{{final: kept, file: filepath.Join("install", "partial"), content: "v1"}}, + } + if pass == 1 { + w.report = []string{"no usable install"} + } + return w + }, nil + }, + }, + { + // The husk was already aside when the restore failed, so recovery + // owes the destination its husk back: the row's whole claim is that + // the destination ends the pass exactly as it was found, with the + // candidate still where it was and the set-aside holder, now empty, + // gone. + id: "X14", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + holder := plantHolder(t, dest, 100, "v1", false) + plantUnusableDest(t, dest) + want := bothPasses(xWant{ + live: xFile{filepath.Join("bin", "README"), "partial"}, + copies: []xCopy{{final: holder, file: filepath.Join("install", "engine"), content: "v1"}}, + report: []string{holder}, + }) + return want, func(t *testing.T, pass int) func() { + return faultDuring(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(holder, "install") && args[1] == dest + }) + } + }, + }, + } +} + +// recoverAroundAHeldLock drives X6 through the production entry point rather +// than calling the reconcile directly: the lock is the row's subject, and +// withDestinationLock is the only place a caller finds out it could not have it. +// Pass one runs with the destination's lock held by a handle this test owns, +// which is what a second process holding it looks like from here; pass two runs +// with it free. +func recoverAroundAHeldLock(t *testing.T, root, dest string, pass int) []string { + t.Helper() + var reports []string + recovered := false + if pass == 1 { + held := lockFor(t, dest) + defer held.release() + // Long enough that a slow machine does not turn a wait into a skip by + // accident, short enough that the row does not sit on the default + // two-minute budget for a lock nothing is going to release. + saved := installLockWait + installLockWait = 200 * time.Millisecond + defer func() { installLockWait = saved }() + err := withDestinationLock(context.Background(), root, filepath.Base(dest), testPublished, func(own *destTxn) error { + recovered = true + return nil + }) + if recovered { + t.Fatal("the lock was held for the whole pass, so recovery must not have taken it") + } + if !errors.Is(err, errInstallInProgress) { + t.Fatalf("a held destination should report the install in progress, got %v", err) + } + return []string{err.Error()} + } + if err := withDestinationLock(context.Background(), root, filepath.Base(dest), testPublished, func(own *destTxn) error { + restoreInterruptedPromotion(own, dest, testPublished, reporterFor(&reports)) + return nil + }); err != nil { + t.Fatalf("the lock is free on this pass: %v", err) + } + return reports +} + +func TestDictationXMatrix(t *testing.T) { + for _, row := range dictationXRows() { + t.Run(row.id, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + want, impede := row.arrange(t, root, dest) + var txn *destTxn + if row.recover == nil { + // One handle for the whole row: the lock is per open file + // description, so a second acquire in this process would + // contend with the first rather than nest. + txn = lockFor(t, dest) + } + + var watch deleteWatch + for pass := 1; pass <= 2; pass++ { + var reports []string + func() { + saved := holderFS + defer func() { holderFS = saved }() + if impede != nil { + defer impede(t, pass)() + } + // After the impediment, so a delete the injected fault + // would have failed is still seen as a delete that was + // asked for. + watch.install(t) + if row.recover != nil { + reports = row.recover(t, root, dest, pass) + return + } + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reports)) + }() + assertDictationXState(t, dest, want(pass), reports, pass) + } + if _, denied := watch.counts(); len(denied) > 0 { + t.Errorf("no row may delete a copy nothing proves was superseded, got %v", denied) + } + }) + } +} + +// assertDictationXState asserts a row's whole terminal state: the destination, +// the content of every copy at the name it is supposed to be at, the full set of +// directories under both prefixes, and the report. Content rather than names +// alone, because a park that moved a name and lost the install passes every name +// assertion there is. +func assertDictationXState(t *testing.T, dest string, want xWant, reports []string, pass int) { + t.Helper() + if want.live.name == "" { + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Errorf("pass %d: %s should be absent: %v", pass, dest, err) + } + } else { + got, err := os.ReadFile(filepath.Join(dest, want.live.name)) + if err != nil || string(got) != want.live.content { + t.Errorf("pass %d: dest %s = %q (err %v), want %q", pass, want.live.name, got, err, want.live.content) + } + } + var paths []string + for _, c := range want.copies { + if c.final == "" { + continue + } + paths = append(paths, c.final) + got, err := os.ReadFile(filepath.Join(c.final, c.file)) + if err != nil || string(got) != c.content { + t.Errorf("pass %d: %s = %q (err %v), want %q", pass, filepath.Join(c.final, c.file), got, err, c.content) + } + } + base := filepath.Base(dest) + assertCopySet(t, filepath.Dir(dest), []string{base + holderSuffix, base + keptSuffix}, paths, pass) + for _, fragment := range want.report { + if !slices.ContainsFunc(reports, func(m string) bool { return strings.Contains(m, fragment) }) { + t.Errorf("pass %d: the report should name %q, got %v", pass, fragment, reports) + } + } + for _, fragment := range want.silent { + if slices.ContainsFunc(reports, func(m string) bool { return strings.Contains(m, fragment) }) { + t.Errorf("pass %d: the report should say nothing about %q, got %v", pass, fragment, reports) + } + } +} From dbbdba4991f1af55ccc4bd753a6fb3fe9f3b6978 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:54:50 -0500 Subject: [PATCH 30/34] fix(daemon/remote): treat an unreadable work-tree probe as a work tree The veto that keeps recovery away from a published work tree ran on stat returning nil, so every error answered the same as a confirmed absence and the directory fell through to its marker. The veto sits ahead of the marker for a reason: a published tree can carry a file named txn at its own root, so a permission or IO error on that one probe let the tree answer for itself. An error that is not ENOENT now retains and reports, which is the same rule already applied to an unreadable candidate a few lines down. --- internal/daemon/remote/bundle.go | 7 +++++++ internal/daemon/remote/bundle_test.go | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index 8ac98c681..1510a5ec7 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -436,6 +436,13 @@ func attributeStagingDir(dir, staging string, seq int64, logf func(string, ...an if _, err := stagingFS.stat(filepath.Join(staging, ".git")); err == nil { logf("remote: %s holds a work tree, not a staged extract; leaving it in place", staging) return "", false + } else if !errors.Is(err, fs.ErrNotExist) { + // Not knowing whether this is a work tree is not the same as knowing it + // is not one. The veto sits ahead of the marker because a published tree + // can carry a file named txn at its own root, so falling through on an + // unreadable probe would let the tree answer for itself. + logf("remote: %s could not be checked for a work tree (%v); leaving it in place", staging, err) + return "", false } m, err := readMarker(staging) if err != nil { diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index 59b85780c..10b853a0a 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -3160,3 +3160,25 @@ func TestRemoveKeptBackupNeverTouchesTheScannedPrefix(t *testing.T) { t.Errorf("the copy must be left intact: %v", err) } } + +// A stat error on the work-tree probe is not proof there is no work tree. R1 +// puts the veto ahead of the marker precisely because a published tree can +// carry a file named txn at its root, so an unreadable probe has to retain +// rather than fall through to a marker the tree itself may have supplied. +func TestAttributeStagingDirRetainsWhenTheWorkTreeProbeFails(t *testing.T) { + dir := t.TempDir() + staging := filepath.Join(dir, ".staging-00000000000000000001-seq") + if err := os.MkdirAll(filepath.Join(staging, ".git"), 0o755); err != nil { + t.Fatal(err) + } + if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: "proj-1", Seq: 1}); err != nil { + t.Fatal(err) + } + injectFault(t, "stat", func(args ...string) bool { + return filepath.Base(args[0]) == ".git" + }, errors.New("injected work-tree probe failure")) + + if _, ok := attributeStagingDir(dir, staging, 1, func(string, ...any) {}); ok { + t.Fatal("a directory whose work-tree probe could not be read must not be attributed") + } +} From d45d82814335390e26768e62bbb0e23147ed9909 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Thu, 3 Sep 2026 00:25:12 -0500 Subject: [PATCH 31/34] fix(daemon/remote,dictation,cli): close the gaps a full review of the branch found A marker that could not be read was folded into "not this transaction's" at both sites, so recovery walked past the copy whose state it failed to establish and published an older one. The two sites scope that differently because their names carry different facts: a holder name carries its destination, so the dictation stop covers exactly that destination, while a staging name does not, so the bundle site stops only when the marker read cleanly during the scan and fails when it is re-read under the lock. An entry the scan could never attribute is retained and reported rather than blocking every link on the host, and nothing can delete it, because recovery only ever deletes candidates it grouped under a destination. destTxn.holds reported a lock it had already released, which is the fail-open direction for the guard every locked entry point consults. Release is now recorded atomically, since a handle can be released on one goroutine while another is still asking whether it is safe to act. The park was the one rename at the bundle site not going through the Windows retry, so a transient sharing violation deferred a retain by a whole restart. The dictation cleanup dropped its removeAll error, stranding a whole superseded copy where the bundle site reports it. The reclaim command was dispatched but missing from help and completions, which left the only path a retained copy has off the disk one an operator cannot find. --- internal/cli/app.go | 1 + internal/cli/completions.go | 1 + internal/cli/kept_backups.go | 5 ++- internal/cli/kept_backups_test.go | 20 ++++++++++ internal/daemon/remote/bundle.go | 39 +++++++++++-------- internal/daemon/remote/bundle_test.go | 34 ++++++++++++++++- internal/dictation/download.go | 46 +++++++++++++++++++---- internal/dictation/download_test.go | 54 ++++++++++++++++++++++++++- 8 files changed, 172 insertions(+), 28 deletions(-) diff --git a/internal/cli/app.go b/internal/cli/app.go index 103c5bb5d..86547aee6 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -1370,6 +1370,7 @@ Commands: usage Summarize token usage and estimated cost cron Schedule agent jobs (foreground, file-backed) repo-info Characterize the current repository (local git only) + kept-backups List and remove copies a recovery pass retained serve Run Zero protocol servers acp Serve the Agent Client Protocol over stdio (editor backend) help Show this help diff --git a/internal/cli/completions.go b/internal/cli/completions.go index 5f9058946..783f272a4 100644 --- a/internal/cli/completions.go +++ b/internal/cli/completions.go @@ -96,6 +96,7 @@ var completionRoot = completionNode{ {names: []string{"usage"}, children: leafNodes("report")}, {names: []string{"cron"}, children: leafNodes("add", "list", "rm", "pause", "resume", "run")}, {names: []string{"repo-info", "repoinfo"}}, + {names: []string{"kept-backups"}, children: leafNodes("list", "remove")}, {names: []string{"serve"}}, {names: []string{"acp"}}, {names: []string{"help"}}, diff --git a/internal/cli/kept_backups.go b/internal/cli/kept_backups.go index e68a3b025..0e1fff825 100644 --- a/internal/cli/kept_backups.go +++ b/internal/cli/kept_backups.go @@ -204,8 +204,9 @@ Recovery never deletes a copy it cannot prove was superseded; it moves that copy under a kept- name and leaves it there. Nothing reclaims one on its own, so this command is how retained copies leave the disk. -Without --bundle-dir both subcommands work on the dictation install root; with -it they work on that daemon bundle dir instead. Weigh the two differently: a +Without --bundle-dir both subcommands work on the dictation install root. With +it, remove works on that daemon bundle dir instead, and list adds the bundle dir +to the dictation listing, so each line names the site it came from. Weigh the two differently: a dictation kept backup is the only offline copy of an engine or a model, while a bundle kept backup is a work tree the client that sent it can upload again. diff --git a/internal/cli/kept_backups_test.go b/internal/cli/kept_backups_test.go index 450beacfc..dda7e2c5e 100644 --- a/internal/cli/kept_backups_test.go +++ b/internal/cli/kept_backups_test.go @@ -175,3 +175,23 @@ func TestKeptBackupsUsageNamesTheCostOfEachSite(t *testing.T) { } } } + +// The reclaim command is the only path a retained copy has off the disk, so a +// command that never appears in help or completions is a reclaim path that +// exists but cannot be found. +func TestKeptBackupsIsDiscoverable(t *testing.T) { + var out bytes.Buffer + if code := Run([]string{"--help"}, &out, &out); code != 0 { + t.Fatalf("--help exit = %d", code) + } + if !strings.Contains(out.String(), "kept-backups") { + t.Error("kept-backups is missing from the command list") + } + var comp bytes.Buffer + if code := Run([]string{"completions", "bash"}, &comp, &comp); code != 0 { + t.Fatalf("completions exit = %d", code) + } + if !strings.Contains(comp.String(), "kept-backups") { + t.Error("kept-backups is missing from the completion tree") + } +} diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index 1510a5ec7..c77da2d4b 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -418,7 +418,7 @@ func scanBundleDir(dir string, logf func(string, ...any)) map[string][]bundleCan logf("remote: %s does not carry a name this code writes; leaving it in place", staging) continue } - id, ok := attributeStagingDir(dir, staging, seq, logf) + id, ok, _ := attributeStagingDir(dir, staging, seq, logf) if !ok { continue } @@ -432,26 +432,30 @@ func scanBundleDir(dir string, logf func(string, ...any)) map[string][]bundleCan // because link ids starting with '.' used to be accepted, so a published work // tree can sit under the exact generated name AND carry a file called txn at its // root. Reading the marker first would let that file license a delete. -func attributeStagingDir(dir, staging string, seq int64, logf func(string, ...any)) (string, bool) { +func attributeStagingDir(dir, staging string, seq int64, logf func(string, ...any)) (id string, ok bool, unreadable bool) { if _, err := stagingFS.stat(filepath.Join(staging, ".git")); err == nil { logf("remote: %s holds a work tree, not a staged extract; leaving it in place", staging) - return "", false + return "", false, false } else if !errors.Is(err, fs.ErrNotExist) { // Not knowing whether this is a work tree is not the same as knowing it // is not one. The veto sits ahead of the marker because a published tree // can carry a file named txn at its own root, so falling through on an // unreadable probe would let the tree answer for itself. logf("remote: %s could not be checked for a work tree (%v); leaving it in place", staging, err) - return "", false + return "", false, true } m, err := readMarker(staging) if err != nil { logf("remote: %s has no usable transaction marker (%v); leaving it in place", staging, err) - return "", false + // A marker that is absent says this transaction never got far enough to + // claim the directory. A marker that could not be read says nothing at + // all, and treating the two alike lets an older copy be published over + // the one whose state is still unknown. + return "", false, !errors.Is(err, errMarkerMissing) } if m.Kind != txnKindBundleExtract { logf("remote: %s carries a %q transaction marker, which no extract wrote; leaving it in place", staging, m.Kind) - return "", false + return "", false, false } if m.Seq != seq { // The name is the authority: createSequencedStagingDir walks up from the @@ -459,21 +463,21 @@ func attributeStagingDir(dir, staging string, seq int64, logf func(string, ...an // marker that disagrees with the name proves nothing about who wrote // either of them. logf("remote: %s carries a marker for sequence %d but is named %d; leaving it in place", staging, m.Seq, seq) - return "", false + return "", false, false } // Everything read off disk goes back through the write path's own checks. A // marker is a file, and a file can be edited by anything that can reach the // bundle dir. - id, err := sanitizeLinkID(m.Dest) + id, err = sanitizeLinkID(m.Dest) if err != nil { logf("remote: staged tree in %s names an invalid link (%v); leaving it in place", staging, err) - return "", false + return "", false, false } if !withinDir(dir, filepath.Join(dir, id)) { logf("remote: staged tree in %s names a link outside the bundle dir; leaving it in place", staging) - return "", false + return "", false, false } - return id, true + return id, true, false } // candidateState is one candidate classified per the recovery design. empty, @@ -565,7 +569,10 @@ func classifyCandidates(dir, id string, cands []bundleCandidate, logf func(strin func classifyCandidate(dir string, c bundleCandidate, logf func(string, ...any)) (candidateState, candidateVerdict) { st := candidateState{bundleCandidate: c} - id, ok := attributeStagingDir(dir, c.path, c.seq, logf) + id, ok, unreadable := attributeStagingDir(dir, c.path, c.seq, logf) + if unreadable { + return st, verdictUnreadable + } if !ok || id != c.dest { return st, verdictDropped } @@ -929,7 +936,7 @@ func createSequencedStagingDir(dir string, n int64) (string, error) { // it is, which is the same fail-safe one pass later. func parkKeptBackup(dir, staging, id string, logf func(string, ...any)) { parked := filepath.Join(dir, keptPrefix+strings.TrimPrefix(filepath.Base(staging), stagingPrefix)) - if err := stagingFS.rename(staging, parked); err != nil { + if err := fsutil.RenameWithRetry(staging, parked, stagingFS.rename); err != nil { logf("remote: could not park the kept backup for %s at %s: %v", id, parked, err) } } @@ -983,7 +990,7 @@ func ListKeptBackups(dir string) ([]KeptBackup, error) { // The same proof recovery requires before it touches a directory: the // work-tree veto, then a marker agreeing with the name and naming a link // the write path would have accepted. - if id, ok := attributeStagingDir(dir, path, seq, discardLog); ok { + if id, ok, _ := attributeStagingDir(dir, path, seq, discardLog); ok { backup.Dest = id backup.Owned = true } @@ -1011,7 +1018,7 @@ func RemoveKeptBackup(dir, name string) error { return fmt.Errorf("remote: %s is not a kept backup; recovery may still act on it, so remove it by hand", name) } path := filepath.Join(dir, name) - id, ok := attributeStagingDir(dir, path, seq, discardLog) + id, ok, _ := attributeStagingDir(dir, path, seq, discardLog) if !ok { return fmt.Errorf("remote: nothing on disk attributes %s to a link; remove it by hand", name) } @@ -1032,7 +1039,7 @@ func RemoveKeptBackup(dir, name string) error { // Re-read under the lock. The attribution above ran with nothing excluding a // live extract, and a directory that stopped being this link's in between is // one that must not be deleted on the strength of the earlier reading. - if again, ok := attributeStagingDir(dir, path, seq, discardLog); !ok || again != id { + if again, ok, _ := attributeStagingDir(dir, path, seq, discardLog); !ok || again != id { return fmt.Errorf("remote: %s is no longer attributable to %s; leaving it in place", name, id) } return stagingFS.removeAll(path) diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index 10b853a0a..a2e0434e8 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -3178,7 +3178,39 @@ func TestAttributeStagingDirRetainsWhenTheWorkTreeProbeFails(t *testing.T) { return filepath.Base(args[0]) == ".git" }, errors.New("injected work-tree probe failure")) - if _, ok := attributeStagingDir(dir, staging, 1, func(string, ...any) {}); ok { + if _, ok, _ := attributeStagingDir(dir, staging, 1, func(string, ...any) {}); ok { t.Fatal("a directory whose work-tree probe could not be read must not be attributed") } } + +// A marker that read cleanly during the scan and fails when it is re-read under +// the lock stops that destination. The scan already named the destination, so +// the stop can be scoped to it; an entry whose marker never read at all carries +// no destination and is retained by the scan instead, since blocking every link +// on this host would cost more than the one wrong version it prevents. +func TestRecoverBundleDirStopsWhenAMarkerGoesUnreadableUnderTheLock(t *testing.T) { + dir := t.TempDir() + newest := stageBackup(t, dir, "", "proj-1", "v2", 2) + older := stageBackup(t, dir, "", "proj-1", "v1", 1) + + var scanned atomic.Bool + injectFault(t, "readFile", func(args ...string) bool { + if !strings.HasPrefix(args[0], newest) { + return false + } + // The scan reads every marker first; the re-read under the lock is the + // one this fault is for. + return !scanned.CompareAndSwap(false, true) + }, errors.New("injected marker read failure")) + + recoverBundleDir(dir, func(string, ...any) {}) + + if _, err := os.Lstat(filepath.Join(dir, "proj-1")); !os.IsNotExist(err) { + t.Fatalf("recovery published a copy while a candidate went unreadable under the lock: %v", err) + } + for _, staging := range []string{newest, older} { + if _, err := os.Stat(filepath.Join(staging, "backup", ".git")); err != nil { + t.Errorf("every copy must be retained until the unreadable one can be read: %v", err) + } + } +} diff --git a/internal/dictation/download.go b/internal/dictation/download.go index 01ded877a..76197b3a9 100644 --- a/internal/dictation/download.go +++ b/internal/dictation/download.go @@ -20,6 +20,7 @@ import ( "slices" "strconv" "strings" + "sync/atomic" "time" "github.com/Gitlawb/zero/internal/fsutil" @@ -862,6 +863,10 @@ type destTxn struct { root string dest string lock *lockutil.FileLock + // released is atomic because holds is the guard every locked entry point + // consults, and a handle can be released on one goroutine while another is + // still asking whether it is safe to act. + released atomic.Bool } // destDir is the destination this handle locks. Callers keep passing their own @@ -872,7 +877,7 @@ func (t *destTxn) destDir() string { return filepath.Join(t.root, t.dest) } // holds reports whether this handle actually locks destDir. A nil handle and a // handle for another destination both fail closed. func (t *destTxn) holds(destDir string) bool { - return t != nil && t.lock != nil && t.destDir() == destDir + return t != nil && t.lock != nil && !t.released.Load() && t.destDir() == destDir } // release drops the Install lock. Idempotent, so a caller can release early and @@ -881,6 +886,9 @@ func (t *destTxn) release() { if t == nil || t.lock == nil { return } + if t.released.Swap(true) { + return + } _ = t.lock.Release() } @@ -1145,7 +1153,7 @@ type holderCandidate struct { // with no such marker behind it, which is retained and reported, never moved. // An entry that only collides with the prefix is neither: it was never this // install's, so it is not even reported on its account. -func ownedHoldersBeside(destDir string) (owned []holderCandidate, unowned []string) { +func ownedHoldersBeside(destDir string) (owned []holderCandidate, unowned []string, unreadable string) { // ReadDir and a prefix rather than filepath.Glob: destDir is a real path, // and a '[' anywhere in it opens a character class to Glob, which then // matches nothing and strands the install this exists to put back. @@ -1153,7 +1161,7 @@ func ownedHoldersBeside(destDir string) (owned []holderCandidate, unowned []stri base := filepath.Base(destDir) entries, err := holderFS.readDir(parent) if err != nil { - return nil, nil + return nil, nil, "" } prefix := base + holderSuffix for _, entry := range entries { @@ -1168,6 +1176,14 @@ func ownedHoldersBeside(destDir string) (owned []holderCandidate, unowned []stri } m, err := readHolderMarker(path) if err != nil { + if !errors.Is(err, errMarkerMissing) { + // A marker this pass could not read is not a marker proving the + // holder is someone else's. Walking past it would let recovery + // install an older copy and let the next pass read that copy as + // proof this one was superseded, which is the provenance loss + // the no-fallback rule exists to prevent. + return nil, nil, path + } unowned = append(unowned, path) continue } @@ -1180,7 +1196,7 @@ func ownedHoldersBeside(destDir string) (owned []holderCandidate, unowned []stri } owned = append(owned, holderCandidate{path: path, seq: seq}) } - return owned, unowned + return owned, unowned, "" } // parkKeptHolder renames a holder under the Kept prefix, which is how a copy @@ -1543,7 +1559,11 @@ func restoreInterruptedPromotion(txn *destTxn, destDir string, published func(st // Only holders this code can prove it wrote for THIS destination. A sibling // that merely collides with the prefix is not a copy of this install and is // never restored from, removed, or reported on its account. - owned, unowned := ownedHoldersBeside(destDir) + owned, unowned, unreadable := ownedHoldersBeside(destDir) + if unreadable != "" { + report(fmt.Sprintf("dictation: leaving every retained copy for %s in place until %s can be read", filepath.Base(destDir), unreadable)) + return + } for _, path := range unowned { report(fmt.Sprintf("Keeping %s: nothing in it identifies it as a copy of %s", path, filepath.Base(destDir))) } @@ -1669,11 +1689,17 @@ func restoreInterruptedPromotion(txn *destTxn, destDir string, published func(st // promotion fails, so destDir is never left holding no install at all. If the // restore fails too, the set-aside copy is kept and named in the error. func promoteStagedDir(txn *destTxn, stageDir, destDir, label string, report func(string)) error { + // Normalized once so every later report is one call rather than a nil check + // a new one can forget. A caller with nowhere to report to still gets the + // same behavior; only the message goes nowhere. + if report == nil { + report = func(string) {} + } // Same reason recovery refuses: the window between the two renames has the // only copy of the install in a holder, and nothing else keeps a concurrent // recovery out of it. if !txn.holds(destDir) { - if report != nil { + { report(fmt.Sprintf("Refusing to install %s: no install lock is held for %s", label, filepath.Base(destDir))) } return fmt.Errorf("promoting staged %s: no install lock is held for %s", label, destDir) @@ -1682,7 +1708,13 @@ func promoteStagedDir(txn *destTxn, stageDir, destDir, label string, report func cleanupHolder := true defer func() { if holder != "" && cleanupHolder { - _ = holderFS.removeAll(holder) + // A failure here strands a whole copy of the previous install under + // a name recovery no longer enumerates, so say so rather than + // leaking it silently: the operator's reclaim command is the only + // thing that removes it afterwards. + if err := holderFS.removeAll(holder); err != nil { + report(fmt.Sprintf("dictation: could not remove the superseded %s copy in %s: %v", label, holder, err)) + } } }() diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index 39c73790d..dc3408b08 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -2041,7 +2041,7 @@ func TestOwnedHoldersBesideIgnoresAPrefixCollidingSibling(t *testing.T) { unmarked := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 3, holderSeqSuffix) makeDir(t, unmarked) - owned, unowned := ownedHoldersBeside(dest) + owned, unowned, _ := ownedHoldersBeside(dest) gotOwned := make([]string, 0, len(owned)) for _, c := range owned { @@ -2092,7 +2092,7 @@ func TestParkKeptHolderMovesTheCopyUnderTheKeptPrefix(t *testing.T) { if _, err := os.Stat(holder); !os.IsNotExist(err) { t.Errorf("the holder should be gone from the scanned prefix, got %v", err) } - owned, unowned := ownedHoldersBeside(dest) + owned, unowned, _ := ownedHoldersBeside(dest) if len(owned) != 0 || len(unowned) != 0 { t.Errorf("a parked copy must leave the scan: owned %v unowned %v", owned, unowned) } @@ -3001,3 +3001,53 @@ func TestRemoveKeptBackupNeverTouchesTheScannedPrefix(t *testing.T) { t.Errorf("the copy must be left intact: %v", err) } } + +// A marker this pass could not read is not a marker proving the holder belongs +// to someone else. Treating the two the same lets recovery walk past the copy +// whose state it failed to establish and install an older one, which is the +// provenance loss the whole no-fallback rule exists to prevent. +func TestRestoreInterruptedPromotionStopsWhenAMarkerCannotBeRead(t *testing.T) { + root := t.TempDir() + destDir := filepath.Join(root, "engine-a") + newest := plantHolder(t, destDir, 2, "v2", false) + older := plantHolder(t, destDir, 1, "v1", false) + txn := lockFor(t, destDir) + + injectFault(t, "readFile", func(args ...string) bool { + return strings.HasPrefix(args[0], newest) + }, errors.New("injected marker read failure")) + + var reported []string + restoreInterruptedPromotion(txn, destDir, testPublished, reporterFor(&reported)) + + if _, err := os.Lstat(destDir); !os.IsNotExist(err) { + t.Fatalf("recovery installed a copy while one holder's marker was unreadable: %v", err) + } + for _, holder := range []string{newest, older} { + if _, err := os.Stat(filepath.Join(holder, "install", "engine")); err != nil { + t.Errorf("every copy must be retained until the unreadable one can be read: %v", err) + } + } + if len(reported) == 0 { + t.Error("an unreadable marker must be reported") + } +} + +// A handle kept past its release must be refused. Reporting the lock as still +// held after it was given up is the fail-open direction: the guard exists so a +// call site that forgets the lifecycle is stopped, not trusted. +func TestDestTxnDoesNotClaimALockItReleased(t *testing.T) { + root := t.TempDir() + destDir := filepath.Join(root, "engine-a") + txn, err := lockDestination(context.Background(), root, "engine-a") + if err != nil { + t.Fatal(err) + } + if !txn.holds(destDir) { + t.Fatal("a fresh handle must hold its own destination") + } + txn.release() + if txn.holds(destDir) { + t.Error("a released handle must not report that it still holds the destination") + } +} From 90503c7a6f0aba700cca6571838670f926a44fff Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Thu, 3 Sep 2026 07:00:25 -0500 Subject: [PATCH 32/34] test(daemon/remote,dictation,cli): make the guards a mutation run walked past load-bearing A mutation pass over the suite deleted or inverted each guard in turn and recorded which ones nothing noticed. These are those, each now covered by a test observed red under the same mutation that exposed it. The reclaim command had no test below its two happy paths, so discarding its error handling entirely kept the package green: a refused removal would have printed a success line and exited zero, and the refusal is the contract its own usage text advertises. Both sites re-attribute a kept copy after winning the lock, and both could drop that re-read with nothing red, which is the window the guard exists for. The recovery paths could write their marker after the destructive rename rather than before it; only the write paths had that ordering pinned. Two tests could not distinguish the behavior they named. The dictation unreadable-holder test held one candidate, so stopping and running out of candidates looked identical, and it now plants an older usable copy beside it. The retry test selected the publish rename by call ordinal and passed against the first and second rename too, so it selects by argument like its neighbours. The standalone pass-two rows of the step-failure tables injected a fault the code never called, because pass one already reached the terminal state. Rows now assert their own fault fired, and the variant that could not fire one is gone rather than standing as coverage. --- internal/cli/kept_backups_test.go | 187 +++++++++++++++++++ internal/daemon/remote/bundle_matrix_test.go | 104 +++++++---- internal/daemon/remote/bundle_test.go | 162 +++++++++++++++- internal/dictation/download_matrix_test.go | 104 +++++++---- internal/dictation/download_test.go | 164 +++++++++++++++- 5 files changed, 636 insertions(+), 85 deletions(-) diff --git a/internal/cli/kept_backups_test.go b/internal/cli/kept_backups_test.go index dda7e2c5e..faca2cd47 100644 --- a/internal/cli/kept_backups_test.go +++ b/internal/cli/kept_backups_test.go @@ -195,3 +195,190 @@ func TestKeptBackupsIsDiscoverable(t *testing.T) { t.Error("kept-backups is missing from the completion tree") } } + +// The refusal and usage paths, over the command's own entry point. The two tests +// above only ever drive a removal that succeeds, which is why replacing the +// remove path's error handling with a discard left this package green: a refusal +// would have printed "Removed" and exited zero with nothing to say so. Every row +// here asserts the exit code, what did and did not reach stdout, and for the +// refusals that the copy is still on disk afterwards. +func TestKeptBackupsRefusalsAndUsage(t *testing.T) { + // plantUnowned writes a directory carrying the Kept grammar and no marker: + // recovery's residue, the entry the usage text promises remove refuses. + plantUnowned := func(t *testing.T, dir string, seq int64) string { + t.Helper() + name := fmt.Sprintf(".kept-%020d-seq", seq) + if err := os.MkdirAll(filepath.Join(dir, name, "backup"), 0o700); err != nil { + t.Fatal(err) + } + return name + } + + for _, tc := range []struct { + name string + // arrange plants whatever the row needs and returns the argument list. + arrange func(t *testing.T, sttRoot, bundleDir string) []string + wantCode int + wantOut string + // denyOut must not appear on stdout. "Removed" is the whole point on the + // refusal rows: a refusal that prints it is a removal as far as the + // operator can tell. + denyOut string + wantErr string + // check runs after the command, for the rows whose claim is on disk. + check func(t *testing.T, sttRoot, bundleDir string) + }{ + { + name: "remove refuses an unowned entry at the dictation root", + arrange: func(t *testing.T, sttRoot, _ string) []string { + return []string{"kept-backups", "remove", plantUnowned(t, sttRoot, 3)} + }, + wantCode: exitCrash, + denyOut: "Removed", + wantErr: "[zero]", + check: func(t *testing.T, sttRoot, _ string) { + if _, err := os.Stat(filepath.Join(sttRoot, fmt.Sprintf(".kept-%020d-seq", 3))); err != nil { + t.Errorf("a refused removal must leave the copy on disk: %v", err) + } + }, + }, + { + name: "remove refuses an unowned entry at the bundle dir", + arrange: func(t *testing.T, _, bundleDir string) []string { + return []string{"kept-backups", "remove", plantUnowned(t, bundleDir, 4), "--bundle-dir", bundleDir} + }, + wantCode: exitCrash, + denyOut: "Removed", + wantErr: "[zero]", + check: func(t *testing.T, _, bundleDir string) { + if _, err := os.Stat(filepath.Join(bundleDir, fmt.Sprintf(".kept-%020d-seq", 4))); err != nil { + t.Errorf("a refused removal must leave the copy on disk: %v", err) + } + }, + }, + { + // The flag can sit before the subcommand and can carry its value + // with an '=', and both forms have to reach the same site: an + // operator who names the bundle dir and is silently answered from + // the dictation root is being shown the wrong disk. + name: "remove lands on the bundle site through the equals form", + arrange: func(t *testing.T, sttRoot, bundleDir string) []string { + plantSTTKept(t, sttRoot, "engine-a", 1, "engine-bytes") + name := plantBundleKept(t, bundleDir, "proj-1", 7, "tree") + return []string{"kept-backups", "--bundle-dir=" + bundleDir, "remove", name} + }, + wantCode: exitSuccess, + wantOut: "Removed", + check: func(t *testing.T, sttRoot, bundleDir string) { + if _, err := os.Stat(filepath.Join(bundleDir, fmt.Sprintf(".kept-%020d-seq", 7))); !os.IsNotExist(err) { + t.Errorf("the bundle copy should be gone, got %v", err) + } + if _, err := os.Stat(filepath.Join(sttRoot, fmt.Sprintf("engine-a.kept-%020d-seq", 1))); err != nil { + t.Errorf("a removal at the bundle site must not touch the dictation root: %v", err) + } + }, + }, + { + name: "list lands on the bundle site through the equals form", + arrange: func(t *testing.T, _, bundleDir string) []string { + plantBundleKept(t, bundleDir, "proj-1", 7, "tree") + return []string{"kept-backups", "--bundle-dir=" + bundleDir, "list"} + }, + wantCode: exitSuccess, + wantOut: "bundle .kept-", + denyOut: "No kept backups.", + }, + { + name: "--bundle-dir with no value", + arrange: func(t *testing.T, _, _ string) []string { + return []string{"kept-backups", "list", "--bundle-dir"} + }, + wantCode: exitUsage, + wantErr: "needs a directory", + }, + { + name: "unknown subcommand", + arrange: func(t *testing.T, _, _ string) []string { + return []string{"kept-backups", "purge"} + }, + wantCode: exitUsage, + wantErr: "unknown subcommand", + }, + { + name: "no subcommand at all", + arrange: func(t *testing.T, _, _ string) []string { + return []string{"kept-backups"} + }, + wantCode: exitUsage, + wantErr: "Usage:", + }, + { + name: "list with a stray argument", + arrange: func(t *testing.T, _, _ string) []string { + return []string{"kept-backups", "list", "proj-1"} + }, + wantCode: exitUsage, + wantErr: "unexpected argument", + }, + { + name: "remove with two names", + arrange: func(t *testing.T, _, _ string) []string { + return []string{"kept-backups", "remove", "a", "b"} + }, + wantCode: exitUsage, + wantErr: "usage: zero kept-backups remove", + }, + { + // An empty root has to say so. Printing nothing at all reads as a + // command that did not run, and this listing is the only way a + // retained copy is found again. + name: "an empty root reports no backups", + arrange: func(t *testing.T, _, bundleDir string) []string { + return []string{"kept-backups", "list", "--bundle-dir", bundleDir} + }, + wantCode: exitSuccess, + wantOut: "No kept backups.", + }, + { + // The dictation root does not exist yet before the first install, + // which is a listing of nothing rather than an error. + name: "a dictation root that was never created reports no backups", + arrange: func(t *testing.T, sttRoot, _ string) []string { + if err := os.RemoveAll(sttRoot); err != nil { + t.Fatal(err) + } + return []string{"kept-backups", "list"} + }, + wantCode: exitSuccess, + wantOut: "No kept backups.", + }, + } { + t.Run(tc.name, func(t *testing.T) { + userConfigPath := filepath.Join(t.TempDir(), "zero", "config.json") + sttRoot := filepath.Join(filepath.Dir(userConfigPath), "stt") + if err := os.MkdirAll(sttRoot, 0o700); err != nil { + t.Fatal(err) + } + bundleDir := t.TempDir() + args := tc.arrange(t, sttRoot, bundleDir) + + var stdout, stderr bytes.Buffer + code := runWithDeps(args, &stdout, &stderr, keptDeps(userConfigPath)) + if code != tc.wantCode { + t.Errorf("exit = %d, want %d (stdout %q, stderr %q)", code, tc.wantCode, stdout.String(), stderr.String()) + } + if tc.wantOut != "" && !strings.Contains(stdout.String(), tc.wantOut) { + t.Errorf("stdout %q is missing %q", stdout.String(), tc.wantOut) + } + if tc.denyOut != "" && strings.Contains(stdout.String(), tc.denyOut) { + t.Errorf("stdout %q must not contain %q", stdout.String(), tc.denyOut) + } + if tc.wantErr != "" && !strings.Contains(stderr.String(), tc.wantErr) { + t.Errorf("stderr %q is missing %q", stderr.String(), tc.wantErr) + } + if tc.check != nil { + tc.check(t, sttRoot, bundleDir) + } + }) + } +} diff --git a/internal/daemon/remote/bundle_matrix_test.go b/internal/daemon/remote/bundle_matrix_test.go index b892fe6c2..21954cef8 100644 --- a/internal/daemon/remote/bundle_matrix_test.go +++ b/internal/daemon/remote/bundle_matrix_test.go @@ -9,6 +9,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" ) @@ -483,43 +484,60 @@ func TestDeleteWatchSeesOnlyTheDeleteRecoveryMayNotMake(t *testing.T) { // a change in how many times recovery stats a directory cannot silently move // which call the row is failing. type recoveryStep struct { - name string - inject func(t *testing.T, newest string) + name string + // inject installs the row's fault and returns whether that fault was ever + // actually called. The return is not bookkeeping: a row that injects a + // failure into a step the pass never reaches asserts only that a clean run + // works, which every other row already proves. + inject func(t *testing.T, newest string) (fired func() bool) +} + +// firedFault wraps a row's match so the row can report whether its fault was +// reached. A nil match means every call of the step, matching injectFault's own +// contract. +func firedFault(match func(args ...string) bool) (wrapped func(args ...string) bool, fired func() bool) { + var hit atomic.Bool + return func(args ...string) bool { + if match != nil && !match(args...) { + return false + } + hit.Store(true) + return true + }, func() bool { return hit.Load() } } func bundleRecoverySteps() []recoveryStep { failed := errors.New("injected recovery failure") + simple := func(step string) func(*testing.T, string) func() bool { + return func(t *testing.T, _ string) func() bool { + match, fired := firedFault(nil) + injectFault(t, step, match, failed) + return fired + } + } return []recoveryStep{ - { - name: "readDir", - inject: func(t *testing.T, _ string) { injectFault(t, "readDir", nil, failed) }, - }, - { - name: "stat", - inject: func(t *testing.T, _ string) { injectFault(t, "stat", nil, failed) }, - }, - { - name: "marker read", - inject: func(t *testing.T, _ string) { injectFault(t, "readFile", nil, failed) }, - }, + {name: "readDir", inject: simple("readDir")}, + {name: "stat", inject: simple("stat")}, + {name: "marker read", inject: simple("readFile")}, { name: "restore rename", - inject: func(t *testing.T, newest string) { - injectFault(t, "rename", func(args ...string) bool { + inject: func(t *testing.T, newest string) func() bool { + match, fired := firedFault(func(args ...string) bool { return args[0] == filepath.Join(newest, "backup") - }, failed) + }) + injectFault(t, "rename", match, failed) + return fired }, }, - { - name: "removeAll", - inject: func(t *testing.T, _ string) { injectFault(t, "removeAll", nil, failed) }, - }, + {name: "removeAll", inject: simple("removeAll")}, { name: "park rename", - inject: func(t *testing.T, _ string) { - injectFault(t, "rename", func(args ...string) bool { + inject: func(t *testing.T, _ string) func() bool { + match, fired := firedFault(func(args ...string) bool { return strings.HasPrefix(filepath.Base(args[1]), keptPrefix) - }, failed) + }) + injectFault(t, "rename", match, failed) + return fired }, }, } @@ -534,12 +552,17 @@ func bundleRecoverySteps() []recoveryStep { // about. func TestBundleRecoveryStepFailures(t *testing.T) { for _, step := range bundleRecoverySteps() { + // A standalone "pass two" variant used to sit between these two. It ran + // pass one clean, which reaches the terminal state, so pass two had no + // candidate left and the fault it injected was never called: the row + // asserted a clean run and nothing else. The both-passes row is the one + // that exercises a fault on pass two, because pass one fails first and + // leaves work behind for it. for _, when := range []struct { name string faulty [2]bool }{ {name: "pass one", faulty: [2]bool{true, false}}, - {name: "pass two", faulty: [2]bool{false, true}}, {name: "both passes", faulty: [2]bool{true, true}}, } { t.Run(step.name+"/"+when.name, func(t *testing.T) { @@ -555,25 +578,38 @@ func TestBundleRecoveryStepFailures(t *testing.T) { scratch := stageScratch(t, dir, "proj-1", 50) var watch deleteWatch - pass := func(faulty bool) { + pass := func(faulty bool) func() bool { saved := stagingFS defer func() { stagingFS = saved }() + fired := func() bool { return true } if faulty { - step.inject(t, newest) + fired = step.inject(t, newest) } // After the fault, so the watch sees the calls the fault // is failing rather than only the ones it lets through. watch.install(t) recoverBundleDir(dir, discardLog) + return fired + } + firedOne := pass(when.faulty[0]) + if when.faulty[0] { + if !firedOne() { + // The pass never called the step this row fails, so the + // row ran a clean recovery under a fault that does not + // exist and every assertion below it is vacuous. + t.Fatalf("the injected %s failure was never called on pass one, so this row proves nothing", step.name) + } + if bundleStepTerminal(dir, dest, older, newest, scratch) { + // Same defect from the other side: a fault that was + // called and changed nothing left the pass free to + // finish. + t.Errorf("the injected %s failure left the pass free to finish, so this row proves nothing", step.name) + } } - pass(when.faulty[0]) - if when.faulty[0] && bundleStepTerminal(dir, dest, older, newest, scratch) { - // The row would pass without the code being tested doing - // anything: a fault that changes nothing is a fault that was - // never reached, and every assertion below it is vacuous. - t.Errorf("the injected %s failure left the pass free to finish, so this row proves nothing", step.name) + firedTwo := pass(when.faulty[1]) + if when.faulty[1] && !firedTwo() { + t.Fatalf("the injected %s failure was never called on pass two, so this row proves nothing", step.name) } - pass(when.faulty[1]) // The fault is gone, and recovery has to be able to finish from // wherever the failed passes left the directory. pass(false) diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index a2e0434e8..c3892f24a 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -2,6 +2,7 @@ package remote import ( "context" + "encoding/json" "errors" "fmt" "math" @@ -642,10 +643,19 @@ func TestExtractBundleRenamesRetryOnTransientErrors(t *testing.T) { t.Fatalf("seed extract: %v", err) } - // The third rename of the second extract is the publish: the marker, then - // the set-aside, then the swap into dest. + // The publish is matched by its arguments, not by a call ordinal: the marker + // write and the set-aside also rename, and an ordinal leaves the test equally + // green when it lands on one of those instead, so nothing would pin the step. + // Only the first attempt fails, because the Windows retry needs a later one + // to succeed on. injected := &os.LinkError{Op: "rename", Old: "repo", New: dest, Err: syscall.Errno(32)} - injectFault(t, "rename", nthCall(3), injected) + var fired atomic.Bool + injectFault(t, "rename", func(args ...string) bool { + if args[1] != dest || filepath.Base(args[0]) != "repo" { + return false + } + return fired.CompareAndSwap(false, true) + }, injected) err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, nil) want := "v0" @@ -661,6 +671,9 @@ func TestExtractBundleRenamesRetryOnTransientErrors(t *testing.T) { if readErr != nil || string(got) != want { t.Fatalf("dest a.txt = %q, err %v, want %q", got, readErr, want) } + if !fired.Load() { + t.Fatal("the publish rename was never reached, so this row proves nothing about the retry") + } } // Missing and unreadable are opposite decisions for recovery: the first says the @@ -2489,19 +2502,26 @@ func TestRecoverBundleDirSkipsAnUnusableNewestCandidate(t *testing.T) { // pass the predicate" is a selection decision; a filesystem error is not a fact // about the copy at all, and acting on it turns a fault into a delete. func TestRecoverBundleDirStopsOnAnUnreadableCandidate(t *testing.T) { - if os.Geteuid() == 0 { - t.Skip("root ignores the directory permissions this test relies on") - } - if runtime.GOOS == "windows" { - t.Skip("POSIX directory permissions") - } for _, tc := range []struct { name string // break makes one stat under the staging dir fail with EACCES without // making the marker unreadable: a candidate whose marker cannot be read // lands in the unowned branch, where every mutation below passes. breakIt func(t *testing.T, staging string) + // portable rows run everywhere. The chmod rows do not, and this + // distinction is the axis the no-fallback rule rests on, so at least one + // row has to hold on the Windows leg and under root. + portable bool }{ + { + name: "the usability probe fails at the seam", + portable: true, + breakIt: func(t *testing.T, staging string) { + injectFault(t, "stat", func(args ...string) bool { + return args[0] == filepath.Join(staging, "backup", ".git") + }, os.ErrPermission) + }, + }, { name: "the set-aside copy cannot be stat'd", breakIt: func(t *testing.T, staging string) { @@ -2536,6 +2556,14 @@ func TestRecoverBundleDirStopsOnAnUnreadableCandidate(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { + if !tc.portable { + if os.Geteuid() == 0 { + t.Skip("root ignores the directory permissions this row relies on") + } + if runtime.GOOS == "windows" { + t.Skip("POSIX directory permissions") + } + } dir := t.TempDir() older := stageBackup(t, dir, "", "proj-1", "v1", 100) newest := stageBackup(t, dir, "", "proj-1", "v2", 200) @@ -2828,6 +2856,65 @@ func TestRecoverBundleDirLeavesAnUnusableDestinationWithNoCandidateAlone(t *test } } +// The marker goes in before the first destructive rename on recovery's own +// set-aside too. Only extractBundle's copy of this window was pinned; a husk +// moved into a staging dir before the marker lands is the only copy of whatever +// was at the destination, sitting in a directory nothing on disk attributes, +// which recovery can then neither restore from nor ever reclaim. +func TestSetAsideHuskWritesTheMarkerBeforeMovingTheDestination(t *testing.T) { + dir := t.TempDir() + // A destination that exists and cannot serve, plus a usable copy beside it: + // the one state that reaches the set-aside, since the husk moves only after + // a candidate has been selected. + dest := plantUnusableDest(t, dir, "proj-1", "partial") + candidate := stageBackup(t, dir, "", "proj-1", "v1", 100) + + injectFault(t, "rename", func(args ...string) bool { + return args[0] == dest + }, errors.New("injected set-aside failure")) + // The staging dir is reaped on this failure path, which is right and would + // also erase what this test is asserting on. + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + recoverBundleDir(dir, discardLog) + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + husk := "" + for _, entry := range entries { + path := filepath.Join(dir, entry.Name()) + if !strings.HasPrefix(entry.Name(), stagingPrefix) || path == candidate { + continue + } + if husk != "" { + t.Fatalf("want one staging dir beside the candidate, got %s and %s", husk, path) + } + husk = path + } + if husk == "" { + t.Fatal("the set-aside allocated no staging dir, so this test proves nothing about the order") + } + m, err := readMarker(husk) + if err != nil { + t.Fatalf("the staging dir was moved into before it carried a marker: %v", err) + } + seq, ok := stagingStamp(filepath.Base(husk)) + if !ok { + t.Fatalf("the allocator wrote a name the grammar rejects: %q", filepath.Base(husk)) + } + if m.Kind != txnKindBundleExtract || m.Dest != "proj-1" || m.Seq != seq { + t.Errorf("marker = %+v, want kind %q dest %q seq %d", m, txnKindBundleExtract, "proj-1", seq) + } + if _, err := os.Lstat(filepath.Join(husk, "backup")); !os.IsNotExist(err) { + t.Errorf("the set-aside failed, so nothing should have moved into the staging dir: %v", err) + } + if got, err := os.ReadFile(filepath.Join(dest, "husk.txt")); err != nil || string(got) != "partial" { + t.Errorf("the destination must be left exactly as found: %q err %v", got, err) + } +} + // A failed restore after the husk was set aside must leave the destination // exactly as it was found. Anything else is recovery that made the state worse // than the crash did. @@ -3123,6 +3210,63 @@ func TestRemoveKeptBackupRefusesWhileTheDestinationIsLocked(t *testing.T) { }) } +// The attribution runs twice: once before the lock and once under it. The first +// read happens with nothing excluding a live extract, so a directory that +// stopped being this link's in between must not be deleted on the strength of +// that reading. Both halves of the second check are driven here, because +// deleting the whole block leaves the package green. +func TestRemoveKeptBackupRefusesWhenAttributionChangesUnderTheLock(t *testing.T) { + t.Run("the marker goes unreadable", func(t *testing.T) { + dir := t.TempDir() + kept := plantKeptBackup(t, dir, "proj-1", "one", 1) + marker := filepath.Join(kept, stagingMarkerFile) + + var read atomic.Bool + injectFault(t, "readFile", func(args ...string) bool { + // The pre-lock attribution reads the marker first; the re-read under + // the lock is the one this fault is for. + return args[0] == marker && !read.CompareAndSwap(false, true) + }, errors.New("injected marker read failure")) + + err := RemoveKeptBackup(dir, filepath.Base(kept)) + if err == nil || !strings.Contains(err.Error(), "no longer attributable") { + t.Fatalf("RemoveKeptBackup = %v, want a refusal naming the attribution that changed", err) + } + if _, err := os.Stat(filepath.Join(kept, "backup", "a.txt")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) + t.Run("the marker names another link", func(t *testing.T) { + dir := t.TempDir() + kept := plantKeptBackup(t, dir, "proj-1", "one", 1) + marker := filepath.Join(kept, stagingMarkerFile) + // A readable marker for a different link is the other half: the re-read + // succeeds and answers with an id the first one did not, which is what a + // directory reused by another extract looks like from here. + other, err := json.Marshal(txnMarker{Kind: txnKindBundleExtract, Dest: "proj-2", Seq: 1}) + if err != nil { + t.Fatal(err) + } + + real := stagingFS + t.Cleanup(func() { stagingFS = real }) + var read atomic.Bool + stagingFS.readFile = func(name string) ([]byte, error) { + if name != marker || read.CompareAndSwap(false, true) { + return real.readFile(name) + } + return other, nil + } + + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err == nil || !strings.Contains(err.Error(), "no longer attributable") { + t.Fatalf("RemoveKeptBackup = %v, want a refusal naming the attribution that changed", err) + } + if _, err := os.Stat(filepath.Join(kept, "backup", "a.txt")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) +} + // The command takes a name, never a path. A separator in the argument would join // to a directory outside the bundle dir, which turns an operator's reclaim into // a remote rm, so the name is refused before any filesystem call happens at all. diff --git a/internal/dictation/download_matrix_test.go b/internal/dictation/download_matrix_test.go index 536fae9b6..327c29272 100644 --- a/internal/dictation/download_matrix_test.go +++ b/internal/dictation/download_matrix_test.go @@ -9,6 +9,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" "time" ) @@ -527,43 +528,60 @@ func keptNamed(destDir string, seq int64) string { // a change in how many times recovery stats a holder cannot silently move which // call the row is failing. type recoveryStep struct { - name string - inject func(t *testing.T, newest string) + name string + // inject installs the row's fault and returns whether that fault was ever + // actually called. The return is not bookkeeping: a row that injects a + // failure into a step the pass never reaches asserts only that a clean run + // works, which every other row already proves. + inject func(t *testing.T, newest string) (fired func() bool) +} + +// firedFault wraps a row's match so the row can report whether its fault was +// reached. A nil match means every call of the step, matching injectFault's own +// contract. +func firedFault(match func(args ...string) bool) (wrapped func(args ...string) bool, fired func() bool) { + var hit atomic.Bool + return func(args ...string) bool { + if match != nil && !match(args...) { + return false + } + hit.Store(true) + return true + }, func() bool { return hit.Load() } } func dictationRecoverySteps() []recoveryStep { failed := errors.New("injected recovery failure") + simple := func(step string) func(*testing.T, string) func() bool { + return func(t *testing.T, _ string) func() bool { + match, fired := firedFault(nil) + injectFault(t, step, match, failed) + return fired + } + } return []recoveryStep{ - { - name: "readDir", - inject: func(t *testing.T, _ string) { injectFault(t, "readDir", nil, failed) }, - }, - { - name: "stat", - inject: func(t *testing.T, _ string) { injectFault(t, "stat", nil, failed) }, - }, - { - name: "marker read", - inject: func(t *testing.T, _ string) { injectFault(t, "readFile", nil, failed) }, - }, + {name: "readDir", inject: simple("readDir")}, + {name: "stat", inject: simple("stat")}, + {name: "marker read", inject: simple("readFile")}, { name: "restore rename", - inject: func(t *testing.T, newest string) { - injectFault(t, "rename", func(args ...string) bool { + inject: func(t *testing.T, newest string) func() bool { + match, fired := firedFault(func(args ...string) bool { return args[0] == filepath.Join(newest, "install") - }, failed) + }) + injectFault(t, "rename", match, failed) + return fired }, }, - { - name: "removeAll", - inject: func(t *testing.T, _ string) { injectFault(t, "removeAll", nil, failed) }, - }, + {name: "removeAll", inject: simple("removeAll")}, { name: "park rename", - inject: func(t *testing.T, _ string) { - injectFault(t, "rename", func(args ...string) bool { + inject: func(t *testing.T, _ string) func() bool { + match, fired := firedFault(func(args ...string) bool { return strings.Contains(filepath.Base(args[1]), keptSuffix) - }, failed) + }) + injectFault(t, "rename", match, failed) + return fired }, }, } @@ -578,12 +596,17 @@ func dictationRecoverySteps() []recoveryStep { // about. func TestDictationRecoveryStepFailures(t *testing.T) { for _, step := range dictationRecoverySteps() { + // A standalone "pass two" variant used to sit between these two. It ran + // pass one clean, which reaches the terminal state, so pass two had no + // candidate left and the fault it injected was never called: the row + // asserted a clean run and nothing else. The both-passes row is the one + // that exercises a fault on pass two, because pass one fails first and + // leaves work behind for it. for _, when := range []struct { name string faulty [2]bool }{ {name: "pass one", faulty: [2]bool{true, false}}, - {name: "pass two", faulty: [2]bool{false, true}}, {name: "both passes", faulty: [2]bool{true, true}}, } { t.Run(step.name+"/"+when.name, func(t *testing.T) { @@ -601,25 +624,38 @@ func TestDictationRecoveryStepFailures(t *testing.T) { txn := lockFor(t, dest) var watch deleteWatch - pass := func(faulty bool) { + pass := func(faulty bool) func() bool { saved := holderFS defer func() { holderFS = saved }() + fired := func() bool { return true } if faulty { - step.inject(t, newest) + fired = step.inject(t, newest) } // After the fault, so the watch sees the calls the fault is // failing rather than only the ones it lets through. watch.install(t) restoreInterruptedPromotion(txn, dest, testPublished, nil) + return fired + } + firedOne := pass(when.faulty[0]) + if when.faulty[0] { + if !firedOne() { + // The pass never called the step this row fails, so the + // row ran a clean recovery under a fault that does not + // exist and every assertion below it is vacuous. + t.Fatalf("the injected %s failure was never called on pass one, so this row proves nothing", step.name) + } + if dictationStepTerminal(dest, older, newest, scratch) { + // Same defect from the other side: a fault that was + // called and changed nothing left the pass free to + // finish. + t.Errorf("the injected %s failure left the pass free to finish, so this row proves nothing", step.name) + } } - pass(when.faulty[0]) - if when.faulty[0] && dictationStepTerminal(dest, older, newest, scratch) { - // The row would pass without the code being tested doing - // anything: a fault that changes nothing is a fault that was - // never reached, and every assertion below it is vacuous. - t.Errorf("the injected %s failure left the pass free to finish, so this row proves nothing", step.name) + firedTwo := pass(when.faulty[1]) + if when.faulty[1] && !firedTwo() { + t.Fatalf("the injected %s failure was never called on pass two, so this row proves nothing", step.name) } - pass(when.faulty[1]) // The fault is gone, and recovery has to be able to finish from // wherever the failed passes left the destination. pass(false) diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index dc3408b08..087582d63 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -2185,6 +2185,65 @@ func TestPromoteStagedDirWritesTheMarkerBeforeSettingAside(t *testing.T) { } } +// The same ordering on recovery's own set-aside. Only promoteStagedDir's copy of +// this window was pinned, and the recovery path writes its own: a husk moved +// into a holder before the marker lands is the only copy of whatever was at the +// destination, sitting in a directory nothing on disk attributes, which recovery +// can then neither restore from nor ever reclaim. +func TestSetAsideUnusableDestWritesTheMarkerBeforeMovingTheHusk(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + // A destination that exists and holds no usable install, plus a usable copy + // beside it: the one state that reaches the set-aside, since the husk moves + // only after a candidate has been selected. + if err := os.MkdirAll(dest, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dest, "husk"), []byte("partial"), 0o644); err != nil { + t.Fatal(err) + } + candidate := plantHolder(t, dest, 1, "the copy", false) + + injectFault(t, "rename", func(args ...string) bool { + return args[0] == dest + }, errors.New("injected set-aside failure")) + // The holder is removed on this failure path, which is right and would also + // erase what this test is asserting on. + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) + + var husk string + for _, h := range holdersFor(t, dest) { + if h != candidate { + if husk != "" { + t.Fatalf("want one holder beside the candidate, got %v", holdersFor(t, dest)) + } + husk = h + } + } + if husk == "" { + t.Fatal("the set-aside allocated no holder, so this test proves nothing about the order") + } + m, err := readHolderMarker(husk) + if err != nil { + t.Fatalf("the holder was moved into before it carried a marker: %v", err) + } + seq, ok := holderStamp(dest, husk) + if !ok { + t.Fatalf("the allocator wrote a name the grammar rejects: %q", filepath.Base(husk)) + } + if m.Kind != holderMarkerKind || m.Dest != filepath.Base(dest) || m.Seq != seq { + t.Errorf("marker = %+v, want kind %q dest %q seq %d", m, holderMarkerKind, filepath.Base(dest), seq) + } + if hasFile(t, filepath.Join(husk, "install")) { + t.Error("the set-aside failed, so nothing should have moved into the holder") + } + if got, err := os.ReadFile(filepath.Join(dest, "husk")); err != nil || string(got) != "partial" { + t.Errorf("the destination must be left exactly as found: %q err %v", got, err) + } +} + // A torn marker write must leave no marker at all: the atomic rename is what // makes "missing" and "ours" the only two answers a crash can produce, so a // partial file can never be read as ownership. @@ -2649,12 +2708,6 @@ func TestRestoreInterruptedPromotionIgnoresAPrefixCollidingSibling(t *testing.T) // anything on it would turn a transient error into a permanent ruling. So a // candidate that cannot be read stops the destination with everything intact. func TestRestoreInterruptedPromotionStopsOnAnUnreadableHolder(t *testing.T) { - if os.Geteuid() == 0 { - t.Skip("root ignores the directory permissions this test relies on") - } - if runtime.GOOS == "windows" { - t.Skip("POSIX directory permissions") - } for _, tc := range []struct { name string // seed makes the holder's install unreadable without touching the holder @@ -2662,7 +2715,20 @@ func TestRestoreInterruptedPromotionStopsOnAnUnreadableHolder(t *testing.T) { // directory is classified unowned, and the branch under test is never // reached. seed func(t *testing.T, holder string) + // portable rows run everywhere. The chmod rows do not, and this + // distinction is the axis the no-fallback rule rests on, so at least one + // row has to hold on the Windows leg and under root. + portable bool }{ + { + name: "the install cannot be listed at the seam", + portable: true, + seed: func(t *testing.T, holder string) { + injectFault(t, "readDir", func(args ...string) bool { + return args[0] == filepath.Join(holder, "install") + }, os.ErrPermission) + }, + }, { name: "install is a symlink through an unreadable directory", seed: func(t *testing.T, holder string) { @@ -2693,9 +2759,24 @@ func TestRestoreInterruptedPromotionStopsOnAnUnreadableHolder(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { + if !tc.portable { + if os.Geteuid() == 0 { + t.Skip("root ignores the directory permissions this row relies on") + } + if runtime.GOOS == "windows" { + t.Skip("POSIX directory permissions") + } + } root := t.TempDir() dest := filepath.Join(root, "engine-1.2.3-linux-x64") - holder := plantHolder(t, dest, 1, "the only copy", false) + // A second usable owned holder sits below the unreadable one, and it + // is what makes "stopped" distinguishable from "ran out of + // candidates": with the unreadable holder as the only candidate, + // turning the stop into a continue leaves every assertion below + // true, because there is nothing left for the pass to publish or + // park either way. + older := plantHolder(t, dest, 1, "the older copy", false) + holder := plantHolder(t, dest, 2, "the only copy", false) tc.seed(t, holder) var reported []string @@ -2716,7 +2797,16 @@ func TestRestoreInterruptedPromotionStopsOnAnUnreadableHolder(t *testing.T) { if slices.Contains(*removed, holder) { t.Errorf("a copy that could not be read must never be handed to a delete: %v", *removed) } - assertNoOtherEntries(t, root, filepath.Base(holder), installLockDir) + // The older copy is the claim the fixture exists for: a pass that + // carried on past the unreadable holder would publish this one, or + // park it on the way, and either is a ruling made on a fault. + if got, err := os.ReadFile(filepath.Join(older, "install", "engine")); err != nil || string(got) != "the older copy" { + t.Errorf("the older copy must be left exactly as found: %q err %v", got, err) + } + if slices.Contains(*removed, older) { + t.Errorf("the older copy must never be handed to a delete: %v", *removed) + } + assertNoOtherEntries(t, root, filepath.Base(older), filepath.Base(holder), installLockDir) assertReports(t, reported, holder) }) } @@ -2959,6 +3049,64 @@ func TestRemoveKeptBackupRefusesWhileTheDestinationIsLocked(t *testing.T) { } } +// The attribution runs twice: once before the lock and once under it. The first +// check happens with nothing excluding a live promotion, so a directory that +// stopped being this install's in between must not be deleted on the strength of +// that reading. Both halves of the second check are driven here, because +// deleting the whole block leaves the package green. +func TestRemoveKeptBackupRefusesWhenAttributionChangesUnderTheLock(t *testing.T) { + t.Run("the marker goes unreadable", func(t *testing.T) { + root := t.TempDir() + kept := plantKeptHolder(t, filepath.Join(root, "engine-a"), 1, "engine") + marker := filepath.Join(kept, holderMarkerFile) + + var read atomic.Bool + injectFault(t, "readFile", func(args ...string) bool { + // The pre-lock check reads the marker first; the re-read under the + // lock is the one this fault is for. + return args[0] == marker && !read.CompareAndSwap(false, true) + }, errors.New("injected marker read failure")) + + err := RemoveKeptBackup(root, filepath.Base(kept)) + if err == nil || !strings.Contains(err.Error(), "no longer attributable") { + t.Fatalf("RemoveKeptBackup = %v, want a refusal naming the attribution that changed", err) + } + if _, err := os.Stat(filepath.Join(kept, "install", "engine")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) + t.Run("the marker names another install", func(t *testing.T) { + root := t.TempDir() + kept := plantKeptHolder(t, filepath.Join(root, "engine-a"), 1, "engine") + marker := filepath.Join(kept, holderMarkerFile) + // A readable marker for a different install is the other half: the + // re-read succeeds and answers with a destination the first one did not, + // which is what a directory reused by another promotion looks like from + // here. + other, err := json.Marshal(txnMarker{Kind: holderMarkerKind, Dest: "engine-z", Seq: 1}) + if err != nil { + t.Fatal(err) + } + + real := holderFS + t.Cleanup(func() { holderFS = real }) + var read atomic.Bool + holderFS.readFile = func(name string) ([]byte, error) { + if name != marker || read.CompareAndSwap(false, true) { + return real.readFile(name) + } + return other, nil + } + + if err := RemoveKeptBackup(root, filepath.Base(kept)); err == nil || !strings.Contains(err.Error(), "no longer attributable") { + t.Fatalf("RemoveKeptBackup = %v, want a refusal naming the attribution that changed", err) + } + if _, err := os.Stat(filepath.Join(kept, "install", "engine")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) +} + // The command takes a name, never a path. The Kept grammar here is a // destination's own name followed by the suffix, so a name carrying a separator // still passes it: "../x.kept-...-seq" parses as destination "../x". Only the From 29fe45528300e5cc564ec9e832e089e78b1b4161 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Thu, 3 Sep 2026 07:05:23 -0500 Subject: [PATCH 33/34] fix(daemon/remote): bound the wait for an extract already running in this process Uploads for one link serialize on two locks, and only the second carried a budget. The first is a plain mutex taken before it, so a queued upload waited on it with no bound and no cancellation: the connection deadline is cleared before the bundle is handled, and the extract's own timeout starts after both locks are held, deliberately, so a queued upload does not spend its clone budget waiting. Nothing was left to end the wait. An authenticated client sending repeatedly to one link id could therefore pin every connection slot the bridge has, each one waiting on a mutex that only the extract ahead of it releases. Before this branch the whole extract ran under a single deadline, so the wait ended even though it burned the clone's budget; splitting the budget out removed the bound along with it. The wait now polls the same try-lock recovery uses, against the same budget the per-link file lock waits out, and honors the caller's context. --- internal/daemon/remote/bundle.go | 28 ++++++++++++++++++++++++++- internal/daemon/remote/bundle_test.go | 25 ++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index c77da2d4b..b6e9343a7 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -271,6 +271,29 @@ func lockExtract(dest string) func() { } } +// waitForExtract takes the in-process lock under the same budget the per-link +// file lock waits out, so a queued upload gives the connection slot back rather +// than holding it for as long as the extract ahead of it runs. A bare Lock here +// is the one step in the whole extract with no bound: it is taken before the +// file lock, so its wait is not covered by that one's deadline. +func waitForExtract(ctx context.Context, dest string) (func(), error) { + deadline := time.NewTimer(gitTimeout) + defer deadline.Stop() + for { + release, held := tryLockExtract(dest) + if !held { + return release, nil + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-deadline.C: + return nil, fmt.Errorf("remote: timed out waiting for an extract in this process to finish with %s", dest) + case <-time.After(extractLockPoll): + } + } +} + // tryLockExtract takes the in-process lock without waiting, reporting held when // a goroutine in this daemon owns dest. Recovery cannot use lockExtract: an // extract holds its destination across the whole clone, so a recovering pass @@ -1162,7 +1185,10 @@ func extractBundle(ctx context.Context, bundleFile, dest string, logf func(strin if err := os.MkdirAll(parent, 0o700); err != nil { return err } - unlock := lockExtract(dest) + unlock, err := waitForExtract(ctx, dest) + if err != nil { + return err + } defer unlock() unlockFile, err := lockExtractFile(ctx, parent, dest) if err != nil { diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index c3892f24a..fbb1a9eab 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -3358,3 +3358,28 @@ func TestRecoverBundleDirStopsWhenAMarkerGoesUnreadableUnderTheLock(t *testing.T } } } + +// The in-process lock is taken before the one carrying a budget, so an upload +// queued behind a live extract used to wait on a bare mutex with no bound at +// all. A client that keeps sending to one link id then pins a connection slot +// per upload and nothing ever gives the slot back. +func TestExtractBundleGivesUpWaitingForTheInProcessLock(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + release := lockExtract(dest) + defer release() + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + done := make(chan error, 1) + go func() { done <- extractBundle(ctx, filepath.Join(dir, "absent.bundle"), dest, nil) }() + + select { + case err := <-done: + if err == nil { + t.Fatal("a queued extract must not report success while another holds the link") + } + case <-time.After(5 * time.Second): + t.Fatal("a queued extract waited on the in-process lock with no bound") + } +} From 942710a9903a7a522f572629d9829c4c645ceb0d Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:25:59 -0500 Subject: [PATCH 34/34] fix(daemon/remote,dictation,cli): unwrap not-exist, refuse aliasing link ids, fail a cut-short listing Three findings from the branch review that were still open. Every not-exist probe at both sites ran through os.IsNotExist, which does not unwrap. The probes read seam results, and a seam hands back a wrapped error, so a wrapped ENOENT took the opposite branch from the same error unwrapped: a marked copy holding nothing read as a copy that could not be read, and stopped a destination with nothing wrong with it. They use errors.Is now. Windows drops a trailing dot when it resolves a path and maps the reserved device names onto devices, so two link ids differing only there name one directory on that platform. Both extract locks key on the id rather than the resolved path, so such a pair would take two different locks and clone into one tree at the same time. Those ids are refused. Surrounding space is not the same problem and is still trimmed: both spellings become one id before any path is built, so they take one lock. The reclaim listing broke out of its loop on a write error and still exited zero. It is the only place a retained copy is visible, so an operator reading a clean exit believed they had seen everything that exists. --- internal/cli/kept_backups.go | 26 ++++++++++----- internal/cli/kept_backups_test.go | 20 ++++++++++++ internal/daemon/remote/bundle.go | 45 ++++++++++++++++++++----- internal/daemon/remote/bundle_test.go | 47 +++++++++++++++++++++++++++ internal/dictation/download.go | 2 +- 5 files changed, 123 insertions(+), 17 deletions(-) diff --git a/internal/cli/kept_backups.go b/internal/cli/kept_backups.go index 0e1fff825..92799d5b0 100644 --- a/internal/cli/kept_backups.go +++ b/internal/cli/kept_backups.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" "io" - "os" + "io/fs" "path/filepath" "github.com/Gitlawb/zero/internal/daemon/remote" @@ -101,16 +101,23 @@ func keptBackupsList(args []string, bundleDir string, stdout io.Writer, stderr i return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash) } sttBackups, err := dictation.ListKeptBackups(root) - if err != nil && !os.IsNotExist(err) { + if err != nil && !errors.Is(err, fs.ErrNotExist) { return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash) } - found := writeKeptBackups(stdout, "stt", sttKeptLines(sttBackups)) + found, err := writeKeptBackups(stdout, "stt", sttKeptLines(sttBackups)) + if err != nil { + return exitCrash + } if bundleDir != "" { bundleBackups, err := remote.ListKeptBackups(bundleDir) - if err != nil && !os.IsNotExist(err) { + if err != nil && !errors.Is(err, fs.ErrNotExist) { return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash) } - found = writeKeptBackups(stdout, "bundle", bundleKeptLines(bundleBackups)) || found + bundleFound, err := writeKeptBackups(stdout, "bundle", bundleKeptLines(bundleBackups)) + if err != nil { + return exitCrash + } + found = bundleFound || found } if !found { if _, err := fmt.Fprintln(stdout, "No kept backups."); err != nil { @@ -151,7 +158,7 @@ func bundleKeptLines(backups []remote.KeptBackup) []keptBackupLine { // the site because it is exactly what `remove` takes; the destination is the // install or link the copy was set aside for, and an entry nothing on disk // attributes says so instead of borrowing a destination from its own name. -func writeKeptBackups(stdout io.Writer, site string, lines []keptBackupLine) bool { +func writeKeptBackups(stdout io.Writer, site string, lines []keptBackupLine) (bool, error) { for _, line := range lines { dest := line.dest if dest == "" { @@ -162,10 +169,13 @@ func writeKeptBackups(stdout io.Writer, site string, lines []keptBackupLine) boo suffix = " unowned" } if _, err := fmt.Fprintf(stdout, "%s %s dest=%s seq=%d bytes=%d%s\n", site, line.name, dest, line.seq, line.bytes, suffix); err != nil { - break + // A listing cut short is not a listing. Reporting success here tells + // an operator they have seen every retained copy when they have not, + // and this command is the only place those copies are visible. + return false, err } } - return len(lines) > 0 + return len(lines) > 0, nil } func keptBackupsRemove(args []string, bundleDir string, stdout io.Writer, stderr io.Writer, deps appDeps) int { diff --git a/internal/cli/kept_backups_test.go b/internal/cli/kept_backups_test.go index faca2cd47..089545d35 100644 --- a/internal/cli/kept_backups_test.go +++ b/internal/cli/kept_backups_test.go @@ -3,6 +3,7 @@ package cli import ( "bytes" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -382,3 +383,22 @@ func TestKeptBackupsRefusalsAndUsage(t *testing.T) { }) } } + +// The listing is the only place a retained copy is visible, so a listing cut +// short must not report success: an operator who sees a clean exit believes +// they have seen every copy that exists. +func TestKeptBackupsListFailsOnAShortWrite(t *testing.T) { + userConfigPath := filepath.Join(t.TempDir(), "config.json") + root := filepath.Join(filepath.Dir(userConfigPath), "stt") + plantSTTKept(t, root, "engine-a", 1, "the only offline copy") + + var stderr bytes.Buffer + if code := runWithDeps([]string{"kept-backups", "list"}, shortWriter{}, &stderr, keptDeps(userConfigPath)); code == exitSuccess { + t.Error("a listing that could not be written must not exit successfully") + } +} + +// shortWriter fails every write, standing in for a closed pipe or a full disk. +type shortWriter struct{} + +func (shortWriter) Write([]byte) (int, error) { return 0, errors.New("short write") } diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index b6e9343a7..0b7bdc838 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -425,7 +425,7 @@ type bundleCandidate struct { func scanBundleDir(dir string, logf func(string, ...any)) map[string][]bundleCandidate { entries, err := stagingFS.readDir(dir) if err != nil { - if !os.IsNotExist(err) { + if !errors.Is(err, fs.ErrNotExist) { logf("remote: could not scan bundle dir %s: %v", dir, err) } return nil @@ -602,7 +602,7 @@ func classifyCandidate(dir string, c bundleCandidate, logf func(string, ...any)) backup := filepath.Join(c.path, "backup") switch _, err := stagingFS.stat(backup); { case err == nil: - case os.IsNotExist(err): + case errors.Is(err, fs.ErrNotExist): // A readable marker with no set-aside content is the one shape that // proves the directory holds nothing: owned, and empty. st.empty = true @@ -614,7 +614,7 @@ func classifyCandidate(dir string, c bundleCandidate, logf func(string, ...any)) switch _, err := stagingFS.stat(filepath.Join(c.path, committedFile)); { case err == nil: st.committed = true - case os.IsNotExist(err): + case errors.Is(err, fs.ErrNotExist): default: logf("remote: could not read the commit flag in %s: %v", c.path, err) return st, verdictUnreadable @@ -622,7 +622,7 @@ func classifyCandidate(dir string, c bundleCandidate, logf func(string, ...any)) switch _, err := stagingFS.stat(filepath.Join(backup, ".git")); { case err == nil: st.usable = true - case os.IsNotExist(err): + case errors.Is(err, fs.ErrNotExist): // Durable: this copy will never pass the predicate, so it is skipped in // selection and kept. That is a different fact from the error below. default: @@ -640,7 +640,7 @@ func classifyCandidate(dir string, c bundleCandidate, logf func(string, ...any)) func classifyDest(dest string, logf func(string, ...any)) (present, usable, ok bool) { switch _, err := stagingFS.stat(dest); { case err == nil: - case os.IsNotExist(err): + case errors.Is(err, fs.ErrNotExist): return false, false, true default: logf("remote: could not read %s: %v; leaving its staged copies in place", dest, err) @@ -649,7 +649,7 @@ func classifyDest(dest string, logf func(string, ...any)) (present, usable, ok b switch _, err := stagingFS.stat(filepath.Join(dest, ".git")); { case err == nil: return true, true, true - case os.IsNotExist(err): + case errors.Is(err, fs.ErrNotExist): return true, false, true default: logf("remote: could not tell whether %s is usable: %v; leaving its staged copies in place", dest, err) @@ -1151,7 +1151,7 @@ func writeMarker(dir string, m txnMarker) error { func readMarker(dir string) (txnMarker, error) { raw, err := stagingFS.readFile(filepath.Join(dir, stagingMarkerFile)) if err != nil { - if os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return txnMarker{}, errMarkerMissing } return txnMarker{}, fmt.Errorf("read transaction marker in %s: %w", dir, err) @@ -1248,7 +1248,7 @@ func extractBundle(ctx context.Context, bundleFile, dest string, logf func(strin restore := func() error { return nil } if err := fsutil.RenameWithRetry(dest, backup, stagingFS.rename); err == nil { restore = func() error { return fsutil.RenameWithRetry(backup, dest, stagingFS.rename) } - } else if !os.IsNotExist(err) { + } else if !errors.Is(err, fs.ErrNotExist) { return err } if err := fsutil.RenameWithRetry(cloneDest, dest, stagingFS.rename); err != nil { @@ -1420,6 +1420,17 @@ func sanitizeLinkID(id string) (string, error) { if strings.HasPrefix(id, ".") { return "", errors.New("remote: link id may not start with '.'") } + // Windows drops a trailing dot or space when it resolves a path, and maps + // the reserved device names onto devices rather than directory entries. An + // id that is not one distinct directory on every platform breaks the thing + // both extract locks assume: they key on the id, so two ids that resolve to + // one tree take two different locks and clone into it at the same time. + if strings.HasSuffix(id, ".") { + return "", errors.New("remote: link id may not end with '.'") + } + if isReservedDeviceName(id) { + return "", errors.New("remote: link id may not be a reserved device name") + } for _, r := range id { switch { case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_', r == '.': @@ -1460,3 +1471,21 @@ func firstLine(s string) string { } return s } + +// isReservedDeviceName reports whether name is one of the Windows device names, +// which resolve to a device on that platform whatever directory they sit in. +// The check is case-insensitive and ignores an extension, because CON, con and +// AUX.txt all resolve the same way. +func isReservedDeviceName(name string) bool { + if i := strings.IndexByte(name, '.'); i >= 0 { + name = name[:i] + } + switch strings.ToUpper(name) { + case "CON", "PRN", "AUX", "NUL": + return true + } + if len(name) == 4 && (strings.EqualFold(name[:3], "COM") || strings.EqualFold(name[:3], "LPT")) { + return name[3] >= '1' && name[3] <= '9' + } + return false +} diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index fbb1a9eab..0704f8401 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "math" "os" "os/exec" @@ -3383,3 +3384,49 @@ func TestExtractBundleGivesUpWaitingForTheInProcessLock(t *testing.T) { t.Fatal("a queued extract waited on the in-process lock with no bound") } } + +// Every not-exist probe in this file runs on a seam result, and a seam can hand +// back a wrapped error. os.IsNotExist does not unwrap, so a wrapped ENOENT took +// the opposite branch from the one the same syscall takes unwrapped: the +// classifier read "no backup here" as "this copy cannot be read" and stopped a +// destination that had nothing wrong with it. +func TestClassifierTreatsAWrappedNotExistAsAbsent(t *testing.T) { + dir := t.TempDir() + staging := stageBackup(t, dir, "", "proj-1", "v1", 1) + if err := os.RemoveAll(filepath.Join(staging, "backup")); err != nil { + t.Fatal(err) + } + injectFault(t, "stat", func(args ...string) bool { + return args[0] == filepath.Join(staging, "backup") + }, fmt.Errorf("seam wrapped: %w", fs.ErrNotExist)) + + recoverBundleDir(dir, func(string, ...any) {}) + + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Errorf("a marked copy holding nothing is owned-and-empty and should be reaped, got %v", err) + } +} + +// Windows strips a trailing dot and a trailing space when it resolves a path, +// so two link ids that differ only there name one directory on that platform. +// The two extract locks key on the id, not on the resolved path, so the pair +// would take different locks and clone into the same tree at the same time. +// Reserved device names resolve away from the directory entirely. +func TestSanitizeLinkIDRefusesNamesThatAreNotDistinctOnWindows(t *testing.T) { + for _, id := range []string{"proj.", "proj..", "CON", "con", "nul", "COM1", "lpt9", "AUX.txt"} { + if got, err := sanitizeLinkID(id); err == nil { + t.Errorf("sanitizeLinkID(%q) = %q, want a refusal: it is not one distinct directory on every platform", id, got) + } + } + for _, id := range []string{"proj-1", "proj.git", "console", "com10", "auxiliary", "nulls"} { + if _, err := sanitizeLinkID(id); err != nil { + t.Errorf("sanitizeLinkID(%q) = %v, want it accepted", id, err) + } + } + // Surrounding space is not aliasing: it is trimmed before any path is + // built, so both spellings become one id and take one lock. + trimmed, err := sanitizeLinkID("proj-1 ") + if err != nil || trimmed != "proj-1" { + t.Errorf(`sanitizeLinkID("proj-1 ") = %q, %v; want it normalized to "proj-1"`, trimmed, err) + } +} diff --git a/internal/dictation/download.go b/internal/dictation/download.go index 76197b3a9..c444b5877 100644 --- a/internal/dictation/download.go +++ b/internal/dictation/download.go @@ -1748,7 +1748,7 @@ func promoteStagedDir(txn *destTxn, stageDir, destDir, label string, report func return fmt.Errorf("setting aside previous %s install: %w", label, err) } restore = func() error { return fsutil.RenameWithRetry(previous, destDir, holderFS.rename) } - } else if !os.IsNotExist(err) { + } else if !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("checking previous %s install: %w", label, err) }