diff --git a/internal/cli/app.go b/internal/cli/app.go index 44fa370ff..86547aee6 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -497,6 +497,8 @@ func runWithDeps(args []string, stdout io.Writer, stderr io.Writer, deps appDeps return runCron(args[1:], stdout, stderr, deps) case "repo-info", "repoinfo": return runRepoInfo(args[1:], stdout, stderr, deps) + case "kept-backups": + return runKeptBackups(args[1:], stdout, stderr, deps) case "serve": return runServe(args[1:], stdout, stderr, deps) case "acp": @@ -1368,6 +1370,7 @@ Commands: usage Summarize token usage and estimated cost cron Schedule agent jobs (foreground, file-backed) repo-info Characterize the current repository (local git only) + kept-backups List and remove copies a recovery pass retained serve Run Zero protocol servers acp Serve the Agent Client Protocol over stdio (editor backend) help Show this help diff --git a/internal/cli/completions.go b/internal/cli/completions.go index 5f9058946..783f272a4 100644 --- a/internal/cli/completions.go +++ b/internal/cli/completions.go @@ -96,6 +96,7 @@ var completionRoot = completionNode{ {names: []string{"usage"}, children: leafNodes("report")}, {names: []string{"cron"}, children: leafNodes("add", "list", "rm", "pause", "resume", "run")}, {names: []string{"repo-info", "repoinfo"}}, + {names: []string{"kept-backups"}, children: leafNodes("list", "remove")}, {names: []string{"serve"}}, {names: []string{"acp"}}, {names: []string{"help"}}, diff --git a/internal/cli/kept_backups.go b/internal/cli/kept_backups.go new file mode 100644 index 000000000..92799d5b0 --- /dev/null +++ b/internal/cli/kept_backups.go @@ -0,0 +1,227 @@ +package cli + +import ( + "errors" + "fmt" + "io" + "io/fs" + "path/filepath" + + "github.com/Gitlawb/zero/internal/daemon/remote" + "github.com/Gitlawb/zero/internal/dictation" + "github.com/Gitlawb/zero/internal/redaction" +) + +// runKeptBackups implements `zero kept-backups`, the only way a copy recovery +// retained ever leaves the disk. Recovery moves a copy it will not restore and +// cannot prove superseded under a Kept prefix its own scan never enumerates, and +// nothing reclaims one on its own, so without this command retention is one-way. +// +// zero kept-backups list [--bundle-dir ] list what is retained +// zero kept-backups remove [--bundle-dir ] remove one by name +func runKeptBackups(args []string, stdout io.Writer, stderr io.Writer, deps appDeps) int { + rest, bundleDir, err := splitBundleDirFlag(args) + if err != nil { + if _, werr := fmt.Fprintf(stderr, "zero kept-backups: %s\n\n", err); werr != nil { + return exitCrash + } + writeKeptBackupsUsage(stderr) + return exitUsage + } + if len(rest) == 0 { + writeKeptBackupsUsage(stderr) + return exitUsage + } + switch rest[0] { + case "list", "ls": + return keptBackupsList(rest[1:], bundleDir, stdout, stderr, deps) + case "remove", "rm": + return keptBackupsRemove(rest[1:], bundleDir, stdout, stderr, deps) + case "-h", "--help", "help": + // Explicit help is a success path, matching the other subcommands: usage + // to stdout, exit 0. Only the error paths below write it to stderr. + writeKeptBackupsUsage(stdout) + return exitSuccess + default: + if _, err := fmt.Fprintf(stderr, "zero kept-backups: unknown subcommand %q\n\n", rest[0]); err != nil { + return exitCrash + } + writeKeptBackupsUsage(stderr) + return exitUsage + } +} + +// splitBundleDirFlag pulls --bundle-dir out of the argument list wherever it +// appears, so it can sit before or after the subcommand and its name argument. +// The daemon has no config key for the bundle dir; it is a serve-remote flag, so +// the operator has to be able to name the same directory here. +func splitBundleDirFlag(args []string) (rest []string, bundleDir string, err error) { + for i := 0; i < len(args); i++ { + arg := args[i] + switch { + case arg == "--bundle-dir": + if i+1 >= len(args) { + return nil, "", errors.New("--bundle-dir needs a directory") + } + bundleDir = args[i+1] + i++ + case len(arg) > len("--bundle-dir=") && arg[:len("--bundle-dir=")] == "--bundle-dir=": + bundleDir = arg[len("--bundle-dir="):] + default: + rest = append(rest, arg) + } + } + return rest, bundleDir, nil +} + +// sttKeptRoot is where the dictation installs live: the same tree the rest of the +// TUI downloads into, derived from userConfigPath rather than the default config +// dir so an overridden config root does not leave this command reading a +// directory nothing writes to. +func sttKeptRoot(deps appDeps) (string, error) { + path, err := deps.userConfigPath() + if err != nil { + return "", err + } + if path == "" { + return "", errors.New("no user config path, so the dictation install root cannot be resolved") + } + return filepath.Join(filepath.Dir(path), "stt"), nil +} + +func keptBackupsList(args []string, bundleDir string, stdout io.Writer, stderr io.Writer, deps appDeps) int { + if len(args) > 0 { + if _, err := fmt.Fprintf(stderr, "zero kept-backups list: unexpected argument %q\n", args[0]); err != nil { + return exitCrash + } + return exitUsage + } + root, err := sttKeptRoot(deps) + if err != nil { + return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash) + } + sttBackups, err := dictation.ListKeptBackups(root) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash) + } + found, err := writeKeptBackups(stdout, "stt", sttKeptLines(sttBackups)) + if err != nil { + return exitCrash + } + if bundleDir != "" { + bundleBackups, err := remote.ListKeptBackups(bundleDir) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash) + } + bundleFound, err := writeKeptBackups(stdout, "bundle", bundleKeptLines(bundleBackups)) + if err != nil { + return exitCrash + } + found = bundleFound || found + } + if !found { + if _, err := fmt.Fprintln(stdout, "No kept backups."); err != nil { + return exitCrash + } + } + return exitSuccess +} + +// keptBackupLine is one retained copy as this command prints it. The two sites +// return their own KeptBackup types, and flattening them here is what keeps the +// output one format rather than two that drift. +type keptBackupLine struct { + name string + dest string + seq int64 + bytes int64 + owned bool +} + +func sttKeptLines(backups []dictation.KeptBackup) []keptBackupLine { + lines := make([]keptBackupLine, 0, len(backups)) + for _, b := range backups { + lines = append(lines, keptBackupLine{name: filepath.Base(b.Path), dest: b.Dest, seq: b.Seq, bytes: b.Bytes, owned: b.Owned}) + } + return lines +} + +func bundleKeptLines(backups []remote.KeptBackup) []keptBackupLine { + lines := make([]keptBackupLine, 0, len(backups)) + for _, b := range backups { + lines = append(lines, keptBackupLine{name: filepath.Base(b.Path), dest: b.Dest, seq: b.Seq, bytes: b.Bytes, owned: b.Owned}) + } + return lines +} + +// writeKeptBackups prints one line per retained copy. The name comes first after +// the site because it is exactly what `remove` takes; the destination is the +// install or link the copy was set aside for, and an entry nothing on disk +// attributes says so instead of borrowing a destination from its own name. +func writeKeptBackups(stdout io.Writer, site string, lines []keptBackupLine) (bool, error) { + for _, line := range lines { + dest := line.dest + if dest == "" { + dest = "-" + } + suffix := "" + if !line.owned { + suffix = " unowned" + } + if _, err := fmt.Fprintf(stdout, "%s %s dest=%s seq=%d bytes=%d%s\n", site, line.name, dest, line.seq, line.bytes, suffix); err != nil { + // A listing cut short is not a listing. Reporting success here tells + // an operator they have seen every retained copy when they have not, + // and this command is the only place those copies are visible. + return false, err + } + } + return len(lines) > 0, nil +} + +func keptBackupsRemove(args []string, bundleDir string, stdout io.Writer, stderr io.Writer, deps appDeps) int { + if len(args) != 1 { + if _, err := fmt.Fprintln(stderr, "usage: zero kept-backups remove [--bundle-dir ]"); err != nil { + return exitCrash + } + return exitUsage + } + name := args[0] + root := bundleDir + remove := func() error { return remote.RemoveKeptBackup(bundleDir, name) } + if bundleDir == "" { + sttRoot, err := sttKeptRoot(deps) + if err != nil { + return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash) + } + root = sttRoot + remove = func() error { return dictation.RemoveKeptBackup(sttRoot, name) } + } + if err := remove(); err != nil { + return writeAppError(stderr, redaction.ErrorMessage(err, redaction.Options{}), exitCrash) + } + if _, err := fmt.Fprintf(stdout, "Removed %s from %s\n", name, root); err != nil { + return exitCrash + } + return exitSuccess +} + +func writeKeptBackupsUsage(w io.Writer) { + _, _ = fmt.Fprint(w, `Usage: + zero kept-backups list [--bundle-dir ] List retained copies + zero kept-backups remove [--bundle-dir ] Remove one by name + +Recovery never deletes a copy it cannot prove was superseded; it moves that copy +under a kept- name and leaves it there. Nothing reclaims one on its own, so this +command is how retained copies leave the disk. + +Without --bundle-dir both subcommands work on the dictation install root. With +it, remove works on that daemon bundle dir instead, and list adds the bundle dir +to the dictation listing, so each line names the site it came from. Weigh the two differently: a +dictation kept backup is the only offline copy of an engine or a model, while a +bundle kept backup is a work tree the client that sent it can upload again. + +Entries marked unowned carry the kept- name with nothing on disk attributing +them. They are reported so they can be found, and remove refuses them; check +what they hold and remove those by hand. +`) +} diff --git a/internal/cli/kept_backups_test.go b/internal/cli/kept_backups_test.go new file mode 100644 index 000000000..089545d35 --- /dev/null +++ b/internal/cli/kept_backups_test.go @@ -0,0 +1,404 @@ +package cli + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// plantKeptTxn writes the transaction marker both install sites read: the kind +// that site writes, the destination the copy was set aside for, and the sequence +// its directory name carries. Written here by hand rather than through either +// package, so the command is exercised against the on-disk format an operator +// actually has, not against a helper that could drift with it. +func plantKeptTxn(t *testing.T, dir, kind, dest string, seq int64) { + t.Helper() + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + payload, err := json.Marshal(map[string]any{"kind": kind, "dest": dest, "seq": seq}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "txn"), payload, 0o600); err != nil { + t.Fatal(err) + } +} + +// plantSTTKept writes a Kept backup of a dictation install: the engine copy an +// operator has to be able to find, because it is the only offline one. +func plantSTTKept(t *testing.T, root, dest string, seq int64, content string) string { + t.Helper() + name := fmt.Sprintf("%s.kept-%020d-seq", dest, seq) + dir := filepath.Join(root, name) + plantKeptTxn(t, dir, "dictation-promote", dest, seq) + if err := os.MkdirAll(filepath.Join(dir, "install"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "install", "engine"), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return name +} + +// plantBundleKept writes a Kept backup of a bundle extract: a tree the client +// that uploaded it can send again, which is why the two sites cost differently. +func plantBundleKept(t *testing.T, dir, linkID string, seq int64, content string) string { + t.Helper() + name := fmt.Sprintf(".kept-%020d-seq", seq) + path := filepath.Join(dir, name) + plantKeptTxn(t, path, "bundle-extract", linkID, seq) + if err := os.MkdirAll(filepath.Join(path, "backup"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "backup", "a.txt"), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return name +} + +func keptDeps(userConfigPath string) appDeps { + return appDeps{userConfigPath: func() (string, error) { return userConfigPath, nil }} +} + +// The listing is the only way a retained copy is found again: recovery never +// enumerates the Kept prefix and nothing reclaims one on its own. It has to name +// the site, because the same command reads two of them and the cost of keeping a +// copy is not the same at both. +func TestKeptBackupsListPrintsBothSites(t *testing.T) { + userConfigPath := filepath.Join(t.TempDir(), "zero", "config.json") + sttRoot := filepath.Join(filepath.Dir(userConfigPath), "stt") + bundleDir := t.TempDir() + + sttName := plantSTTKept(t, sttRoot, "engine-a", 1, "engine-bytes") + bundleName := plantBundleKept(t, bundleDir, "proj-1", 7, "tree") + // Kept grammar, nothing attributing it: recovery's residue, which the + // operator has to see beside the real backups to reclaim it by hand. + unownedName := fmt.Sprintf(".kept-%020d-seq", 8) + if err := os.MkdirAll(filepath.Join(bundleDir, unownedName), 0o700); err != nil { + t.Fatal(err) + } + + var stdout, stderr bytes.Buffer + if code := runWithDeps([]string{"kept-backups", "list", "--bundle-dir", bundleDir}, &stdout, &stderr, keptDeps(userConfigPath)); code != 0 { + t.Fatalf("exit = %d, stderr = %s", code, stderr.String()) + } + + lines := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(stdout.String()), "\n") { + fields := strings.Fields(line) + if len(fields) > 1 { + lines[fields[1]] = line + } + } + stt, ok := lines[sttName] + if !ok { + t.Fatalf("the dictation kept backup is missing from:\n%s", stdout.String()) + } + for _, want := range []string{"stt", "dest=engine-a", "seq=1", "bytes="} { + if !strings.Contains(stt, want) { + t.Errorf("dictation line %q is missing %q", stt, want) + } + } + if strings.Contains(stt, "bytes=0") { + t.Errorf("dictation line %q reports no bytes for a copy that holds some", stt) + } + bundle, ok := lines[bundleName] + if !ok { + t.Fatalf("the bundle kept backup is missing from:\n%s", stdout.String()) + } + for _, want := range []string{"bundle", "dest=proj-1", "seq=7", "bytes="} { + if !strings.Contains(bundle, want) { + t.Errorf("bundle line %q is missing %q", bundle, want) + } + } + unowned, ok := lines[unownedName] + if !ok { + t.Fatalf("the unowned entry is missing from:\n%s", stdout.String()) + } + if !strings.Contains(unowned, "unowned") { + t.Errorf("unowned line %q does not say so", unowned) + } +} + +// Which site a removal lands on is decided by --bundle-dir alone. Getting that +// wrong deletes a copy at one site while the operator watches the other, so the +// name has to resolve at the site the flag names and nowhere else. +func TestKeptBackupsRemoveNamesTheSite(t *testing.T) { + userConfigPath := filepath.Join(t.TempDir(), "zero", "config.json") + sttRoot := filepath.Join(filepath.Dir(userConfigPath), "stt") + bundleDir := t.TempDir() + sttName := plantSTTKept(t, sttRoot, "engine-a", 1, "engine-bytes") + bundleName := plantBundleKept(t, bundleDir, "proj-1", 7, "tree") + + var stdout, stderr bytes.Buffer + if code := runWithDeps([]string{"kept-backups", "remove"}, &stdout, &stderr, keptDeps(userConfigPath)); code != exitUsage { + t.Fatalf("remove with no name: exit = %d, want %d (stderr %s)", code, exitUsage, stderr.String()) + } + + stdout.Reset() + stderr.Reset() + if code := runWithDeps([]string{"kept-backups", "remove", sttName}, &stdout, &stderr, keptDeps(userConfigPath)); code != 0 { + t.Fatalf("remove at the dictation root: exit = %d, stderr = %s", code, stderr.String()) + } + if _, err := os.Stat(filepath.Join(sttRoot, sttName)); !os.IsNotExist(err) { + t.Errorf("the dictation kept backup should be gone, got %v", err) + } + if _, err := os.Stat(filepath.Join(bundleDir, bundleName)); err != nil { + t.Errorf("a removal with no --bundle-dir must not touch the bundle site: %v", err) + } + + stdout.Reset() + stderr.Reset() + if code := runWithDeps([]string{"kept-backups", "remove", bundleName, "--bundle-dir", bundleDir}, &stdout, &stderr, keptDeps(userConfigPath)); code != 0 { + t.Fatalf("remove at the bundle dir: exit = %d, stderr = %s", code, stderr.String()) + } + if _, err := os.Stat(filepath.Join(bundleDir, bundleName)); !os.IsNotExist(err) { + t.Errorf("the bundle kept backup should be gone, got %v", err) + } +} + +// The two sites cost differently and only the usage text says so, so a run that +// stops printing it leaves the operator deciding blind. +func TestKeptBackupsUsageNamesTheCostOfEachSite(t *testing.T) { + var stdout, stderr bytes.Buffer + if code := runWithDeps([]string{"kept-backups", "-h"}, &stdout, &stderr, keptDeps("")); code != exitSuccess { + t.Fatalf("exit = %d, stderr = %s", code, stderr.String()) + } + for _, want := range []string{"only offline copy", "upload"} { + if !strings.Contains(stdout.String(), want) { + t.Errorf("usage is missing %q:\n%s", want, stdout.String()) + } + } +} + +// The reclaim command is the only path a retained copy has off the disk, so a +// command that never appears in help or completions is a reclaim path that +// exists but cannot be found. +func TestKeptBackupsIsDiscoverable(t *testing.T) { + var out bytes.Buffer + if code := Run([]string{"--help"}, &out, &out); code != 0 { + t.Fatalf("--help exit = %d", code) + } + if !strings.Contains(out.String(), "kept-backups") { + t.Error("kept-backups is missing from the command list") + } + var comp bytes.Buffer + if code := Run([]string{"completions", "bash"}, &comp, &comp); code != 0 { + t.Fatalf("completions exit = %d", code) + } + if !strings.Contains(comp.String(), "kept-backups") { + t.Error("kept-backups is missing from the completion tree") + } +} + +// The refusal and usage paths, over the command's own entry point. The two tests +// above only ever drive a removal that succeeds, which is why replacing the +// remove path's error handling with a discard left this package green: a refusal +// would have printed "Removed" and exited zero with nothing to say so. Every row +// here asserts the exit code, what did and did not reach stdout, and for the +// refusals that the copy is still on disk afterwards. +func TestKeptBackupsRefusalsAndUsage(t *testing.T) { + // plantUnowned writes a directory carrying the Kept grammar and no marker: + // recovery's residue, the entry the usage text promises remove refuses. + plantUnowned := func(t *testing.T, dir string, seq int64) string { + t.Helper() + name := fmt.Sprintf(".kept-%020d-seq", seq) + if err := os.MkdirAll(filepath.Join(dir, name, "backup"), 0o700); err != nil { + t.Fatal(err) + } + return name + } + + for _, tc := range []struct { + name string + // arrange plants whatever the row needs and returns the argument list. + arrange func(t *testing.T, sttRoot, bundleDir string) []string + wantCode int + wantOut string + // denyOut must not appear on stdout. "Removed" is the whole point on the + // refusal rows: a refusal that prints it is a removal as far as the + // operator can tell. + denyOut string + wantErr string + // check runs after the command, for the rows whose claim is on disk. + check func(t *testing.T, sttRoot, bundleDir string) + }{ + { + name: "remove refuses an unowned entry at the dictation root", + arrange: func(t *testing.T, sttRoot, _ string) []string { + return []string{"kept-backups", "remove", plantUnowned(t, sttRoot, 3)} + }, + wantCode: exitCrash, + denyOut: "Removed", + wantErr: "[zero]", + check: func(t *testing.T, sttRoot, _ string) { + if _, err := os.Stat(filepath.Join(sttRoot, fmt.Sprintf(".kept-%020d-seq", 3))); err != nil { + t.Errorf("a refused removal must leave the copy on disk: %v", err) + } + }, + }, + { + name: "remove refuses an unowned entry at the bundle dir", + arrange: func(t *testing.T, _, bundleDir string) []string { + return []string{"kept-backups", "remove", plantUnowned(t, bundleDir, 4), "--bundle-dir", bundleDir} + }, + wantCode: exitCrash, + denyOut: "Removed", + wantErr: "[zero]", + check: func(t *testing.T, _, bundleDir string) { + if _, err := os.Stat(filepath.Join(bundleDir, fmt.Sprintf(".kept-%020d-seq", 4))); err != nil { + t.Errorf("a refused removal must leave the copy on disk: %v", err) + } + }, + }, + { + // The flag can sit before the subcommand and can carry its value + // with an '=', and both forms have to reach the same site: an + // operator who names the bundle dir and is silently answered from + // the dictation root is being shown the wrong disk. + name: "remove lands on the bundle site through the equals form", + arrange: func(t *testing.T, sttRoot, bundleDir string) []string { + plantSTTKept(t, sttRoot, "engine-a", 1, "engine-bytes") + name := plantBundleKept(t, bundleDir, "proj-1", 7, "tree") + return []string{"kept-backups", "--bundle-dir=" + bundleDir, "remove", name} + }, + wantCode: exitSuccess, + wantOut: "Removed", + check: func(t *testing.T, sttRoot, bundleDir string) { + if _, err := os.Stat(filepath.Join(bundleDir, fmt.Sprintf(".kept-%020d-seq", 7))); !os.IsNotExist(err) { + t.Errorf("the bundle copy should be gone, got %v", err) + } + if _, err := os.Stat(filepath.Join(sttRoot, fmt.Sprintf("engine-a.kept-%020d-seq", 1))); err != nil { + t.Errorf("a removal at the bundle site must not touch the dictation root: %v", err) + } + }, + }, + { + name: "list lands on the bundle site through the equals form", + arrange: func(t *testing.T, _, bundleDir string) []string { + plantBundleKept(t, bundleDir, "proj-1", 7, "tree") + return []string{"kept-backups", "--bundle-dir=" + bundleDir, "list"} + }, + wantCode: exitSuccess, + wantOut: "bundle .kept-", + denyOut: "No kept backups.", + }, + { + name: "--bundle-dir with no value", + arrange: func(t *testing.T, _, _ string) []string { + return []string{"kept-backups", "list", "--bundle-dir"} + }, + wantCode: exitUsage, + wantErr: "needs a directory", + }, + { + name: "unknown subcommand", + arrange: func(t *testing.T, _, _ string) []string { + return []string{"kept-backups", "purge"} + }, + wantCode: exitUsage, + wantErr: "unknown subcommand", + }, + { + name: "no subcommand at all", + arrange: func(t *testing.T, _, _ string) []string { + return []string{"kept-backups"} + }, + wantCode: exitUsage, + wantErr: "Usage:", + }, + { + name: "list with a stray argument", + arrange: func(t *testing.T, _, _ string) []string { + return []string{"kept-backups", "list", "proj-1"} + }, + wantCode: exitUsage, + wantErr: "unexpected argument", + }, + { + name: "remove with two names", + arrange: func(t *testing.T, _, _ string) []string { + return []string{"kept-backups", "remove", "a", "b"} + }, + wantCode: exitUsage, + wantErr: "usage: zero kept-backups remove", + }, + { + // An empty root has to say so. Printing nothing at all reads as a + // command that did not run, and this listing is the only way a + // retained copy is found again. + name: "an empty root reports no backups", + arrange: func(t *testing.T, _, bundleDir string) []string { + return []string{"kept-backups", "list", "--bundle-dir", bundleDir} + }, + wantCode: exitSuccess, + wantOut: "No kept backups.", + }, + { + // The dictation root does not exist yet before the first install, + // which is a listing of nothing rather than an error. + name: "a dictation root that was never created reports no backups", + arrange: func(t *testing.T, sttRoot, _ string) []string { + if err := os.RemoveAll(sttRoot); err != nil { + t.Fatal(err) + } + return []string{"kept-backups", "list"} + }, + wantCode: exitSuccess, + wantOut: "No kept backups.", + }, + } { + t.Run(tc.name, func(t *testing.T) { + userConfigPath := filepath.Join(t.TempDir(), "zero", "config.json") + sttRoot := filepath.Join(filepath.Dir(userConfigPath), "stt") + if err := os.MkdirAll(sttRoot, 0o700); err != nil { + t.Fatal(err) + } + bundleDir := t.TempDir() + args := tc.arrange(t, sttRoot, bundleDir) + + var stdout, stderr bytes.Buffer + code := runWithDeps(args, &stdout, &stderr, keptDeps(userConfigPath)) + if code != tc.wantCode { + t.Errorf("exit = %d, want %d (stdout %q, stderr %q)", code, tc.wantCode, stdout.String(), stderr.String()) + } + if tc.wantOut != "" && !strings.Contains(stdout.String(), tc.wantOut) { + t.Errorf("stdout %q is missing %q", stdout.String(), tc.wantOut) + } + if tc.denyOut != "" && strings.Contains(stdout.String(), tc.denyOut) { + t.Errorf("stdout %q must not contain %q", stdout.String(), tc.denyOut) + } + if tc.wantErr != "" && !strings.Contains(stderr.String(), tc.wantErr) { + t.Errorf("stderr %q is missing %q", stderr.String(), tc.wantErr) + } + if tc.check != nil { + tc.check(t, sttRoot, bundleDir) + } + }) + } +} + +// The listing is the only place a retained copy is visible, so a listing cut +// short must not report success: an operator who sees a clean exit believes +// they have seen every copy that exists. +func TestKeptBackupsListFailsOnAShortWrite(t *testing.T) { + userConfigPath := filepath.Join(t.TempDir(), "config.json") + root := filepath.Join(filepath.Dir(userConfigPath), "stt") + plantSTTKept(t, root, "engine-a", 1, "the only offline copy") + + var stderr bytes.Buffer + if code := runWithDeps([]string{"kept-backups", "list"}, shortWriter{}, &stderr, keptDeps(userConfigPath)); code == exitSuccess { + t.Error("a listing that could not be written must not exit successfully") + } +} + +// shortWriter fails every write, standing in for a closed pipe or a full disk. +type shortWriter struct{} + +func (shortWriter) Write([]byte) (int, error) { return 0, errors.New("short write") } diff --git a/internal/daemon/remote/bridge.go b/internal/daemon/remote/bridge.go index ef4057acc..5066813f1 100644 --- a/internal/daemon/remote/bridge.go +++ b/internal/daemon/remote/bridge.go @@ -100,6 +100,16 @@ func NewBridge(opts BridgeOptions) (*Bridge, error) { if maxBundleBytes <= 0 { maxBundleBytes = defaultMaxBundleBytes } + bundleDir := strings.TrimSpace(opts.BundleDir) + if bundleDir != "" { + // A previous run may have died mid-swap, leaving a link's only tree in a + // staging dir. Nothing serves yet, so repair before the first upload. + recoverBundleDir(bundleDir, func(format string, args ...any) { + if opts.Log != nil { + opts.Log(fmt.Sprintf(format, args...)) + } + }) + } return &Bridge{ server: opts.Server, auth: opts.Authenticator, @@ -107,7 +117,7 @@ func NewBridge(opts BridgeOptions) (*Bridge, error) { minVersion: minVersion, handshakeTimeout: handshakeTimeout, authFailDelay: authFailDelay, - bundleDir: strings.TrimSpace(opts.BundleDir), + bundleDir: bundleDir, maxBundleBytes: maxBundleBytes, log: opts.Log, sem: make(chan struct{}, maxConns), diff --git a/internal/daemon/remote/bundle.go b/internal/daemon/remote/bundle.go index 398c1bcb2..0b7bdc838 100644 --- a/internal/daemon/remote/bundle.go +++ b/internal/daemon/remote/bundle.go @@ -1,6 +1,7 @@ package remote import ( + "cmp" "context" "crypto/sha256" "encoding/hex" @@ -8,14 +9,22 @@ import ( "errors" "fmt" "io" + "io/fs" + "maps" + "math" "net" "os" "os/exec" "path/filepath" + "slices" + "strconv" "strings" + "sync" "time" "github.com/Gitlawb/zero/internal/daemon" + "github.com/Gitlawb/zero/internal/fsutil" + "github.com/Gitlawb/zero/internal/lockutil" ) // gitTimeout bounds a single git invocation (bundle create/verify, clone) so a @@ -129,16 +138,18 @@ func (b *Bridge) receiveBundle(conn net.Conn) bundleResult { return bundleResult{Message: "stage bundle: " + err.Error()} } - ctx, cancel := context.WithTimeout(context.Background(), gitTimeout) - defer cancel() - if err := gitBundleVerify(ctx, tmpName); err != nil { + verifyCtx, cancelVerify := context.WithTimeout(context.Background(), gitTimeout) + defer cancelVerify() + if err := gitBundleVerify(verifyCtx, tmpName); err != nil { return bundleResult{Message: "bundle verify: " + err.Error()} } dest := filepath.Join(b.bundleDir, id) if !withinDir(b.bundleDir, dest) { return bundleResult{Message: "invalid link id"} } - if err := extractBundle(ctx, tmpName, dest); err != nil { + // extractBundle starts the clone's own gitTimeout once it holds the lock for + // dest, so an upload queued behind another does not spend that budget waiting. + if err := extractBundle(context.Background(), tmpName, dest, b.logf); err != nil { return bundleResult{Message: "extract bundle: " + err.Error()} } return bundleResult{OK: true, Path: dest} @@ -168,27 +179,1099 @@ func streamFramesToFile(r io.Reader, w io.Writer, size int64) error { return nil } -// extractBundle clones bundleFile into a staging dir, then atomically renames it -// over dest (replacing any prior extraction for this link id). git clone needs a -// non-existent target, so the staging+rename keeps the live dest intact on error. -func extractBundle(ctx context.Context, bundleFile, dest string) error { +// stagingPrefix names the per-extract staging directories created beside dest. +// sanitizeLinkID refuses every dot-prefixed id so a link can never name one. +const stagingPrefix = ".staging-" + +// keptPrefix names a staged backup that recovery decided to keep rather than +// order against a tree it had just put back. Recovery runs again on every start, +// and by then it can no longer tell that tree from one a later extract +// published, so the copy is parked under a name the scan does not enumerate. +// sanitizeLinkID refuses every dot-prefixed id, so this can never take a link's +// name. +const keptPrefix = ".kept-" + +// lockDirName holds the per-link advisory lock files that serialize extracts +// across processes. Dot-prefixed for the same reason stagingPrefix is. +const lockDirName = ".extract-locks" + +// stagingMarkerFile records, inside a staging dir, the transaction that created +// it: its kind, the link it is for, and the sequence in its own name. Without it +// a crash leaves an orphan nothing can attribute, and recovery has to retain a +// copy it can never name. +const stagingMarkerFile = "txn" + +// committedFile records, inside a staging dir, that the publish rename landed. +// It is the only evidence that the copy in backup was superseded, so recovery +// has to keep a backup that carries no flag. +const committedFile = "committed" + +// txnKindBundleExtract is the marker kind extractBundle writes. A marker naming +// another kind belongs to another site's transaction and is not this code's to +// act on. +const txnKindBundleExtract = "bundle-extract" + +// extractLockPoll is how often a cross-process extract lock is retried. +const extractLockPoll = 50 * time.Millisecond + +// fsOps is the filesystem seam the write path, the allocator, and recovery take +// every step through. A step that calls os directly cannot be made to fail, and +// the crash each of these steps exists to survive is then only reasoned about. +// Every field is one call, so a test can fail the second remove or the rename +// whose source is one particular backup and leave the rest of the pass real. +type fsOps struct { + rename func(from, to string) error + removeAll func(path string) error + stat func(name string) (os.FileInfo, error) + lstat func(name string) (os.FileInfo, error) + readDir func(name string) ([]os.DirEntry, error) + readFile func(name string) ([]byte, error) + mkdir func(name string, perm os.FileMode) error + writeFile func(name string, data []byte, perm os.FileMode) error + create func(name string, flag int, perm os.FileMode) (*os.File, error) + createTemp func(dir, pattern string) (*os.File, error) +} + +// stagingFS is the seam every filesystem step in this file goes through. Tests +// swap a field and restore it; nothing else writes to it. +var stagingFS = fsOps{ + rename: os.Rename, + removeAll: os.RemoveAll, + stat: os.Stat, + lstat: os.Lstat, + readDir: os.ReadDir, + readFile: os.ReadFile, + mkdir: os.Mkdir, + writeFile: os.WriteFile, + create: os.OpenFile, + createTemp: os.CreateTemp, +} + +// extractLocks serializes extracts per destination. Each bundle upload runs in +// its own connection goroutine, so two uploads of one link id would otherwise +// interleave their swap steps and clobber each other. +var extractLocks = struct { + mu sync.Mutex + locks map[string]*extractLock +}{locks: map[string]*extractLock{}} + +type extractLock struct { + mu sync.Mutex + refs int +} + +// lockExtract blocks until dest is free and returns its release func. Entries +// are refcounted so the map cannot grow with every link id ever uploaded. +func lockExtract(dest string) func() { + entry := holdExtractRef(dest) + entry.mu.Lock() + return func() { + entry.mu.Unlock() + dropExtractRef(dest) + } +} + +// waitForExtract takes the in-process lock under the same budget the per-link +// file lock waits out, so a queued upload gives the connection slot back rather +// than holding it for as long as the extract ahead of it runs. A bare Lock here +// is the one step in the whole extract with no bound: it is taken before the +// file lock, so its wait is not covered by that one's deadline. +func waitForExtract(ctx context.Context, dest string) (func(), error) { + deadline := time.NewTimer(gitTimeout) + defer deadline.Stop() + for { + release, held := tryLockExtract(dest) + if !held { + return release, nil + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-deadline.C: + return nil, fmt.Errorf("remote: timed out waiting for an extract in this process to finish with %s", dest) + case <-time.After(extractLockPoll): + } + } +} + +// tryLockExtract takes the in-process lock without waiting, reporting held when +// a goroutine in this daemon owns dest. Recovery cannot use lockExtract: an +// extract holds its destination across the whole clone, so a recovering pass +// that blocked here would wait out that clone before deciding anything about a +// destination it was going to skip either way. +func tryLockExtract(dest string) (release func(), held bool) { + entry := holdExtractRef(dest) + if !entry.mu.TryLock() { + dropExtractRef(dest) + return nil, true + } + return func() { + entry.mu.Unlock() + dropExtractRef(dest) + }, false +} + +// holdExtractRef returns dest's lock entry with a reference taken, so the entry +// cannot be dropped from the map while a caller is waiting on it. +func holdExtractRef(dest string) *extractLock { + extractLocks.mu.Lock() + defer extractLocks.mu.Unlock() + entry := extractLocks.locks[dest] + if entry == nil { + entry = &extractLock{} + extractLocks.locks[dest] = entry + } + entry.refs++ + return entry +} + +// dropExtractRef releases one reference and forgets the entry once nothing holds +// it, which is what keeps the map from growing with every link id ever seen. +func dropExtractRef(dest string) { + extractLocks.mu.Lock() + defer extractLocks.mu.Unlock() + entry := extractLocks.locks[dest] + if entry == nil { + return + } + entry.refs-- + if entry.refs == 0 { + delete(extractLocks.locks, dest) + } +} + +// lockExtractFile takes the cross-process advisory lock for dest, waiting until +// ctx is done or the wait budget runs out. The in-process lock already excludes +// this daemon's own goroutines; this excludes a second daemon sharing the dir. +func lockExtractFile(ctx context.Context, bundleDir, dest string) (func(), error) { + deadline := time.NewTimer(gitTimeout) + defer deadline.Stop() + for { + release, held, err := tryLockExtractFile(bundleDir, dest) + if err != nil { + return nil, err + } + if !held { + return release, nil + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-deadline.C: + return nil, fmt.Errorf("remote: timed out waiting for the extract lock on %s", dest) + case <-time.After(extractLockPoll): + } + } +} + +// tryLockExtractFile takes the per-link advisory lock without waiting. It +// reports held when a live extract owns the link, which is never an error: the +// caller either waits or leaves that link alone. +func tryLockExtractFile(bundleDir, dest string) (release func(), held bool, err error) { + lockDir := filepath.Join(bundleDir, lockDirName) + if err := os.MkdirAll(lockDir, 0o700); err != nil { + return nil, false, err + } + lock, err := lockutil.TryAcquireFileLockAt(bundleDir, filepath.Join(lockDir, filepath.Base(dest)+".lock")) + if err != nil { + if errors.Is(err, lockutil.ErrLockHeld) { + return nil, true, nil + } + return nil, false, err + } + return func() { _ = lock.Release() }, false, nil +} + +// recoverBundleDir repairs what a crash left behind in dir. It runs in two +// halves: a scan that discovers and classifies every candidate without touching +// anything, and a reconcile per destination that takes that destination's locks, +// re-validates everything it is about to act on, and only then acts. Splitting +// them is what keeps a decision from being made against state nobody was +// holding, and what keeps one destination's fault from reaching another's. +// +// Nothing here reads a clock, and nothing here remembers the previous pass. Both +// were sources of deletes the on-disk state did not license: an mtime says when +// a directory was written, not who owns it, and a per-pass map makes the second +// pass over the same directory reach a different answer from the first. +// +// It is called once at bridge construction, before any upload is served, and +// never removes a staging dir a live extract may hold. +func recoverBundleDir(dir string, logf func(string, ...any)) { + if logf == nil { + logf = func(string, ...any) {} + } + byDest := scanBundleDir(dir, logf) + // Sorted, so two passes over one directory visit destinations in the same + // order and a failure is reproducible rather than a function of readDir. + for _, id := range slices.Sorted(maps.Keys(byDest)) { + reconcileLink(dir, id, byDest[id], logf) + } +} + +// bundleCandidate is a staging directory the scan attributed to a destination: +// its name passed the strict grammar, it holds no work tree at its root, and its +// marker agrees with both the name and a link id the write path would accept. +type bundleCandidate struct { + path string + seq int64 + dest string +} + +// scanBundleDir reads dir once and groups every attributable staging directory +// by the destination its marker names. It mutates nothing: a directory that is +// going to be deleted is decided on under the destination's lock, and this runs +// before any lock is held. Entries it cannot attribute are reported here, once +// per pass, because a copy nothing names is one an operator cannot find. +func scanBundleDir(dir string, logf func(string, ...any)) map[string][]bundleCandidate { + entries, err := stagingFS.readDir(dir) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + logf("remote: could not scan bundle dir %s: %v", dir, err) + } + return nil + } + byDest := map[string][]bundleCandidate{} + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), stagingPrefix) { + continue + } + staging := filepath.Join(dir, entry.Name()) + seq, ok := stagingStamp(entry.Name()) + if !ok { + logf("remote: %s does not carry a name this code writes; leaving it in place", staging) + continue + } + id, ok, _ := attributeStagingDir(dir, staging, seq, logf) + if !ok { + continue + } + byDest[id] = append(byDest[id], bundleCandidate{path: staging, seq: seq, dest: id}) + } + return byDest +} + +// attributeStagingDir decides whether staging is this code's to act on. The +// order is load-bearing: the work-tree veto runs before the marker is read, +// because link ids starting with '.' used to be accepted, so a published work +// tree can sit under the exact generated name AND carry a file called txn at its +// root. Reading the marker first would let that file license a delete. +func attributeStagingDir(dir, staging string, seq int64, logf func(string, ...any)) (id string, ok bool, unreadable bool) { + if _, err := stagingFS.stat(filepath.Join(staging, ".git")); err == nil { + logf("remote: %s holds a work tree, not a staged extract; leaving it in place", staging) + return "", false, false + } else if !errors.Is(err, fs.ErrNotExist) { + // Not knowing whether this is a work tree is not the same as knowing it + // is not one. The veto sits ahead of the marker because a published tree + // can carry a file named txn at its own root, so falling through on an + // unreadable probe would let the tree answer for itself. + logf("remote: %s could not be checked for a work tree (%v); leaving it in place", staging, err) + return "", false, true + } + m, err := readMarker(staging) + if err != nil { + logf("remote: %s has no usable transaction marker (%v); leaving it in place", staging, err) + // A marker that is absent says this transaction never got far enough to + // claim the directory. A marker that could not be read says nothing at + // all, and treating the two alike lets an older copy be published over + // the one whose state is still unknown. + return "", false, !errors.Is(err, errMarkerMissing) + } + if m.Kind != txnKindBundleExtract { + logf("remote: %s carries a %q transaction marker, which no extract wrote; leaving it in place", staging, m.Kind) + return "", false, false + } + if m.Seq != seq { + // The name is the authority: createSequencedStagingDir walks up from the + // number it asked for, and it reads the name back before using it. A + // marker that disagrees with the name proves nothing about who wrote + // either of them. + logf("remote: %s carries a marker for sequence %d but is named %d; leaving it in place", staging, m.Seq, seq) + return "", false, false + } + // Everything read off disk goes back through the write path's own checks. A + // marker is a file, and a file can be edited by anything that can reach the + // bundle dir. + id, err = sanitizeLinkID(m.Dest) + if err != nil { + logf("remote: staged tree in %s names an invalid link (%v); leaving it in place", staging, err) + return "", false, false + } + if !withinDir(dir, filepath.Join(dir, id)) { + logf("remote: staged tree in %s names a link outside the bundle dir; leaving it in place", staging) + return "", false, false + } + return id, true, false +} + +// candidateState is one candidate classified per the recovery design. empty, +// committed and usable are three independent facts, and the disposition is a +// function of them plus the destination's own state; nothing else. +type candidateState struct { + bundleCandidate + empty bool + committed bool + usable bool +} + +// candidateVerdict separates the two ways a candidate leaves the reconcile +// without being acted on. Dropped means it is no longer attributable, which is +// this candidate's business alone. Unreadable means a filesystem fault, which +// says nothing about the candidate and stops the whole destination: acting on +// the rest would be deciding against a directory that could not be read. +type candidateVerdict int + +const ( + verdictOwned candidateVerdict = iota + verdictDropped + verdictUnreadable +) + +// reconcileLink classifies and then acts on every candidate for one destination, +// with that destination's in-process and cross-process locks held across both. +// The lock covers the check and the action together: a live extract mid-swap is +// indistinguishable on disk from a crashed one, and the lock is the only thing +// that tells them apart. +func reconcileLink(dir, id string, cands []bundleCandidate, logf func(string, ...any)) { + dest := filepath.Join(dir, id) + unlock, held := tryLockExtract(dest) + if held { + logf("remote: an extract in this process holds %s; leaving its staged copies alone", id) + return + } + defer unlock() + unlockFile, heldFile, err := tryLockExtractFile(dir, dest) + if err != nil { + logf("remote: could not lock %s while recovering it: %v", id, err) + return + } + if heldFile { + logf("remote: another process holds %s; leaving its staged copies alone", id) + return + } + defer unlockFile() + + states, ok := classifyCandidates(dir, id, cands, logf) + if !ok { + return + } + present, usable, ok := classifyDest(dest, logf) + if !ok { + return + } + if present && usable { + reconcileAgainstUsableDest(dir, id, states, logf) + return + } + restoreForDest(dir, id, dest, present, states, logf) +} + +// classifyCandidates re-validates every candidate under the lock and returns +// them newest first. The scan's reading is deliberately not trusted: it ran +// before the lock, so anything it saw could have been changed by the process the +// lock now excludes. +func classifyCandidates(dir, id string, cands []bundleCandidate, logf func(string, ...any)) ([]candidateState, bool) { + states := make([]candidateState, 0, len(cands)) + for _, c := range cands { + st, verdict := classifyCandidate(dir, c, logf) + switch verdict { + case verdictOwned: + states = append(states, st) + case verdictDropped: + continue + case verdictUnreadable: + logf("remote: leaving every staged copy for %s in place until %s can be read", id, c.path) + return nil, false + } + } + // Sequence order is transaction order: an extract that reads an existing + // entry always numbers above it, so a higher sequence is a later + // transaction whatever the clock did in between. + slices.SortStableFunc(states, func(a, b candidateState) int { return cmp.Compare(b.seq, a.seq) }) + return states, true +} + +func classifyCandidate(dir string, c bundleCandidate, logf func(string, ...any)) (candidateState, candidateVerdict) { + st := candidateState{bundleCandidate: c} + id, ok, unreadable := attributeStagingDir(dir, c.path, c.seq, logf) + if unreadable { + return st, verdictUnreadable + } + if !ok || id != c.dest { + return st, verdictDropped + } + backup := filepath.Join(c.path, "backup") + switch _, err := stagingFS.stat(backup); { + case err == nil: + case errors.Is(err, fs.ErrNotExist): + // A readable marker with no set-aside content is the one shape that + // proves the directory holds nothing: owned, and empty. + st.empty = true + return st, verdictOwned + default: + logf("remote: could not read the staged tree in %s: %v", c.path, err) + return st, verdictUnreadable + } + switch _, err := stagingFS.stat(filepath.Join(c.path, committedFile)); { + case err == nil: + st.committed = true + case errors.Is(err, fs.ErrNotExist): + default: + logf("remote: could not read the commit flag in %s: %v", c.path, err) + return st, verdictUnreadable + } + switch _, err := stagingFS.stat(filepath.Join(backup, ".git")); { + case err == nil: + st.usable = true + case errors.Is(err, fs.ErrNotExist): + // Durable: this copy will never pass the predicate, so it is skipped in + // selection and kept. That is a different fact from the error below. + default: + logf("remote: could not tell whether the staged tree in %s is usable: %v", c.path, err) + return st, verdictUnreadable + } + return st, verdictOwned +} + +// classifyDest reports whether dest exists and whether it can serve. Usability +// is dest/.git present, checked structurally. isGitWorktree shells out to git +// rev-parse, which discovers upward, so a bundle dir under any enclosing +// checkout would answer "usable" for an empty destination and license deleting +// every copy beside it. +func classifyDest(dest string, logf func(string, ...any)) (present, usable, ok bool) { + switch _, err := stagingFS.stat(dest); { + case err == nil: + case errors.Is(err, fs.ErrNotExist): + return false, false, true + default: + logf("remote: could not read %s: %v; leaving its staged copies in place", dest, err) + return false, false, false + } + switch _, err := stagingFS.stat(filepath.Join(dest, ".git")); { + case err == nil: + return true, true, true + case errors.Is(err, fs.ErrNotExist): + return true, false, true + default: + logf("remote: could not tell whether %s is usable: %v; leaving its staged copies in place", dest, err) + return true, false, false + } +} + +// reconcileAgainstUsableDest handles the destination that is present and can +// serve. Only a commit flag licenses a delete here: the flag is written by the +// transaction that published over that exact copy, so it is evidence about this +// copy. "The destination exists" is not, and deleting on it is what loses the +// last copy of a tree when the destination came from anywhere else. +func reconcileAgainstUsableDest(dir, id string, states []candidateState, logf func(string, ...any)) { + for _, c := range states { + switch { + case c.empty: + reapOwnedEmpty(c.path, logf) + case c.committed: + // The upload that published dest reported success to its client + // while this whole copy of the prior tree was still on disk. Say + // what is being reclaimed, so a bridge running without a logger is + // not the difference between the space being accounted for and not. + logf("remote: reclaiming the staged tree in %s that %s's live tree superseded", c.path, id) + if err := stagingFS.removeAll(c.path); err != nil { + logf("remote: could not remove superseded staging dir %s: %v", c.path, err) + } + default: + logf("remote: keeping the staged tree in %s: nothing proves %s published over it", c.path, id) + parkKeptBackup(dir, c.path, id, logf) + } + } +} + +// restoreForDest handles the destination that is absent or cannot serve. It +// selects before it moves anything: a destination with no usable candidate is +// left exactly as it was found, because taking its husk apart with nothing to +// put in its place leaves the operator with strictly less than they had. +func restoreForDest(dir, id, dest string, present bool, states []candidateState, logf func(string, ...any)) { + winner := selectCandidate(states) + if winner < 0 { + logf("remote: %s has no usable staged copy to restore; leaving it as it is", id) + parkRemaining(dir, id, states, -1, logf) + return + } + husk := "" + if present { + var err error + husk, err = setAsideHusk(dir, id, dest, logf) + if err != nil { + logf("remote: could not set the unusable tree at %s aside: %v; leaving %s alone", dest, err, id) + return + } + } + sel := states[winner] + if err := fsutil.RenameWithRetry(filepath.Join(sel.path, "backup"), dest, stagingFS.rename); err != nil { + logf("remote: could not restore the staged tree for %s from %s: %v", id, sel.path, err) + // No fallback to an older copy: installing one would put a tree at dest + // that the next pass reads as having published over the newer copy, + // which is how a retained copy turns into a deleted one. + if husk != "" { + putHuskBack(dest, husk, logf) + } + return + } + logf("remote: restored the work tree for %s from %s after an interrupted extract", id, sel.path) + // The winner's directory is now owned and empty, which is the one delete + // that needs no flag. + reapOwnedEmpty(sel.path, logf) + if husk != "" { + logf("remote: keeping the tree that could not serve at %s", dest) + parkKeptBackup(dir, husk, id, logf) + } + parkRemaining(dir, id, states, winner, logf) +} + +// selectCandidate is the whole selection rule: the newest uncommitted usable +// copy, and only when there is none, the newest committed usable one. A +// committed copy is second because the transaction that flagged it published +// something over it; an uncommitted one is the copy no publish is known to have +// replaced. +func selectCandidate(states []candidateState) int { + for i, c := range states { + if !c.empty && !c.committed && c.usable { + return i + } + } + for i, c := range states { + if !c.empty && c.committed && c.usable { + return i + } + } + return -1 +} + +// parkRemaining keeps every candidate selection did not restore. Nothing here is +// deleted: without a commit flag beside a usable destination there is no +// evidence any of these was superseded, and a copy with no evidence against it +// may be the last one. +func parkRemaining(dir, id string, states []candidateState, winner int, logf func(string, ...any)) { + for i, c := range states { + if i == winner { + continue + } + if c.empty { + reapOwnedEmpty(c.path, logf) + continue + } + logf("remote: keeping the staged tree in %s that %s was not restored from", c.path, id) + parkKeptBackup(dir, c.path, id, logf) + } +} + +// reapOwnedEmpty removes a staging dir that carries a valid marker and holds no +// set-aside content. The marker is what makes this safe: a directory with none +// names no destination, so no lock excludes the live allocation that may be +// sitting between its Mkdir and its marker write right now. +func reapOwnedEmpty(staging string, logf func(string, ...any)) { + if err := stagingFS.removeAll(staging); err != nil { + logf("remote: could not remove the empty staging dir %s: %v", staging, err) + } +} + +// setAsideHusk moves a destination that cannot serve into a fresh sequenced +// staging dir, so the restore has somewhere to land. It allocates and attributes +// before it moves anything, so a failure never leaves a copy of a tree in a +// directory no marker names. +func setAsideHusk(dir, id, dest string, logf func(string, ...any)) (string, error) { + seq, err := nextStagingSeq(dir) + if err != nil { + return "", err + } + staging, err := createSequencedStagingDir(dir, seq) + if err != nil { + return "", err + } + claimed, _ := stagingStamp(filepath.Base(staging)) + if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: id, Seq: claimed}); err != nil { + // Nothing has moved yet, so the directory holds nothing and removing it + // loses nothing. Leaving it would be residue no marker attributes. + reapOwnedEmpty(staging, logf) + return "", err + } + if err := fsutil.RenameWithRetry(dest, filepath.Join(staging, "backup"), stagingFS.rename); err != nil { + reapOwnedEmpty(staging, logf) + return "", err + } + return staging, nil +} + +// putHuskBack undoes a set-aside whose restore then failed, so the destination +// is left exactly as recovery found it. If the move back fails too, the husk +// stays where it is as an owned candidate of its own and both failures are +// named: the copy is still on disk, which is the property that matters. +func putHuskBack(dest, husk string, logf func(string, ...any)) { + if err := fsutil.RenameWithRetry(filepath.Join(husk, "backup"), dest, stagingFS.rename); err != nil { + logf("remote: and could not put the tree at %s back from %s: %v", dest, husk, err) + return + } + reapOwnedEmpty(husk, logf) +} + +// stagingStamp reads back the per-directory sequence createSequencedStagingDir +// put in a staging name. The grammar is exact: the prefix, stagingSeqDigits +// ASCII digits, the suffix, nothing else. No released version wrote a stamped +// name at all, so a looser parse buys no migration and costs ownership: v0.8.0 +// staged under os.MkdirTemp(parent, ".staging-*"), whose decimal suffix carries +// no second '-', and dot-prefixed link ids were once accepted, so a published +// work tree can sit under any name at all. One that reads as a sequence gets a +// say in the ordering, and one at the maximum stops the allocator for every link +// in the directory. +func stagingStamp(name string) (int64, bool) { + digits, ok := strings.CutPrefix(name, stagingPrefix) + if !ok { + return 0, false + } + digits, ok = strings.CutSuffix(digits, stagingSeqSuffix) + if !ok || len(digits) != stagingSeqDigits { + return 0, false + } + for i := 0; i < len(digits); i++ { + // ParseInt accepts a leading sign, and a link id may contain '-', so + // without this a legacy name reads back as a negative sequence the + // writer could never have emitted. + if digits[i] < '0' || digits[i] > '9' { + return 0, false + } + } + stamp, err := strconv.ParseInt(digits, 10, 64) + if err != nil { + // Digits alone can still name a number past int64. + return 0, false + } + return stamp, true +} + +// stagingSeqSuffix closes a sequenced staging name, and stagingSeqDigits is the +// width createSequencedStagingDir's %020d writes. The parser requires both +// exactly, and createSequencedStagingDir reads its own name back before using +// it, so a format string that drifted from either would fail there rather than +// putting a name on disk that reads as unstamped: an unstamped entry sorts last +// and is always retained, so the ordering key would simply stop existing with +// nothing failing. +const ( + stagingSeqSuffix = "-seq" + stagingSeqDigits = 20 +) + +// stagingSeqAttempts bounds the walk up from a taken number, in the spirit of +// the retry limit os.MkdirTemp applies to its own random names. +const stagingSeqAttempts = 10000 + +// nextStagingSeq is the number a new staging dir should claim: one past the +// highest already in the directory. That is what makes the order survive a +// clock that moves backward. An extract that reads an existing entry always +// allocates above it, and no clock correction can invert that, whereas the +// wall-clock stamp this replaces inverted whenever the clock did. +// +// Kept backups count. parkKeptBackup derives the parked name from the staging +// name, so a number handed out twice makes the second park land on an occupied +// name; that rename refuses, the backup stays under stagingPrefix, and the next +// pass deletes it as superseded. Counting kept names keeps a parked number out +// of circulation. Recovery still does not enumerate them. +func nextStagingSeq(dir string) (int64, error) { + entries, err := stagingFS.readDir(dir) + if err != nil { + return 0, err + } + var high int64 + for _, entry := range entries { + if !entry.IsDir() { + continue + } + // Read only the names this package writes. The sibling directories here + // are the per-link work trees, and a link id is whatever the uploading + // client sent; stagingStamp trims its prefix with TrimPrefix, which is a + // no-op on a name that lacks it, so an unfiltered scan would read a link + // named "2024-project" as sequence 2024 and one named for int64's + // maximum as a permanent refusal to allocate anything. + name := entry.Name() + switch { + case strings.HasPrefix(name, keptPrefix): + name = stagingPrefix + strings.TrimPrefix(name, keptPrefix) + case strings.HasPrefix(name, stagingPrefix): + default: + continue + } + stamp, ok := stagingStamp(name) + if !ok || stamp <= high { + continue + } + // A dir with a .git at its own root is a work tree, not something this + // package wrote: link ids starting with '.' used to be accepted, so one + // can carry the exact generated name. The clone and the set-aside tree a + // live staging dir holds are one level down, in repo/ and backup/, so + // this vetoes the legacy tree without releasing a number that is still + // spoken for. + if _, err := stagingFS.stat(filepath.Join(dir, entry.Name(), ".git")); err == nil { + continue + } + high = stamp + } + if high == math.MaxInt64 { + // The addition below would wrap negative, and %020d of a negative + // renders a '-' the parser reads as empty digits, so the entry would + // drop out of the ordering without saying so. Refuse instead. + return 0, fmt.Errorf("remote: %s holds a staging name at the maximum sequence; remove it before extracting again", dir) + } + return high + 1, nil +} + +// createSequencedStagingDir claims the first free name from n upward. Exclusive +// creation is what arbitrates: two extracts racing in one directory cannot both +// win a name, and the loser walks up to a number strictly above the winner's. +func createSequencedStagingDir(dir string, n int64) (string, error) { + if n < 1 { + return "", fmt.Errorf("remote: refusing to allocate staging sequence %d", n) + } + for i := 0; i < stagingSeqAttempts; i++ { + name := fmt.Sprintf("%s%020d%s", stagingPrefix, n, stagingSeqSuffix) + // The writer and the reader agree on the format or nothing is written. + // Checking here rather than trusting the format string is what keeps a + // silently unstamped name from reaching disk. + if stamp, ok := stagingStamp(name); !ok || stamp != n { + return "", fmt.Errorf("remote: staging name %q does not read back as sequence %d", name, n) + } + path := filepath.Join(dir, name) + err := stagingFS.mkdir(path, 0o700) + if err == nil { + return path, nil + } + if !errors.Is(err, fs.ErrExist) { + return "", err + } + if n == math.MaxInt64 { + return "", fmt.Errorf("remote: staging sequence exhausted in %s", dir) + } + n++ + } + return "", fmt.Errorf("remote: could not claim a staging name in %s after %d attempts", dir, stagingSeqAttempts) +} + +// parkKeptBackup moves a staging dir out of the prefix recovery scans, so the +// next pass leaves alone what this one deliberately kept instead of reading the +// restored tree at dest as a later extract publishing over it. A rename onto an +// existing directory fails rather than replacing it, so a legacy link that +// happens to carry the parked name is never clobbered; the copy just stays where +// it is, which is the same fail-safe one pass later. +func parkKeptBackup(dir, staging, id string, logf func(string, ...any)) { + parked := filepath.Join(dir, keptPrefix+strings.TrimPrefix(filepath.Base(staging), stagingPrefix)) + if err := fsutil.RenameWithRetry(staging, parked, stagingFS.rename); err != nil { + logf("remote: could not park the kept backup for %s at %s: %v", id, parked, err) + } +} + +// keptStamp reads the sequence out of a Kept backup name. It is the same +// grammar the scanned prefix uses with the other prefix in front, and it is +// deliberately the ONLY prefix this file's operator commands accept: a directory +// under stagingPrefix may be the staging dir of an extract running right now, +// and recovery has a disposition for it either way. +func keptStamp(name string) (int64, bool) { + rest, ok := strings.CutPrefix(name, keptPrefix) + if !ok { + return 0, false + } + return stagingStamp(stagingPrefix + rest) +} + +// KeptBackup is one copy recovery moved under the Kept prefix rather than +// deleted, as an operator sees it. Recovery never enumerates that prefix, so +// this listing is the only way a retained copy is found again; nothing reclaims +// one on its own. Owned false is a directory carrying the Kept name that nothing +// on disk attributes, listed beside the real backups so recovery's residue is +// visible rather than silent until a disk fills. +type KeptBackup struct { + Path string + Dest string + Seq int64 + Bytes int64 + Owned bool +} + +// ListKeptBackups reports every Kept backup in a bundle dir. The destination +// comes off the marker and never off the name: a bundle Kept name carries no +// link id at all, so a name-derived destination would be an invention. +func ListKeptBackups(dir string) ([]KeptBackup, error) { + entries, err := stagingFS.readDir(dir) + if err != nil { + return nil, err + } + var out []KeptBackup + for _, entry := range entries { + if !entry.IsDir() { + continue + } + seq, ok := keptStamp(entry.Name()) + if !ok { + continue + } + path := filepath.Join(dir, entry.Name()) + backup := KeptBackup{Path: path, Seq: seq, Bytes: dirBytes(path)} + // The same proof recovery requires before it touches a directory: the + // work-tree veto, then a marker agreeing with the name and naming a link + // the write path would have accepted. + if id, ok, _ := attributeStagingDir(dir, path, seq, discardLog); ok { + backup.Dest = id + backup.Owned = true + } + out = append(out, backup) + } + slices.SortFunc(out, func(a, b KeptBackup) int { return cmp.Compare(a.Path, b.Path) }) + return out, nil +} + +// RemoveKeptBackup deletes the one Kept backup an operator named. It is not an +// rm: the name has to be a single component under dir, pass the Kept grammar, +// hold no work tree at its root, and carry a marker whose kind, link, and +// sequence all agree, and the link's extract locks have to be free. Anything +// else is refused and left for the operator to remove by hand, because this is +// the only command that deletes a Kept backup and a wrong answer here is the +// last copy of a tree. +func RemoveKeptBackup(dir, name string) error { + if name == "" || name == "." || name == ".." || name != filepath.Base(name) { + // Ahead of every filesystem call: a name carrying a separator joins to a + // path outside dir, which would make this a remote rm. + return fmt.Errorf("remote: %q is not the name of an entry in %s", name, dir) + } + seq, ok := keptStamp(name) + if !ok { + return fmt.Errorf("remote: %s is not a kept backup; recovery may still act on it, so remove it by hand", name) + } + path := filepath.Join(dir, name) + id, ok, _ := attributeStagingDir(dir, path, seq, discardLog) + if !ok { + return fmt.Errorf("remote: nothing on disk attributes %s to a link; remove it by hand", name) + } + dest := filepath.Join(dir, id) + unlock, held := tryLockExtract(dest) + if held { + return fmt.Errorf("remote: an extract in this process holds %s; try again once it finishes", id) + } + defer unlock() + unlockFile, heldFile, err := tryLockExtractFile(dir, dest) + if err != nil { + return err + } + if heldFile { + return fmt.Errorf("remote: another process holds %s; try again once it finishes", id) + } + defer unlockFile() + // Re-read under the lock. The attribution above ran with nothing excluding a + // live extract, and a directory that stopped being this link's in between is + // one that must not be deleted on the strength of the earlier reading. + if again, ok, _ := attributeStagingDir(dir, path, seq, discardLog); !ok || again != id { + return fmt.Errorf("remote: %s is no longer attributable to %s; leaving it in place", name, id) + } + return stagingFS.removeAll(path) +} + +// dirBytes sums the regular files under path. Unreadable entries are skipped +// rather than failing the listing: a size an operator cannot see is a worse +// answer than a size that is short, and the entry itself is still reported. +func dirBytes(path string) int64 { + var total int64 + _ = filepath.WalkDir(path, func(_ string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return nil + } + total += info.Size() + return nil + }) + return total +} + +// discardLog is the reporter the operator commands hand the recovery helpers +// they reuse. Those helpers narrate every refusal for the recovery pass's log; +// here the refusal comes back as an error instead. +func discardLog(string, ...any) {} + +// txnMarker is what a staging dir carries to prove which transaction created it. +// Seq repeats the number in the directory's own name: a marker that disagrees +// with its name proves nothing about who wrote either, so the pair is what makes +// the directory owned rather than the name alone. +type txnMarker struct { + Kind string `json:"kind"` + Dest string `json:"dest"` + Seq int64 `json:"seq"` +} + +// errMarkerMissing reports that a staging dir carries no marker at all, which is +// a different fact from a marker that could not be read: the first is a crash +// before the marker was written, the second is a filesystem fault. Deciding +// between them off a bare error would fold a fault into "not ours". +var errMarkerMissing = errors.New("remote: staging dir carries no transaction marker") + +// writeMarker publishes m into dir under stagingMarkerFile, through a temp file +// in that same directory so the rename is the only step a reader can observe. A +// plain write can be torn by a crash, and a half-written marker parses as +// nothing while still occupying the name recovery reads to decide ownership. +func writeMarker(dir string, m txnMarker) error { + payload, err := json.Marshal(m) + if err != nil { + return err + } + tmp, err := stagingFS.createTemp(dir, stagingMarkerFile+"-*") + if err != nil { + return err + } + name := tmp.Name() + // Sync before the rename: the rename can reach disk ahead of the bytes it + // names, which is exactly the partial marker the temp file exists to avoid. + if _, err := tmp.Write(payload); err != nil { + _ = tmp.Close() + _ = stagingFS.removeAll(name) + return err + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + _ = stagingFS.removeAll(name) + return err + } + if err := tmp.Close(); err != nil { + _ = stagingFS.removeAll(name) + return err + } + if err := fsutil.RenameWithRetry(name, filepath.Join(dir, stagingMarkerFile), stagingFS.rename); err != nil { + _ = stagingFS.removeAll(name) + return err + } + return nil +} + +// readMarker reads back what writeMarker published. A dir with no marker yields +// errMarkerMissing; every other failure is wrapped, so a caller can tell "this +// was never ours" from "this could not be read", which are opposite decisions. +func readMarker(dir string) (txnMarker, error) { + raw, err := stagingFS.readFile(filepath.Join(dir, stagingMarkerFile)) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return txnMarker{}, errMarkerMissing + } + return txnMarker{}, fmt.Errorf("read transaction marker in %s: %w", dir, err) + } + var m txnMarker + if err := json.Unmarshal(raw, &m); err != nil { + return txnMarker{}, fmt.Errorf("parse transaction marker in %s: %w", dir, err) + } + return m, nil +} + +// extractBundle clones bundleFile into a staging dir beside dest, then swaps the +// clone into place (replacing any prior extraction for this link id). git clone +// needs a non-existent target, hence the staging dir. The steps run in this +// order: allocate the sequenced staging dir, write its transaction marker, clone +// into it, set the live tree aside, publish the clone, record the commit flag, +// remove the staging dir. The marker precedes the first destructive rename and +// the flag follows the publish, so every copy this leaves on disk names the +// transaction that wrote it and says whether that transaction committed. The +// live tree is moved aside rather than deleted and is put back if the publish +// fails, so on every error return dest holds either the prior extraction or the +// new one, never neither. Swapping a directory is two renames and cannot be made +// atomic, so a crash between them leaves dest absent with the prior tree in +// staging/backup, which the marker is what lets recovery attribute and put back. +// logf may be nil. +func extractBundle(ctx context.Context, bundleFile, dest string, logf func(string, ...any)) error { + if logf == nil { + logf = func(string, ...any) {} + } parent := filepath.Dir(dest) if err := os.MkdirAll(parent, 0o700); err != nil { return err } - staging, err := os.MkdirTemp(parent, ".staging-*") + unlock, err := waitForExtract(ctx, dest) if err != nil { return err } - defer func() { _ = os.RemoveAll(staging) }() + defer unlock() + unlockFile, err := lockExtractFile(ctx, parent, dest) + if err != nil { + return err + } + defer unlockFile() + + // The sequence records this extract's place in the order, which is what lets + // recovery tell an older leftover staging dir from a newer one. It is one + // past the highest number already in the directory, claimed by exclusive + // creation, so a concurrent extract for another link cannot take the same + // value and a clock that moves backward cannot invert the order. Every number + // on disk is one of these per-directory sequences: no released version wrote + // a stamped name at all (v0.8.0 staged under os.MkdirTemp's random suffix), + // so nothing here is ordering against a wall clock it inherited. + seq, err := nextStagingSeq(parent) + if err != nil { + return err + } + staging, err := createSequencedStagingDir(parent, seq) + if err != nil { + return err + } + // Staging also holds the prior tree while the swap is in flight, so it is + // only cleaned up once dest is known to hold one of the two trees. + cleanupStaging := true + defer func() { + if !cleanupStaging { + return + } + // A failure here strands a whole copy of the prior tree under a + // dot-prefixed dir nothing else enumerates, so say so rather than + // leaking it silently. + if err := stagingFS.removeAll(staging); err != nil { + logf("remote: could not remove bundle staging dir %s: %v", staging, err) + } + }() + // Attribute the staging dir before anything else touches the filesystem: from + // here on every failure can leave a copy of a tree behind, and a copy no + // marker names is one recovery can neither put back nor ever reclaim. The + // name is the authority on the sequence, because createSequencedStagingDir + // walks up from seq when a name is taken, and a marker that disagrees with + // its own directory name proves nothing. + claimed, _ := stagingStamp(filepath.Base(staging)) + if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: filepath.Base(dest), Seq: claimed}); err != nil { + return err + } + cloneCtx, cancelClone := context.WithTimeout(ctx, gitTimeout) + defer cancelClone() cloneDest := filepath.Join(staging, "repo") - if err := gitClone(ctx, bundleFile, cloneDest); err != nil { + if err := gitClone(cloneCtx, bundleFile, cloneDest); err != nil { + return err + } + + // Every rename stays inside parent, so none of them crosses a filesystem. + backup := filepath.Join(staging, "backup") + restore := func() error { return nil } + if err := fsutil.RenameWithRetry(dest, backup, stagingFS.rename); err == nil { + restore = func() error { return fsutil.RenameWithRetry(backup, dest, stagingFS.rename) } + } else if !errors.Is(err, fs.ErrNotExist) { return err } - if err := os.RemoveAll(dest); err != nil { + if err := fsutil.RenameWithRetry(cloneDest, dest, stagingFS.rename); err != nil { + if restoreErr := restore(); restoreErr != nil { + // dest is empty and the only copy of the prior tree is the backup, + // so keep staging rather than deleting the tree on the way out. + cleanupStaging = false + return fmt.Errorf("publish extraction: %w (prior tree left in %s: %v)", err, backup, restoreErr) + } return err } - return os.Rename(cloneDest, dest) + // The flag is the only evidence that the copy in backup was published over. + // Without it that copy has to be kept, so a failure here costs one retained + // tree and never the publish, which has already landed and is reported as + // the success it is. + flag, err := stagingFS.create(filepath.Join(staging, committedFile), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + cleanupStaging = false + logf("remote: published %s but could not record the commit flag in %s: %v", dest, staging, err) + return nil + } + _ = flag.Close() + return nil } // ---- client side ----------------------------------------------------------- @@ -323,8 +1406,9 @@ func runGit(ctx context.Context, dir string, args ...string) error { } // sanitizeLinkID validates a link id used as a single path component under the -// bundle dir. It allows letters, digits, '-', '_', '.', forbids the traversal -// names, and caps the length — so it can never escape the bundle dir. +// bundle dir. It allows letters, digits, '-', '_', '.', forbids a leading '.', +// and caps the length, so an id can never escape the bundle dir and can never +// name one of the stagingPrefix directories an extract creates beside it. func sanitizeLinkID(id string) (string, error) { id = strings.TrimSpace(id) if id == "" { @@ -333,8 +1417,19 @@ func sanitizeLinkID(id string) (string, error) { if len(id) > 128 { return "", errors.New("remote: link id too long (max 128)") } - if id == "." || id == ".." { - return "", errors.New("remote: invalid link id") + if strings.HasPrefix(id, ".") { + return "", errors.New("remote: link id may not start with '.'") + } + // Windows drops a trailing dot or space when it resolves a path, and maps + // the reserved device names onto devices rather than directory entries. An + // id that is not one distinct directory on every platform breaks the thing + // both extract locks assume: they key on the id, so two ids that resolve to + // one tree take two different locks and clone into it at the same time. + if strings.HasSuffix(id, ".") { + return "", errors.New("remote: link id may not end with '.'") + } + if isReservedDeviceName(id) { + return "", errors.New("remote: link id may not be a reserved device name") } for _, r := range id { switch { @@ -376,3 +1471,21 @@ func firstLine(s string) string { } return s } + +// isReservedDeviceName reports whether name is one of the Windows device names, +// which resolve to a device on that platform whatever directory they sit in. +// The check is case-insensitive and ignores an extension, because CON, con and +// AUX.txt all resolve the same way. +func isReservedDeviceName(name string) bool { + if i := strings.IndexByte(name, '.'); i >= 0 { + name = name[:i] + } + switch strings.ToUpper(name) { + case "CON", "PRN", "AUX", "NUL": + return true + } + if len(name) == 4 && (strings.EqualFold(name[:3], "COM") || strings.EqualFold(name[:3], "LPT")) { + return name[3] >= '1' && name[3] <= '9' + } + return false +} diff --git a/internal/daemon/remote/bundle_matrix_test.go b/internal/daemon/remote/bundle_matrix_test.go new file mode 100644 index 000000000..21954cef8 --- /dev/null +++ b/internal/daemon/remote/bundle_matrix_test.go @@ -0,0 +1,1245 @@ +package remote + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" +) + +// The crash rows of the recovery plan's Acceptance Examples, as code. Each row +// drives the REAL extract into the on-disk state a stop at one step leaves, then +// runs recovery twice and asserts both halves the table asserts: which tree is +// live, and where every other copy of a tree ended up. Nothing here plants a +// staging dir by hand, so the fixture is whatever the write path actually +// writes; a hand-built one would only prove recovery agrees with the test's idea +// of the write path. +// +// A second pass is asserted because recovery keeps no memory: a copy retained on +// pass one must not be reclassified on pass two, and a copy deleted on pass one +// must not come back. + +// copyDisposition is where one copy of a tree ends up. The five values are the +// only terminal states the table uses, so a row that means "retained" has to say +// which kind of retention, and a delete can never be written down as anything +// else. +type copyDisposition int + +const ( + // copyDeleted: gone from disk, under neither prefix. + copyDeleted copyDisposition = iota + // copyAtStaging: still under the name the extract allocated for it. + copyAtStaging + // copyAtKept: moved under the Kept prefix, where the scan does not look. + copyAtKept + // copyInPlaceUnowned: still at its name and carrying no marker, so nothing + // on disk attributes it to this code. + copyInPlaceUnowned + // copyAtDest: restored, so the copy's content is the live tree and its + // directory is gone. + copyAtDest +) + +// bundleCrashRow is one row of the table. Each of these rows leaves exactly one +// staged copy, and the assertion compares the WHOLE set of copies on disk +// against that one, so a row can never pass while a second copy nobody expected +// sits beside it. +type bundleCrashRow struct { + id string + // arrange runs the real extract with faults injected to stop it at one step. + // The faults are unwound before recovery runs, so recovery sees a crashed + // tree and not a failing filesystem. + arrange func(t *testing.T, dir, dest string) + wantLive string + wantCopy copyDisposition + // report1 and report2 assert the copy is named in the recovery report on + // that pass. A copy recovery retains but never names is one an operator + // cannot find, which the table counts as a failure of the row. + report1 bool + report2 bool +} + +// arrangeInterruptedSwap stops the extract between its two swap renames: the +// prior tree is aside in backup, the publish failed, and the restore failed too, +// so the destination is absent and that backup is the only copy of it. S4 and S7 +// differ in how the writer got here and are identical on disk, which is why they +// share one arrangement. +func arrangeInterruptedSwap(t *testing.T, dir, dest string) { + t.Helper() + injected := errors.New("injected swap failure") + injectFault(t, "rename", func(args ...string) bool { + from := filepath.Base(args[0]) + return from == "repo" || from == "backup" + }, injected) + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil); err == nil { + t.Fatal("an extract whose publish and restore both fail must report an error") + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatalf("the interrupted swap should leave %s absent, got %v", dest, err) + } +} + +func bundleCrashRows() []bundleCrashRow { + return []bundleCrashRow{ + { + // A stop between the directory and its marker leaves a directory + // nothing on disk attributes. Deleting it would need proof this code + // wrote it, and the marker is that proof, so it is retained and + // named rather than reaped on the strength of its name. + id: "S1", + arrange: func(t *testing.T, dir, dest string) { + injectFault(t, "createTemp", func(args ...string) bool { + return args[1] == stagingMarkerFile+"-*" + }, errors.New("injected marker temp failure")) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil); err == nil { + t.Fatal("an extract whose marker cannot be written must fail") + } + }, + wantLive: "v0", + wantCopy: copyInPlaceUnowned, + report1: true, + report2: true, + }, + { + // The marker landed and the clone did not, so the directory is owned + // and holds no copy of any tree. Nothing can be lost by removing it, + // and leaving it is what accumulates scratch forever. + id: "S2", + arrange: func(t *testing.T, dir, dest string) { + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := extractBundle(context.Background(), filepath.Join(dir, "missing.bundle"), dest, nil); err == nil { + t.Fatal("cloning a bundle that does not exist must fail") + } + }, + wantLive: "v0", + wantCopy: copyDeleted, + }, + { + // The prior tree is aside and the destination is absent, so the + // staged copy is the only copy of it and has to come back. + id: "S4", + arrange: arrangeInterruptedSwap, + wantLive: "v0", + wantCopy: copyAtDest, + }, + { + // The publish landed and the flag did not. Without the flag there is + // no evidence the copy in backup was superseded, so it may not be + // deleted; parking it is what keeps it out of the next pass's way + // without dropping the last copy of a tree. + id: "S5", + arrange: func(t *testing.T, dir, dest string) { + injectFault(t, "create", func(args ...string) bool { + return filepath.Base(args[0]) == committedFile + }, errors.New("injected commit flag failure")) + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil); err != nil { + t.Fatalf("a committed publish must be reported as success: %v", err) + } + }, + wantLive: "v1", + wantCopy: copyAtKept, + report1: true, + }, + { + // The publish failed and the restore put the tree back, so the + // directory holds no copy of anything: owned and empty. + id: "S6", + arrange: func(t *testing.T, dir, dest string) { + injected := errors.New("injected publish failure") + injectFault(t, "rename", func(args ...string) bool { + return filepath.Base(args[0]) == "repo" + }, injected) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil); !errors.Is(err, injected) { + t.Fatalf("extract = %v, want the injected publish failure", err) + } + }, + wantLive: "v0", + wantCopy: copyDeleted, + }, + { + // The writer's own retain branch: publish failed, restore failed, the + // copy was kept and named in the error. On disk this is S4, and + // recovery must not tell them apart, because nothing on disk does. + id: "S7", + arrange: arrangeInterruptedSwap, + wantLive: "v0", + wantCopy: copyAtDest, + }, + { + // The flag is there, so the copy in backup is provably superseded by + // the tree now live at the destination. This is the one and only + // shape that licenses deleting a copy of a tree. + id: "S8", + arrange: func(t *testing.T, dir, dest string) { + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil); err != nil { + t.Fatalf("a cleanup failure must be logged, not returned: %v", err) + } + }, + wantLive: "v1", + wantCopy: copyDeleted, + }, + } +} + +func TestBundleCrashMatrix(t *testing.T) { + for _, row := range bundleCrashRows() { + t.Run(row.id, func(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + // The faults belong to the crash, not to recovery: restoring the + // seam here is what makes the next lines a recovery over a real + // filesystem rather than over a failing one. + func() { + saved := stagingFS + defer func() { stagingFS = saved }() + row.arrange(t, dir, dest) + }() + staging := soleStaging(t, dir) + + for pass, wantReport := range []bool{row.report1, row.report2} { + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { + logged = append(logged, fmt.Sprintf(format, args...)) + }) + assertBundleTerminalState(t, dir, dest, staging, row.wantCopy, row.wantLive, pass+1) + if wantReport { + assertCopyReported(t, logged, stagingSeqDigitsOf(t, staging), pass+1) + } + } + }) + } +} + +// assertBundleTerminalState asserts both halves of a row: the live tree, and the +// full set of copies still on disk under either prefix. The set is compared +// whole rather than one path at a time, so an extra copy nobody expected fails +// the row instead of going unnoticed. +func assertBundleTerminalState(t *testing.T, dir, dest, staging string, want copyDisposition, wantLive string, pass int) { + t.Helper() + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + switch { + case wantLive == "": + if err == nil { + t.Errorf("pass %d: the destination should be absent, it holds %q", pass, got) + } + case err != nil || string(got) != wantLive: + t.Errorf("pass %d: dest a.txt = %q (err %v), want %q", pass, got, err, wantLive) + } + + var wantPaths []string + switch want { + case copyAtStaging, copyInPlaceUnowned: + wantPaths = []string{staging} + case copyAtKept: + wantPaths = []string{parkedStaging(staging)} + } + assertCopySet(t, dir, []string{stagingPrefix, keptPrefix}, wantPaths, pass) + + switch want { + case copyInPlaceUnowned: + // Retained is not the whole claim: the reason it is retained is that + // nothing on disk attributes it, and a row that stopped checking that + // would pass over a copy recovery could have proved was its own. + if _, err := readMarker(staging); !errors.Is(err, errMarkerMissing) { + t.Errorf("pass %d: %s should carry no marker, readMarker = %v", pass, staging, err) + } + case copyAtDest: + if _, err := os.Stat(dest); err != nil { + t.Errorf("pass %d: the copy should have been restored to the destination: %v", pass, err) + } + } +} + +// stagingSeqDigitsOf is the sequence a copy carries in its name. It survives a +// park, so a report assertion keyed on it holds whether the copy is still under +// the staging prefix or has moved under the Kept one. +func stagingSeqDigitsOf(t *testing.T, staging string) string { + t.Helper() + base := filepath.Base(staging) + digits := strings.TrimSuffix(strings.TrimPrefix(base, stagingPrefix), stagingSeqSuffix) + if len(digits) != stagingSeqDigits { + t.Fatalf("staging name %q carries no sequence", base) + } + return digits +} + +// A copy recovery keeps and never names is one an operator cannot find, so the +// report is half of every retain row rather than a nicety. +func assertCopyReported(t *testing.T, logged []string, seq string, pass int) { + t.Helper() + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, seq) }) { + t.Errorf("pass %d: the retained copy (sequence %s) should be named in the report, got %v", pass, seq, logged) + } +} + +// assertCopySet compares every directory under the given prefixes against the +// paths the row expects, resolving both sides through EvalSymlinks because the +// temp root is a symlink on macOS and a path built from t.TempDir would then +// never equal one read back out of the directory. +func assertCopySet(t *testing.T, parent string, prefixes, want []string, pass int) { + t.Helper() + entries, err := os.ReadDir(parent) + if err != nil { + t.Fatal(err) + } + var got []string + for _, entry := range entries { + if !entry.IsDir() { + continue + } + for _, prefix := range prefixes { + if strings.HasPrefix(entry.Name(), prefix) { + got = append(got, resolvedPath(t, filepath.Join(parent, entry.Name()))) + break + } + } + } + wantResolved := make([]string, 0, len(want)) + for _, path := range want { + wantResolved = append(wantResolved, resolvedPath(t, path)) + } + slices.Sort(got) + slices.Sort(wantResolved) + if !slices.Equal(got, wantResolved) { + t.Errorf("pass %d: copies on disk = %v, want %v", pass, got, wantResolved) + } +} + +// resolvedPath resolves symlinks so the two sides of a path comparison are the +// same path. A path that does not exist cannot be resolved, and its cleaned form +// is what the comparison then reports. +func resolvedPath(t *testing.T, path string) string { + t.Helper() + if resolved, err := filepath.EvalSymlinks(path); err == nil { + return resolved + } + return filepath.Clean(path) +} + +// Ownership is a name plus a marker that agrees with it, and neither half alone. +// A directory that only carries the generated name is a legacy work tree or a +// sibling someone renamed, and recovery restoring from one, or reaping one, is +// the whole class of defect the marker exists to close. Each case here is a +// single check dropped: the marker, its kind, its sequence, its destination. +func TestOwnershipCannotBeForgedBySiblingNames(t *testing.T) { + const forgedSeq = 7 + for _, tc := range []struct { + name string + marker *txnMarker + }{ + {name: "no marker at all"}, + {name: "another site's kind", marker: &txnMarker{Kind: "dictation-promote", Dest: "proj-1", Seq: forgedSeq}}, + {name: "sequence disagrees with the name", marker: &txnMarker{Kind: txnKindBundleExtract, Dest: "proj-1", Seq: forgedSeq + 1}}, + {name: "destination is not a link id", marker: &txnMarker{Kind: txnKindBundleExtract, Dest: ".hidden", Seq: forgedSeq}}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + // Built outside and renamed in, so nothing about it was ever written + // by this code: the name is the only thing it shares with a staging + // dir, and it is shaped like a restorable one. + unrelated := filepath.Join(t.TempDir(), "unrelated") + if err := os.MkdirAll(filepath.Join(unrelated, "backup", ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(unrelated, "backup", "a.txt"), []byte("forged"), 0o644); err != nil { + t.Fatal(err) + } + forged := filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, forgedSeq, stagingSeqSuffix)) + if err := os.Rename(unrelated, forged); err != nil { + t.Fatal(err) + } + if tc.marker != nil { + if err := writeMarker(forged, *tc.marker); err != nil { + t.Fatal(err) + } + } + + for pass := 1; pass <= 2; pass++ { + recoverBundleDir(dir, nil) + got, err := os.ReadFile(filepath.Join(forged, "backup", "a.txt")) + if err != nil || string(got) != "forged" { + t.Fatalf("pass %d: the forged directory must be left exactly as it was, got %q (err %v)", pass, got, err) + } + // A restore, a park, or a reap all show up here: any of them + // either creates a destination or moves the directory. + assertNoOtherEntries(t, dir, filepath.Base(forged), lockDirName) + } + }) + } +} + +// assertNoOtherEntries fails when anything but the named entries is in dir. It +// is how "nothing was owned" is asserted without naming every way ownership +// could have been acted on: a restore, a park, and a reap all change this set. +func assertNoOtherEntries(t *testing.T, dir string, allowed ...string) { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !slices.Contains(allowed, entry.Name()) { + t.Errorf("%s should hold only %v, found %s", dir, allowed, entry.Name()) + } + } +} + +// ---- the regression net: recovery's own steps, and a live extract ---------- + +// deleteWatch records every removeAll recovery asks for, and flags the ones no +// classification licenses: a directory that still holds a set-aside tree with no +// commit flag beside it. It is asserted AT THE SEAM rather than off the disk +// because a fault-injected removeAll leaves the tree exactly where a removeAll +// that was never requested does, so a row reading the surviving files could not +// tell a recovery that did the right thing from one that tried to do the wrong +// thing and was stopped by the injected fault. +type deleteWatch struct { + mu sync.Mutex + all []string + denied []string +} + +// install swaps removeAll for a recorder, restoring the seam in t.Cleanup. The +// order matters: only a watch installed AFTER a fault sees the calls that fault +// is failing, and cleanups unwind in reverse, so the seam still ends the test as +// the real filesystem. +func (w *deleteWatch) install(t *testing.T) { + t.Helper() + real := stagingFS + t.Cleanup(func() { stagingFS = real }) + stagingFS.removeAll = func(path string) error { + w.mu.Lock() + w.all = append(w.all, path) + if holdsUncommittedCopy(path) { + w.denied = append(w.denied, path) + } + w.mu.Unlock() + return real.removeAll(path) + } +} + +func (w *deleteWatch) counts() (all, denied []string) { + w.mu.Lock() + defer w.mu.Unlock() + return slices.Clone(w.all), slices.Clone(w.denied) +} + +// holdsUncommittedCopy reports whether path still holds a copy of a tree that +// nothing proves was published over. Recovery may park one of these forever; it +// may never delete one, whatever step just failed. +func holdsUncommittedCopy(path string) bool { + if _, err := os.Stat(filepath.Join(path, "backup")); err != nil { + return false + } + _, err := os.Stat(filepath.Join(path, committedFile)) + return os.IsNotExist(err) +} + +// The watch is the whole regression net below, so it gets its own proof: a net +// that cannot see the delete it exists to catch would make every row under it +// pass for the wrong reason. The three shapes are the two deletes recovery is +// licensed to make and the one it is not. +func TestDeleteWatchSeesOnlyTheDeleteRecoveryMayNotMake(t *testing.T) { + dir := t.TempDir() + uncommitted := stageBackup(t, dir, "", "proj-1", "v1", 100) + committed := stageBackup(t, dir, "", "proj-1", "v2", 200) + markCommitted(t, committed) + scratch := stageScratch(t, dir, "proj-1", 50) + + var watch deleteWatch + func() { + saved := stagingFS + defer func() { stagingFS = saved }() + watch.install(t) + for _, path := range []string{committed, scratch, uncommitted} { + if err := stagingFS.removeAll(path); err != nil { + t.Fatal(err) + } + } + }() + + all, denied := watch.counts() + if len(all) != 3 { + t.Errorf("the watch should have seen all three deletes, got %v", all) + } + if !slices.Equal(denied, []string{uncommitted}) { + t.Errorf("only the uncommitted copy is a violation, got %v", denied) + } +} + +// recoveryStep is one filesystem step of a recovery pass, with the fault that +// fails it. Each is matched where it is reached rather than by call ordinal, so +// a change in how many times recovery stats a directory cannot silently move +// which call the row is failing. +type recoveryStep struct { + name string + // inject installs the row's fault and returns whether that fault was ever + // actually called. The return is not bookkeeping: a row that injects a + // failure into a step the pass never reaches asserts only that a clean run + // works, which every other row already proves. + inject func(t *testing.T, newest string) (fired func() bool) +} + +// firedFault wraps a row's match so the row can report whether its fault was +// reached. A nil match means every call of the step, matching injectFault's own +// contract. +func firedFault(match func(args ...string) bool) (wrapped func(args ...string) bool, fired func() bool) { + var hit atomic.Bool + return func(args ...string) bool { + if match != nil && !match(args...) { + return false + } + hit.Store(true) + return true + }, func() bool { return hit.Load() } +} + +func bundleRecoverySteps() []recoveryStep { + failed := errors.New("injected recovery failure") + simple := func(step string) func(*testing.T, string) func() bool { + return func(t *testing.T, _ string) func() bool { + match, fired := firedFault(nil) + injectFault(t, step, match, failed) + return fired + } + } + return []recoveryStep{ + {name: "readDir", inject: simple("readDir")}, + {name: "stat", inject: simple("stat")}, + {name: "marker read", inject: simple("readFile")}, + { + name: "restore rename", + inject: func(t *testing.T, newest string) func() bool { + match, fired := firedFault(func(args ...string) bool { + return args[0] == filepath.Join(newest, "backup") + }) + injectFault(t, "rename", match, failed) + return fired + }, + }, + {name: "removeAll", inject: simple("removeAll")}, + { + name: "park rename", + inject: func(t *testing.T, _ string) func() bool { + match, fired := firedFault(func(args ...string) bool { + return strings.HasPrefix(filepath.Base(args[1]), keptPrefix) + }) + injectFault(t, "rename", match, failed) + return fired + }, + }, + } +} + +// Every recovery step, failed on pass one, on pass two, and on both. Two claims +// per case, and the first is the one the whole branch exists for: no failure of +// any step ever produces a delete of a copy nothing proves was superseded. The +// second is that a fault only DELAYS recovery: a later pass with the fault gone +// reaches the same terminal state a clean run reaches, so a step that failed +// once has not left the destination in a state recovery can no longer reason +// about. +func TestBundleRecoveryStepFailures(t *testing.T) { + for _, step := range bundleRecoverySteps() { + // A standalone "pass two" variant used to sit between these two. It ran + // pass one clean, which reaches the terminal state, so pass two had no + // candidate left and the fault it injected was never called: the row + // asserted a clean run and nothing else. The both-passes row is the one + // that exercises a fault on pass two, because pass one fails first and + // leaves work behind for it. + for _, when := range []struct { + name string + faulty [2]bool + }{ + {name: "pass one", faulty: [2]bool{true, false}}, + {name: "both passes", faulty: [2]bool{true, true}}, + } { + t.Run(step.name+"/"+when.name, func(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + // Two uncommitted usable copies and one owned scratch dir, with + // the destination gone: the state that drives every step in the + // list through a real decision. One copy is restored, one is + // parked, and the scratch is the only delete on the happy path, + // so a step that fails has something to corrupt. + older := stageBackup(t, dir, "", "proj-1", "v1", 100) + newest := stageBackup(t, dir, "", "proj-1", "v2", 200) + scratch := stageScratch(t, dir, "proj-1", 50) + + var watch deleteWatch + pass := func(faulty bool) func() bool { + saved := stagingFS + defer func() { stagingFS = saved }() + fired := func() bool { return true } + if faulty { + fired = step.inject(t, newest) + } + // After the fault, so the watch sees the calls the fault + // is failing rather than only the ones it lets through. + watch.install(t) + recoverBundleDir(dir, discardLog) + return fired + } + firedOne := pass(when.faulty[0]) + if when.faulty[0] { + if !firedOne() { + // The pass never called the step this row fails, so the + // row ran a clean recovery under a fault that does not + // exist and every assertion below it is vacuous. + t.Fatalf("the injected %s failure was never called on pass one, so this row proves nothing", step.name) + } + if bundleStepTerminal(dir, dest, older, newest, scratch) { + // Same defect from the other side: a fault that was + // called and changed nothing left the pass free to + // finish. + t.Errorf("the injected %s failure left the pass free to finish, so this row proves nothing", step.name) + } + } + firedTwo := pass(when.faulty[1]) + if when.faulty[1] && !firedTwo() { + t.Fatalf("the injected %s failure was never called on pass two, so this row proves nothing", step.name) + } + // The fault is gone, and recovery has to be able to finish from + // wherever the failed passes left the directory. + pass(false) + + all, denied := watch.counts() + if len(denied) > 0 { + t.Errorf("no step failure may produce a delete of an uncommitted copy, got %v", denied) + } + if len(all) == 0 { + t.Fatal("the watch saw no delete at all, so it proves nothing about this pass") + } + assertBundleStepTerminalState(t, dir, dest, older, newest, scratch) + }) + } + } +} + +// bundleStepTerminal reports whether the fixture reached the state a clean run +// reaches. It is a predicate rather than an assertion because it is asked twice +// for opposite reasons: a faulted pass must NOT have got here, and the pass +// after the fault is gone must have. +func bundleStepTerminal(dir, dest, older, newest, scratch string) bool { + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v2" { + return false + } + if _, err := os.Stat(filepath.Join(parkedStaging(older), "backup", "a.txt")); err != nil { + return false + } + for _, gone := range []string{newest, scratch} { + if _, err := os.Stat(gone); !os.IsNotExist(err) { + return false + } + } + return true +} + +// assertBundleStepTerminalState is where the fixture above has to end up once +// nothing is failing: the newest copy live, its own directory gone, the older +// one kept under the Kept prefix with its tree intact, and the scratch dir +// reaped. Content is read back rather than names checked, because a park that +// moved a name and lost the tree passes every name assertion. +func assertBundleStepTerminalState(t *testing.T, dir, dest, older, newest, scratch string) { + t.Helper() + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v2" { + t.Errorf("dest a.txt = %q (err %v), want the newest copy %q", got, err, "v2") + } + parked := parkedStaging(older) + kept, err := os.ReadFile(filepath.Join(parked, "backup", "a.txt")) + if err != nil || string(kept) != "v1" { + t.Errorf("the parked copy's a.txt = %q (err %v), want %q", kept, err, "v1") + } + for _, gone := range []string{newest, scratch} { + if _, err := os.Stat(gone); !os.IsNotExist(err) { + t.Errorf("%s should be gone once recovery finished: %v", gone, err) + } + } + assertCopySet(t, dir, []string{stagingPrefix, keptPrefix}, []string{parked}, 3) +} + +// gateAt parks the write path at one named boundary: it signals when the call +// whose arguments match is reached, holds that call until release, and passes +// every other call through. blockStep gates EVERY call to a step, which for a +// rename would park an extract at the rename inside its marker write rather than +// at the set-aside or the publish, so a matched gate is what puts the writer at +// the boundary a row is actually about. +func gateAt(t *testing.T, match func(from, to string) bool) (reached <-chan struct{}, release func()) { + t.Helper() + real := stagingFS + t.Cleanup(func() { stagingFS = real }) + arrived := make(chan struct{}) + gate := make(chan struct{}) + var arrive, open sync.Once + release = func() { open.Do(func() { close(gate) }) } + t.Cleanup(release) + stagingFS.rename = func(from, to string) error { + if match(from, to) { + arrive.Do(func() { close(arrived) }) + <-gate + } + return real.rename(from, to) + } + return arrived, release +} + +// signalRemoveAll reports when a removeAll of path is reached. It wraps whatever +// is already installed, so layering it over blockStep signals BEFORE the call +// parks rather than after it is released. +func signalRemoveAll(t *testing.T, match func(path string) bool) <-chan struct{} { + t.Helper() + installed := stagingFS + t.Cleanup(func() { stagingFS = installed }) + arrived := make(chan struct{}) + var arrive sync.Once + stagingFS.removeAll = func(p string) error { + if match(p) { + arrive.Do(func() { close(arrived) }) + } + return installed.removeAll(p) + } + return arrived +} + +// A recovery pass and a live extract for one destination, running at the same +// time, at each of the three boundaries where the destination's only copy is in +// a staging dir. Nothing here asserts that recovery wins: the claim is that +// neither side destroys the other's copy, that the extract that was already +// running completes, and that the recovering side either waits or says out loud +// that it skipped. The lock is what makes that true, and this is the only test +// that drives both sides of it at once, under -race, through one package-level +// seam. +func TestBundleRecoveryRacesALivePromotion(t *testing.T) { + for _, tc := range []struct { + name string + // gate parks the extract at this row's boundary and reports when it is + // there. staging is the dir the extract allocated, known only once the + // extract is under way for the reap row, which is why this takes dir. + gate func(t *testing.T, dir, dest string) (reached <-chan struct{}, release func()) + }{ + { + // The prior tree is on its way into the staging dir: for that + // instant the destination still holds it and the staging dir holds + // nothing. + name: "set-aside", + gate: func(t *testing.T, dir, dest string) (<-chan struct{}, func()) { + return gateAt(t, func(from, to string) bool { + return from == dest && filepath.Base(to) == "backup" + }) + }, + }, + { + // The destination is absent and the staging dir holds the only copy + // of the prior tree. A recovering pass that acted here would restore + // that copy over the publish this extract is in the middle of. + name: "publish", + gate: func(t *testing.T, dir, dest string) (<-chan struct{}, func()) { + return gateAt(t, func(from, to string) bool { + return filepath.Base(from) == "repo" && to == dest + }) + }, + }, + { + // The publish landed and the commit flag is written, so the copy in + // the staging dir is superseded and the extract is deleting it. A + // recovering pass that reached the same directory would be deleting + // it too. + name: "reap", + gate: func(t *testing.T, dir, dest string) (<-chan struct{}, func()) { + release := blockStep(t, "removeAll") + // Any staging dir under dir: the extract's reap is the only + // removeAll either side makes on this path, since the seed + // extract finished before the gate was installed. + return signalRemoveAll(t, func(path string) bool { + return filepath.Dir(path) == dir && strings.HasPrefix(filepath.Base(path), stagingPrefix) + }), release + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + // Built before the seam is gated: git writes it through os, but a + // gate installed around a step this still reaches would park the + // fixture rather than the extract under test. + bundle := testBundle(t, "a.txt", "v1") + + var watch deleteWatch + watch.install(t) + reached, release := tc.gate(t, dir, dest) + + var extractErr error + extracted := make(chan struct{}) + go func() { + defer close(extracted) + extractErr = extractBundle(context.Background(), bundle, dest, nil) + }() + <-reached + + var logs []string + recovered := make(chan struct{}) + go func() { + defer close(recovered) + var mu sync.Mutex + recoverBundleDir(dir, func(format string, args ...any) { + mu.Lock() + defer mu.Unlock() + logs = append(logs, fmt.Sprintf(format, args...)) + }) + }() + <-recovered + release() + <-extracted + + if extractErr != nil { + t.Errorf("the extract that was already running must finish: %v", extractErr) + } + // Skipped, not raced: the extract holds the destination's lock from + // before its first rename until after its last delete, and the + // recovering pass is required to say so rather than act. + if !logged(logs, "holds proj-1") { + t.Errorf("the recovering pass should report the destination it skipped, got %v", logs) + } + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v1" { + t.Errorf("dest a.txt = %q (err %v), want the extract's own tree %q", got, err, "v1") + } + if _, denied := watch.counts(); len(denied) > 0 { + t.Errorf("neither side may delete an uncommitted copy, got %v", denied) + } + // The extract cleaned up after itself, and recovery left nothing of + // its own behind. + assertCopySet(t, dir, []string{stagingPrefix, keptPrefix}, nil, 1) + }) + } +} + +// ---- the rows no single crash of the writer can reach ---------------------- + +// The X rows of the Acceptance Examples. They need arrangements the six crash +// shapes cannot produce: a held lock, more than one candidate, a destination +// that exists and cannot serve, a restore that fails, or a legacy directory +// planted by hand. Everything else about them is the crash table's contract: +// the whole set of copies under both prefixes is compared, not one path at a +// time, and every row runs twice because recovery keeps no memory. + +// xFile is the destination after a pass. An empty name means it must be absent, +// which is a real terminal state here and not a missing expectation. +type xFile struct { + name string + content string +} + +// xCopy is one copy and where it must be when the pass ends. final is written +// out rather than derived from a disposition, because these rows retain copies +// under three different names (its own, the Kept one, a freshly allocated one) +// and a derived name would encode the test's guess at the rule the row exists to +// check. An empty final is a delete. +type xCopy struct { + final string + file string + content string +} + +type xWant struct { + live xFile + copies []xCopy + // report is the fragments this pass's report must name. A copy recovery + // keeps and never names is one an operator cannot find. + report []string +} + +type bundleXRow struct { + id string + // arrange plants the state and returns what each pass must produce, plus, + // where the row needs one, the fault or held lock that makes the row's + // situation happen during a given pass. They come back together so both can + // close over the paths the arrangement planted. + arrange func(t *testing.T, dir, dest string) (want func(pass int) xWant, impede func(t *testing.T, pass int) func()) +} + +func stagingNamed(dir string, seq int64) string { + return filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, seq, stagingSeqSuffix)) +} + +func keptNamed(dir string, seq int64) string { + return filepath.Join(dir, fmt.Sprintf("%s%020d%s", keptPrefix, seq, stagingSeqSuffix)) +} + +// bothPasses is the common case: a row whose two passes look identical, because +// pass one reached a terminal state and pass two has nothing left to do. +func bothPasses(w xWant) func(int) xWant { + return func(int) xWant { return w } +} + +// faultDuring installs one fault for the length of a pass and returns the undo. +// The pass-scoped restore is the point: a row that fails a step on pass one only +// needs the seam back before pass two runs. +func faultDuring(t *testing.T, step string, match func(args ...string) bool) func() { + t.Helper() + saved := stagingFS + injectFault(t, step, match, errors.New("injected recovery failure")) + return func() { stagingFS = saved } +} + +func bundleXRows() []bundleXRow { + return []bundleXRow{ + { + // A destination that exists and cannot serve is not a reason to keep + // a usable copy out of it, and the husk is not this code's to delete + // either: it moves into a fresh sequenced directory of its own and + // is parked from there. + id: "X1", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + stageBackup(t, dir, "", "proj-1", "v1", 100) + plantUnusableDest(t, dir, "proj-1", "husk") + return func(pass int) xWant { + w := xWant{ + live: xFile{"a.txt", "v1"}, + copies: []xCopy{{final: keptNamed(dir, 101), file: filepath.Join("backup", "husk.txt"), content: "husk"}}, + } + if pass == 1 { + w.report = []string{"keeping the tree that could not serve"} + } + return w + }, nil + }, + }, + { + // A Kept backup is permanent. A later publish is not evidence about + // it: the commit flag goes into the copy the publishing transaction + // set aside, and a Kept backup is never that copy. + id: "X2", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + plantUsableDest(t, dir, "proj-1", "live") + kept := plantKeptBackup(t, dir, "proj-1", "unproven", 500) + return bothPasses(xWant{ + live: xFile{"a.txt", "live"}, + copies: []xCopy{{final: kept, file: filepath.Join("backup", "a.txt"), content: "unproven"}}, + }), nil + }, + }, + { + // The newest usable copy cannot be put back. Falling through to the + // older one would put a tree at the destination that the next pass + // reads as having published over the newer copy, which is how a + // retained copy turns into a deleted one two passes later. + id: "X3", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + older := stageBackup(t, dir, "", "proj-1", "v1", 100) + newest := stageBackup(t, dir, "", "proj-1", "v2", 200) + want := bothPasses(xWant{ + copies: []xCopy{ + {final: older, file: filepath.Join("backup", "a.txt"), content: "v1"}, + {final: newest, file: filepath.Join("backup", "a.txt"), content: "v2"}, + }, + report: []string{newest}, + }) + return want, func(t *testing.T, pass int) func() { + return faultDuring(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(newest, "backup") + }) + } + }, + }, + { + // Unusable is a selection decision, not a delete: the newest copy is + // skipped for one that can serve and is then kept like any other + // copy nothing proves was superseded. + id: "X4", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + stageBackup(t, dir, "", "proj-1", "v1", 100) + newest := stageBackup(t, dir, "", "proj-1", "v2", 200) + if err := os.RemoveAll(filepath.Join(newest, "backup", ".git")); err != nil { + t.Fatal(err) + } + return func(pass int) xWant { + w := xWant{ + live: xFile{"a.txt", "v1"}, + copies: []xCopy{{final: parkedStaging(newest), file: filepath.Join("backup", "a.txt"), content: "v2"}}, + } + if pass == 1 { + w.report = []string{newest} + } + return w + }, nil + }, + }, + { + // Unreadable is not unusable. A filesystem fault says nothing about + // the copy, so the whole destination stops rather than ruling on the + // copies that could be read while the newest one could not. + id: "X5", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + cand := stageBackup(t, dir, "", "proj-1", "v1", 100) + want := bothPasses(xWant{ + copies: []xCopy{{final: cand, file: filepath.Join("backup", "a.txt"), content: "v1"}}, + report: []string{cand}, + }) + return want, func(t *testing.T, pass int) func() { + return faultDuring(t, "stat", func(args ...string) bool { + return args[0] == filepath.Join(cand, "backup") + }) + } + }, + }, + { + // A destination someone else holds is one recovery knows nothing + // about: on disk a live extract mid-swap and a crashed one are the + // same thing, and the lock is the only thing that tells them apart. + id: "X6", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + cand := stageBackup(t, dir, "", "proj-1", "v1", 100) + want := func(pass int) xWant { + if pass == 1 { + return xWant{ + copies: []xCopy{{final: cand, file: filepath.Join("backup", "a.txt"), content: "v1"}}, + report: []string{"holds proj-1"}, + } + } + return xWant{live: xFile{"a.txt", "v1"}} + } + return want, func(t *testing.T, pass int) func() { + if pass != 1 { + return func() {} + } + return lockExtract(dest) + } + }, + }, + { + // A published work tree can sit under the exact generated name, so + // the marker beside it proves nothing: the work-tree veto runs first + // and the directory is left alone whatever the marker says. + id: "X7", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + legacy := stagingNamed(dir, 300) + if err := os.MkdirAll(filepath.Join(legacy, ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(legacy, "a.txt"), []byte("legacy"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeMarker(legacy, txnMarker{Kind: txnKindBundleExtract, Dest: "proj-1", Seq: 300}); err != nil { + t.Fatal(err) + } + return bothPasses(xWant{ + copies: []xCopy{{final: legacy, file: "a.txt", content: "legacy"}}, + report: []string{"holds a work tree"}, + }), nil + }, + }, + { + // A name that shares the prefix and fails the grammar carries no + // order at all, so it is not a candidate for anything: not restored, + // not parked, not deleted. It is still reported, because at this + // site a dot-prefixed sibling nothing attributes is residue an + // operator has to be able to see. + id: "X8", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + sibling := stageBackup(t, dir, "foo", "proj-1", "sib", 0) + return bothPasses(xWant{ + copies: []xCopy{{final: sibling, file: filepath.Join("backup", "a.txt"), content: "sib"}}, + report: []string{"does not carry a name this code writes"}, + }), nil + }, + }, + { + // The destination went away outside this code and the only copy left + // carries a commit flag. Committed is second in selection, never + // excluded from it: a copy that was superseded by a tree that is no + // longer there is still a copy of something. + id: "X9", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + markCommitted(t, stageBackup(t, dir, "", "proj-1", "v1", 100)) + return bothPasses(xWant{live: xFile{"a.txt", "v1"}}), nil + }, + }, + { + // Two copies, one destination: the newest goes back and the older is + // kept rather than deleted, because nothing proves anything + // published over it either. + id: "X10", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + older := stageBackup(t, dir, "", "proj-1", "v1", 100) + stageBackup(t, dir, "", "proj-1", "v2", 200) + return func(pass int) xWant { + w := xWant{ + live: xFile{"a.txt", "v2"}, + copies: []xCopy{{final: parkedStaging(older), file: filepath.Join("backup", "a.txt"), content: "v1"}}, + } + if pass == 1 { + w.report = []string{older} + } + return w + }, nil + }, + }, + { + // v0.8.0 staged under os.MkdirTemp, whose suffix carries no sequence + // at all, and a hand-planted name can fail the grammar in the other + // direction. Neither is orderable, so both are retained in place and + // reported rather than guessed at. + id: "X11", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + residue := stageBackup(t, dir, "3921749", "proj-1", "resid", 0) + tied := stageBackup(t, dir, "0000000000000000000x"+stagingSeqSuffix, "proj-1", "tied", 0) + return bothPasses(xWant{ + copies: []xCopy{ + {final: residue, file: filepath.Join("backup", "a.txt"), content: "resid"}, + {final: tied, file: filepath.Join("backup", "a.txt"), content: "tied"}, + }, + report: []string{"does not carry a name this code writes"}, + }), nil + }, + }, + { + // Same as X1 with the only candidate committed. The husk still moves + // aside first, and the copy that replaces it is the committed one, + // because there is no uncommitted copy to prefer. + id: "X12", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + markCommitted(t, stageBackup(t, dir, "", "proj-1", "v1", 100)) + plantUnusableDest(t, dir, "proj-1", "husk") + return func(pass int) xWant { + w := xWant{ + live: xFile{"a.txt", "v1"}, + copies: []xCopy{{final: keptNamed(dir, 101), file: filepath.Join("backup", "husk.txt"), content: "husk"}}, + } + if pass == 1 { + w.report = []string{"keeping the tree that could not serve"} + } + return w + }, nil + }, + }, + { + // Nothing beside it can replace the husk, so it is not taken apart: + // an operator with an unusable destination has strictly more than + // one with no destination at all. + id: "X13", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + cand := stageBackup(t, dir, "", "proj-1", "v1", 100) + if err := os.RemoveAll(filepath.Join(cand, "backup", ".git")); err != nil { + t.Fatal(err) + } + plantUnusableDest(t, dir, "proj-1", "husk") + return func(pass int) xWant { + w := xWant{ + live: xFile{"husk.txt", "husk"}, + copies: []xCopy{{final: parkedStaging(cand), file: filepath.Join("backup", "a.txt"), content: "v1"}}, + } + if pass == 1 { + w.report = []string{"no usable staged copy"} + } + return w + }, nil + }, + }, + { + // The husk was already aside when the restore failed, so recovery + // owes the destination its husk back: the row's whole claim is that + // the destination ends the pass exactly as it was found, with the + // candidate still where it was and the set-aside directory, now + // empty, gone. + id: "X14", + arrange: func(t *testing.T, dir, dest string) (func(int) xWant, func(*testing.T, int) func()) { + cand := stageBackup(t, dir, "", "proj-1", "v1", 100) + plantUnusableDest(t, dir, "proj-1", "husk") + want := bothPasses(xWant{ + live: xFile{"husk.txt", "husk"}, + copies: []xCopy{{final: cand, file: filepath.Join("backup", "a.txt"), content: "v1"}}, + report: []string{cand}, + }) + return want, func(t *testing.T, pass int) func() { + return faultDuring(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(cand, "backup") && args[1] == dest + }) + } + }, + }, + } +} + +func TestBundleXMatrix(t *testing.T) { + for _, row := range bundleXRows() { + t.Run(row.id, func(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + want, impede := row.arrange(t, dir, dest) + + var watch deleteWatch + for pass := 1; pass <= 2; pass++ { + var logs []string + func() { + saved := stagingFS + defer func() { stagingFS = saved }() + if impede != nil { + defer impede(t, pass)() + } + // After the impediment, so a delete the injected fault + // would have failed is still seen as a delete that was + // asked for. + watch.install(t) + logs = recoverAndLog(t, dir) + }() + assertBundleXState(t, dir, dest, want(pass), logs, pass) + } + if _, denied := watch.counts(); len(denied) > 0 { + t.Errorf("no row may delete a copy nothing proves was superseded, got %v", denied) + } + }) + } +} + +// assertBundleXState asserts a row's whole terminal state: the destination, the +// content of every copy at the name it is supposed to be at, the full set of +// directories under both prefixes, and the report. Content rather than names +// alone, because a park that moved a name and lost the tree passes every name +// assertion there is. +func assertBundleXState(t *testing.T, dir, dest string, want xWant, logs []string, pass int) { + t.Helper() + if want.live.name == "" { + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Errorf("pass %d: %s should be absent: %v", pass, dest, err) + } + } else { + got, err := os.ReadFile(filepath.Join(dest, want.live.name)) + if err != nil || string(got) != want.live.content { + t.Errorf("pass %d: dest %s = %q (err %v), want %q", pass, want.live.name, got, err, want.live.content) + } + } + var paths []string + for _, c := range want.copies { + if c.final == "" { + continue + } + paths = append(paths, c.final) + got, err := os.ReadFile(filepath.Join(c.final, c.file)) + if err != nil || string(got) != c.content { + t.Errorf("pass %d: %s = %q (err %v), want %q", pass, filepath.Join(c.final, c.file), got, err, c.content) + } + } + assertCopySet(t, dir, []string{stagingPrefix, keptPrefix}, paths, pass) + for _, fragment := range want.report { + if !logged(logs, fragment) { + t.Errorf("pass %d: the report should name %q, got %v", pass, fragment, logs) + } + } +} diff --git a/internal/daemon/remote/bundle_test.go b/internal/daemon/remote/bundle_test.go index 7f711a3bd..0704f8401 100644 --- a/internal/daemon/remote/bundle_test.go +++ b/internal/daemon/remote/bundle_test.go @@ -2,11 +2,26 @@ package remote import ( "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "math" "os" "os/exec" "path/filepath" + "reflect" "runtime" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" "testing" + "time" + + "github.com/Gitlawb/zero/internal/lockutil" ) // initTestRepo creates a temp git work tree with one committed file and returns @@ -191,3 +206,3227 @@ func TestBridgeBundleRejectsBadToken(t *testing.T) { t.Fatal("bundle upload with a bad token must be refused") } } + +// testBundle commits content to a throwaway repo and returns a bundle of it. +func testBundle(t *testing.T, file, content string) string { + t.Helper() + repo := initTestRepo(t, file, content) + out := filepath.Join(t.TempDir(), "b.bundle") + ctx, cancel := context.WithTimeout(context.Background(), gitTimeout) + defer cancel() + if err := gitBundleCreate(ctx, repo, out); err != nil { + t.Fatalf("bundle create: %v", err) + } + return out +} + +// An extract that cannot clear the live tree must leave that tree alone. The old +// code removed dest before it had anything to publish, so a partial removal -- +// here a subdirectory the daemon cannot delete -- destroyed the prior extraction +// and the deferred staging cleanup then deleted the replacement. +func TestExtractBundleKeepsPriorTreeWhenLiveTreeCannotBeCleared(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root ignores the directory permissions this test relies on") + } + if runtime.GOOS == "windows" { + t.Skip("POSIX directory permissions") + } + ctx := context.Background() + dest := filepath.Join(t.TempDir(), "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + locked := filepath.Join(dest, "locked") + if err := os.MkdirAll(locked, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(locked, "keep.txt"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chmod(locked, 0o500); err != nil { + t.Fatal(err) + } + // The publish moves the old tree (locked subdir included) into staging, so + // restore write permission wherever it ended up or TempDir cleanup fails. + t.Cleanup(func() { + _ = filepath.WalkDir(filepath.Dir(dest), func(path string, d os.DirEntry, err error) error { + if err == nil && d.IsDir() { + _ = os.Chmod(path, 0o700) + } + return nil + }) + }) + + var logged []string + logf := func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) } + if err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, logf); err != nil { + t.Fatalf("extract over an undeletable subtree: %v", err) + } + // The prior tree moved into staging, so the cleanup cannot delete it either. + // That strands a whole copy of the repo and must not pass silently. + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "staging dir") }) { + t.Errorf("a staging dir that could not be removed was not reported: %v", logged) + } + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil { + t.Fatalf("dest holds neither the old tree nor the new one: %v", err) + } + if string(got) != "v1" { + t.Fatalf("a.txt = %q, want the newly published %q", got, "v1") + } +} + +// Every bundle upload is handled in its own goroutine, so two uploads of one +// link id can extract at the same time. The old code let them delete and rename +// over each other: extracts failed with "directory not empty" and one call's +// removal could wipe a tree another had already published. +func TestExtractBundleConcurrentSameDestAlwaysLeavesATree(t *testing.T) { + ctx := context.Background() + dest := filepath.Join(t.TempDir(), "proj-1") + first := testBundle(t, "a.txt", "v0") + second := testBundle(t, "a.txt", "v1") + if err := extractBundle(ctx, first, dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + // The unsafe window is the two back-to-back renames after the clone. Without + // this delay it is narrow enough that the test still passes a good fraction + // of the time with the lock removed, which would make it a guard in name only. + real := stagingFS.rename + stagingFS.rename = func(from, to string) error { + time.Sleep(2 * time.Millisecond) + return real(from, to) + } + t.Cleanup(func() { stagingFS.rename = real }) + + var wg sync.WaitGroup + var mu sync.Mutex + var failures []error + for round := 0; round < 15; round++ { + for _, src := range []string{first, second} { + wg.Add(1) + go func(src string) { + defer wg.Done() + if err := extractBundle(ctx, src, dest, nil); err != nil { + mu.Lock() + failures = append(failures, err) + mu.Unlock() + } + }(src) + } + wg.Wait() + } + + if len(failures) > 0 { + t.Errorf("concurrent extracts of one link id failed: %v", failures) + } + if _, err := os.Stat(filepath.Join(dest, ".git")); err != nil { + t.Errorf("dest is not a work tree after concurrent extracts: %v", err) + } + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil { + t.Fatalf("dest holds no extraction: %v", err) + } + if string(got) != "v0" && string(got) != "v1" { + t.Fatalf("a.txt = %q, want one of the uploaded trees", got) + } +} + +// Staging directories live beside dest under a reserved dot prefix, so a link id +// must never be able to name one. +func TestSanitizeLinkIDRejectsDotPrefixedIDs(t *testing.T) { + for _, id := range []string{".", "..", ".staging-1", ".staging-abc", ".git", "..foo", ".hidden"} { + if _, err := sanitizeLinkID(id); err == nil { + t.Errorf("sanitizeLinkID(%q) was accepted; it can collide with a staging dir", id) + } + } + for _, id := range []string{"proj-1", "a.b", "x_1", "A1", "repo.git"} { + if _, err := sanitizeLinkID(id); err != nil { + t.Errorf("sanitizeLinkID(%q) = %v, want accepted", id, err) + } + } +} + +// The publish rename is the one step whose failure the old code could not come +// back from: dest was already deleted. It must now put the prior tree back. +func TestExtractBundleRestoresPriorTreeWhenPublishFails(t *testing.T) { + ctx := context.Background() + dest := filepath.Join(t.TempDir(), "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + // Fail only the publish, so the restore rename that follows it still runs. + // The clone is what moves in from repo; the set-aside and the restore both + // have to stay real for the prior tree to come back. + real := stagingFS.rename + stagingFS.rename = func(from, to string) error { + if filepath.Base(from) == "repo" { + return errors.New("injected publish failure") + } + return real(from, to) + } + t.Cleanup(func() { stagingFS.rename = real }) + + if err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, nil); err == nil { + t.Fatal("a failed publish must be reported") + } + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil { + t.Fatalf("prior extraction was not restored: %v", err) + } + if string(got) != "v0" { + t.Fatalf("a.txt = %q, want the prior %q", got, "v0") + } +} + +// If the publish fails and the prior tree cannot be put back either, dest is +// empty and the backup is the only copy left. It must survive the cleanup so an +// operator can recover it by hand. +func TestExtractBundleKeepsBackupWhenRestoreAlsoFails(t *testing.T) { + ctx := context.Background() + bundleDir := t.TempDir() + dest := filepath.Join(bundleDir, "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + // The publish and the restore are the two renames into dest; the set-aside + // that fills the backup stays real, or there is no retained copy to assert on. + real := stagingFS.rename + stagingFS.rename = func(from, to string) error { + if to == dest { + return errors.New("injected rename failure") + } + return real(from, to) + } + t.Cleanup(func() { stagingFS.rename = real }) + + err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, nil) + if err == nil { + t.Fatal("a failed publish must be reported") + } + if !strings.Contains(err.Error(), "prior tree left in") { + t.Errorf("error should point at the retained backup, got: %v", err) + } + + entries, readErr := os.ReadDir(bundleDir) + if readErr != nil { + t.Fatal(readErr) + } + found := "" + for _, e := range entries { + if strings.HasPrefix(e.Name(), stagingPrefix) { + if _, statErr := os.Stat(filepath.Join(bundleDir, e.Name(), "backup", "a.txt")); statErr == nil { + found = e.Name() + } + } + } + if found == "" { + t.Fatal("the prior tree was deleted along with the staging dir") + } + got, readErr := os.ReadFile(filepath.Join(bundleDir, found, "backup", "a.txt")) + if readErr != nil || string(got) != "v0" { + t.Fatalf("retained backup = %q, err %v, want %q", got, readErr, "v0") + } + + // The retained tree carries its link marker, so the next bridge start puts + // it back rather than leaving the link empty forever. + stagingFS.rename = real + recoverBundleDir(bundleDir, nil) + got, readErr = os.ReadFile(filepath.Join(dest, "a.txt")) + if readErr != nil { + t.Fatalf("recovery did not restore the retained tree: %v", readErr) + } + if string(got) != "v0" { + t.Fatalf("recovered a.txt = %q, want %q", got, "v0") + } +} + +// soleStaging returns the one staging dir under dir, and fails if there is not +// exactly one. A test that asserts on "the" staging dir has to prove that is +// what it found, or a second one left by an earlier step reads as a pass. +func soleStaging(t *testing.T, dir string) string { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + found := []string{} + for _, e := range entries { + if e.IsDir() && strings.HasPrefix(e.Name(), stagingPrefix) { + found = append(found, filepath.Join(dir, e.Name())) + } + } + if len(found) != 1 { + t.Fatalf("staging dirs in %s = %v, want exactly one", dir, found) + } + return found[0] +} + +// The marker is what lets recovery name the copy a crash leaves behind, so it +// has to be on disk before anything destructive runs. A clone that fails is the +// first step after it, and the marker must already be readable there. +func TestExtractBundleWritesTheMarkerBeforeTheClone(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + // The staging dir is cleaned up on the way out of a failed extract, so the + // cleanup is blocked to leave the on-disk state this test is about. + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + err := extractBundle(context.Background(), filepath.Join(dir, "missing.bundle"), dest, nil) + if err == nil { + t.Fatal("cloning a bundle that does not exist must fail") + } + + staging := soleStaging(t, dir) + seq, ok := stagingStamp(filepath.Base(staging)) + if !ok { + t.Fatalf("staging name %s carries no sequence", staging) + } + m, err := readMarker(staging) + if err != nil { + t.Fatalf("read the marker the extract should have written: %v", err) + } + if want := (txnMarker{Kind: txnKindBundleExtract, Dest: "proj-1", Seq: seq}); m != want { + t.Fatalf("marker = %+v, want %+v", m, want) + } + if _, err := os.Stat(filepath.Join(staging, "backup")); !os.IsNotExist(err) { + t.Errorf("nothing should have been set aside before the clone, got %v", err) + } +} + +// A torn marker occupies the name recovery reads and parses as nothing, so the +// bytes reach their name through a rename or not at all. +func TestExtractBundleMarkerIsAtomic(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + injected := errors.New("injected marker rename failure") + injectFault(t, "rename", func(args ...string) bool { return filepath.Base(args[1]) == stagingMarkerFile }, injected) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, nil) + if !errors.Is(err, injected) { + t.Fatalf("extract = %v, want the injected marker rename failure", err) + } + + staging := soleStaging(t, dir) + entries, readErr := os.ReadDir(staging) + if readErr != nil { + t.Fatal(readErr) + } + for _, e := range entries { + if e.Name() == stagingMarkerFile { + t.Errorf("a marker that never got renamed must not occupy %s", filepath.Join(staging, stagingMarkerFile)) + } + } + got, readErr := os.ReadFile(filepath.Join(dest, "a.txt")) + if readErr != nil || string(got) != "v0" { + t.Fatalf("dest a.txt = %q, err %v, want the prior tree %q untouched", got, readErr, "v0") + } + if _, err := os.Stat(filepath.Join(staging, "backup")); !os.IsNotExist(err) { + t.Errorf("a failed marker must stop the extract before the set-aside, got %v", err) + } +} + +// The flag is the evidence a later recovery needs that the copy in backup was +// superseded. It only means that if the publish already landed. +func TestExtractBundleCreatesTheCommitFlagAfterPublish(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + // Blocking the final cleanup is what leaves the committed staging dir on + // disk to assert on; a cleanup failure is logged, never returned. + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + if err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, nil); err != nil { + t.Fatalf("extract = %v, want the cleanup failure to be logged, not returned", err) + } + + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("dest a.txt = %q, err %v, want %q", got, err, "v1") + } + staging := soleStaging(t, dir) + got, err = os.ReadFile(filepath.Join(staging, "backup", "a.txt")) + if err != nil || string(got) != "v0" { + t.Fatalf("backup a.txt = %q, err %v, want the prior tree %q", got, err, "v0") + } + if _, err := os.Stat(filepath.Join(staging, committedFile)); err != nil { + t.Fatalf("a published extract must record the commit flag: %v", err) + } +} + +// Without the flag the copy in backup has no proof it was superseded, so the +// staging dir stays: deleting it would drop the only evidence recovery has. +func TestExtractBundleFailedCommitFlagKeepsTheStagingDir(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + injectFault(t, "create", func(args ...string) bool { return filepath.Base(args[0]) == committedFile }, errors.New("injected commit flag failure")) + var logged []string + err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, func(format string, args ...any) { + logged = append(logged, fmt.Sprintf(format, args...)) + }) + if err != nil { + t.Fatalf("extract = %v, want a committed publish reported as success", err) + } + + got, readErr := os.ReadFile(filepath.Join(dest, "a.txt")) + if readErr != nil || string(got) != "v1" { + t.Fatalf("dest a.txt = %q, err %v, want the published tree %q", got, readErr, "v1") + } + staging := soleStaging(t, dir) + got, readErr = os.ReadFile(filepath.Join(staging, "backup", "a.txt")) + if readErr != nil || string(got) != "v0" { + t.Fatalf("backup a.txt = %q, err %v, want the retained prior tree %q", got, readErr, "v0") + } + if _, err := os.Stat(filepath.Join(staging, committedFile)); !os.IsNotExist(err) { + t.Errorf("the flag failed to be created, so it must not exist: %v", err) + } + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, staging) }) { + t.Errorf("the retained staging dir should be named in the log, got %v", logged) + } +} + +// A publish that never landed supersedes nothing, so no flag may appear beside +// the copy that is still the live tree's only other copy. +func TestExtractBundlePublishFailureLeavesNoCommitFlag(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + injected := errors.New("injected publish failure") + injectFault(t, "rename", func(args ...string) bool { return filepath.Base(args[0]) == "repo" }, injected) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, nil) + if !errors.Is(err, injected) { + t.Fatalf("extract = %v, want the injected publish failure", err) + } + + got, readErr := os.ReadFile(filepath.Join(dest, "a.txt")) + if readErr != nil || string(got) != "v0" { + t.Fatalf("dest a.txt = %q, err %v, want the restored prior tree %q", got, readErr, "v0") + } + staging := soleStaging(t, dir) + if _, err := os.Stat(filepath.Join(staging, committedFile)); !os.IsNotExist(err) { + t.Errorf("a failed publish must leave no commit flag, got %v", err) + } +} + +// Every rename goes through fsutil.RenameWithRetry, which absorbs the momentary +// sharing violation an open file produces on Windows. The retry is Windows-only +// (fsutil/rename.go: ten attempts, 10 ms apart, guarded by runtime.GOOS), so on +// every other platform the assertion is only that the error comes back +// unchanged. This guard is falsifiable on Windows CI alone. +func TestExtractBundleRenamesRetryOnTransientErrors(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(ctx, testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + // The publish is matched by its arguments, not by a call ordinal: the marker + // write and the set-aside also rename, and an ordinal leaves the test equally + // green when it lands on one of those instead, so nothing would pin the step. + // Only the first attempt fails, because the Windows retry needs a later one + // to succeed on. + injected := &os.LinkError{Op: "rename", Old: "repo", New: dest, Err: syscall.Errno(32)} + var fired atomic.Bool + injectFault(t, "rename", func(args ...string) bool { + if args[1] != dest || filepath.Base(args[0]) != "repo" { + return false + } + return fired.CompareAndSwap(false, true) + }, injected) + + err := extractBundle(ctx, testBundle(t, "a.txt", "v1"), dest, nil) + want := "v0" + if runtime.GOOS == "windows" { + if err != nil { + t.Fatalf("extract = %v, want the sharing violation absorbed by the retry", err) + } + want = "v1" + } else if !errors.Is(err, injected) { + t.Fatalf("extract = %v, want the sharing violation returned unchanged", err) + } + got, readErr := os.ReadFile(filepath.Join(dest, "a.txt")) + if readErr != nil || string(got) != want { + t.Fatalf("dest a.txt = %q, err %v, want %q", got, readErr, want) + } + if !fired.Load() { + t.Fatal("the publish rename was never reached, so this row proves nothing about the retry") + } +} + +// Missing and unreadable are opposite decisions for recovery: the first says the +// directory was never this code's to act on, the second says the filesystem +// failed and nothing may be concluded. Folding one into the other is what turns +// a fault into a licence. +func TestReadMarkerDistinguishesMissingFromUnreadable(t *testing.T) { + dir := t.TempDir() + + if _, err := readMarker(dir); !errors.Is(err, errMarkerMissing) { + t.Fatalf("readMarker of a dir with no marker = %v, want errMarkerMissing", err) + } + + // What a released version left at this name is not JSON; it parses as + // nothing, which is a marker that exists and cannot be trusted. + if err := os.WriteFile(filepath.Join(dir, stagingMarkerFile), []byte("proj-1"), 0o600); err != nil { + t.Fatal(err) + } + _, err := readMarker(dir) + if err == nil { + t.Fatal("an unparseable marker must be an error") + } + if errors.Is(err, errMarkerMissing) { + t.Fatalf("an unparseable marker = %v, want an error distinct from errMarkerMissing", err) + } + + injected := errors.New("injected marker read failure") + injectFault(t, "readFile", nil, injected) + _, err = readMarker(dir) + if !errors.Is(err, injected) { + t.Fatalf("readMarker over a failing read = %v, want the injected failure", err) + } + if errors.Is(err, errMarkerMissing) { + t.Fatalf("a read failure = %v, want an error distinct from errMarkerMissing", err) + } +} + +// The link marker is read off disk, so a hostile or corrupt one must not steer a +// rename anywhere outside the bundle dir. +func TestRecoverBundleDirRefusesAMarkerThatEscapesTheBundleDir(t *testing.T) { + for _, marker := range []string{"../evil", "/etc/evil", "..", "a/b", ".hidden"} { + dir := t.TempDir() + staging := plantInterruptedExtract(t, dir, "proj-1", "v0") + // Only the destination is hostile: a marker whose sequence disagrees + // with its own name is dropped before the escape check ever runs, and + // the test would then prove nothing about that check. + seq, _ := stagingStamp(filepath.Base(staging)) + if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: marker, Seq: seq}); err != nil { + t.Fatal(err) + } + outside := filepath.Join(filepath.Dir(dir), "evil") + + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }) + + if _, err := os.Stat(outside); !os.IsNotExist(err) { + t.Errorf("marker %q created %s: %v", marker, outside, err) + } + if _, err := os.Stat(filepath.Join(staging, "backup", "a.txt")); err != nil { + t.Errorf("marker %q: the tree should be left in place, not moved: %v", marker, err) + } + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "leaving it in place") }) { + t.Errorf("marker %q should be reported, got %v", marker, logged) + } + } +} + +// plantInterruptedExtract builds the on-disk state a crash between the two swap +// renames leaves behind: the link's only tree sitting in a staging dir. +func plantInterruptedExtract(t *testing.T, bundleDir, linkID, content string) string { + t.Helper() + // The name has to pass the strict grammar and the marker has to agree with + // it, or recovery reads the fixture as unowned and every assertion built on + // it passes for the wrong reason. + seq, err := nextStagingSeq(bundleDir) + if err != nil { + t.Fatal(err) + } + return stageBackup(t, bundleDir, "", linkID, content, seq) +} + +func TestRecoverBundleDirRestoresInterruptedExtract(t *testing.T) { + dir := t.TempDir() + staging := plantInterruptedExtract(t, dir, "proj-1", "v0") + + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }) + + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil { + t.Fatalf("the interrupted extract was not restored: %v", err) + } + if string(got) != "v0" { + t.Fatalf("restored a.txt = %q, want %q", got, "v0") + } + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Errorf("staging should be gone after a successful restore, got %v", err) + } + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "restored the work tree") }) { + t.Errorf("a restore should be reported, got %v", logged) + } +} + +// A copy the live tree provably published over is reclaimed. "Provably" is the +// commit flag inside that copy and a destination that can actually serve, not +// the destination merely existing: that reading deletes the last copy of a tree +// whenever the destination came from anywhere else. +func TestRecoverBundleDirDropsBackupWhenTheLinkAlreadyHasATree(t *testing.T) { + dir := t.TempDir() + live := plantUsableDest(t, dir, "proj-1", "live") + staging := plantInterruptedExtract(t, dir, "proj-1", "stale") + markCommitted(t, staging) + + recoverBundleDir(dir, nil) + + got, err := os.ReadFile(filepath.Join(live, "a.txt")) + if err != nil || string(got) != "live" { + t.Fatalf("the live tree must win: got %q err %v", got, err) + } + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Errorf("a superseded backup should be removed, got %v", err) + } +} + +func TestRecoverBundleDirKeepsABackupItCannotAttribute(t *testing.T) { + dir := t.TempDir() + staging := plantInterruptedExtract(t, dir, "proj-1", "v0") + if err := os.Remove(filepath.Join(staging, stagingMarkerFile)); err != nil { + t.Fatal(err) + } + + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }) + + if _, err := os.Stat(filepath.Join(staging, "backup", "a.txt")); err != nil { + t.Errorf("an unattributable tree must be left alone, not deleted: %v", err) + } + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "no usable transaction marker") }) { + t.Errorf("an unattributable tree should be reported, got %v", logged) + } +} + +func TestRecoverBundleDirLeavesAStagingDirAnExtractCouldStillOwn(t *testing.T) { + dir := t.TempDir() + staging := filepath.Join(dir, stagingPrefix+"live") + if err := os.MkdirAll(filepath.Join(staging, "repo"), 0o700); err != nil { + t.Fatal(err) + } + + recoverBundleDir(dir, nil) + + if _, err := os.Stat(staging); err != nil { + t.Errorf("a fresh staging dir may belong to a running clone: %v", err) + } +} + +// A second daemon sharing the bundle dir must not extract the same link at the +// same time; the in-process lock cannot see it, so an advisory file lock does. +func TestExtractBundleWaitsForACrossProcessLock(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + b := testBundle(t, "a.txt", "v0") + + lockDir := filepath.Join(dir, lockDirName) + if err := os.MkdirAll(lockDir, 0o700); err != nil { + t.Fatal(err) + } + held, err := lockutil.TryAcquireFileLockAt(dir, filepath.Join(lockDir, "proj-1.lock")) + if err != nil { + t.Fatalf("take the lock as the other daemon would: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + if err := extractBundle(ctx, b, dest, nil); err == nil { + t.Fatal("an extract must not proceed while another process holds the link") + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Errorf("a blocked extract must not touch dest, got %v", err) + } + + // Once the other holder is gone the same extract goes through. + if err := held.Release(); err != nil { + t.Fatal(err) + } + if err := extractBundle(context.Background(), b, dest, nil); err != nil { + t.Fatalf("extract after the lock was released: %v", err) + } + if _, err := os.Stat(filepath.Join(dest, "a.txt")); err != nil { + t.Errorf("extract did not publish: %v", err) + } +} + +// End to end over TLS: two clients uploading the same link id at once must both +// succeed and leave one valid work tree. +func TestBridgeConcurrentUploadsOfOneLinkID(t *testing.T) { + srv := newBridgeServer(t, staticLauncher()) + auth, _ := NewTokenAuthenticator("tok") + bundleRoot := t.TempDir() + addr, ca := startBridge(t, srv, BridgeOptions{Authenticator: auth, BundleDir: bundleRoot}) + + first := initTestRepo(t, "a.txt", "v0") + second := initTestRepo(t, "a.txt", "v1") + cfg := RemoteConfig{Address: addr, Token: "tok", CACertFile: ca} + + var wg sync.WaitGroup + var mu sync.Mutex + var failures []error + for _, repo := range []string{first, second, first, second} { + wg.Add(1) + go func(repo string) { + defer wg.Done() + if _, err := UploadRepoBundle(cfg, repo, "proj-1"); err != nil { + mu.Lock() + failures = append(failures, err) + mu.Unlock() + } + }(repo) + } + wg.Wait() + + if len(failures) > 0 { + t.Errorf("concurrent uploads of one link id failed: %v", failures) + } + dest := filepath.Join(bundleRoot, "proj-1") + if _, err := os.Stat(filepath.Join(dest, ".git")); err != nil { + t.Fatalf("the link has no work tree after concurrent uploads: %v", err) + } + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil { + t.Fatalf("extracted tree is missing its file: %v", err) + } + if string(got) != "v0" && string(got) != "v1" { + t.Fatalf("a.txt = %q, want one of the uploaded trees", got) + } +} + +// End to end: a link id that could name a staging dir is refused, and refused +// before the client dials. +func TestBridgeRejectsDotPrefixedLinkID(t *testing.T) { + srv := newBridgeServer(t, staticLauncher()) + auth, _ := NewTokenAuthenticator("tok") + bundleRoot := t.TempDir() + addr, ca := startBridge(t, srv, BridgeOptions{Authenticator: auth, BundleDir: bundleRoot}) + + repo := initTestRepo(t, "a.txt", "v0") + for _, id := range []string{".staging-1", ".git", ".hidden"} { + if _, err := UploadRepoBundle(RemoteConfig{Address: addr, Token: "tok", CACertFile: ca}, repo, id); err == nil { + t.Errorf("upload with link id %q should be refused", id) + } + if _, err := os.Stat(filepath.Join(bundleRoot, id)); !os.IsNotExist(err) { + t.Errorf("link id %q must not create %s: %v", id, filepath.Join(bundleRoot, id), err) + } + } +} + +// End to end: a bridge started over a bundle dir a crash left mid-swap puts the +// tree back before it serves anything, and the link is usable again. +func TestBridgeRecoversInterruptedExtractOnStart(t *testing.T) { + srv := newBridgeServer(t, staticLauncher()) + auth, _ := NewTokenAuthenticator("tok") + bundleRoot := t.TempDir() + staging := plantInterruptedExtract(t, bundleRoot, "proj-1", "recovered") + + addr, ca := startBridge(t, srv, BridgeOptions{Authenticator: auth, BundleDir: bundleRoot}) + + got, err := os.ReadFile(filepath.Join(bundleRoot, "proj-1", "a.txt")) + if err != nil { + t.Fatalf("the bridge did not restore the interrupted extract: %v", err) + } + if string(got) != "recovered" { + t.Fatalf("restored a.txt = %q, want %q", got, "recovered") + } + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Errorf("staging should be cleared after recovery, got %v", err) + } + // The recovered link still accepts a fresh upload. + repo := initTestRepo(t, "a.txt", "v2") + if _, err := UploadRepoBundle(RemoteConfig{Address: addr, Token: "tok", CACertFile: ca}, repo, "proj-1"); err != nil { + t.Fatalf("upload to a recovered link: %v", err) + } + got, err = os.ReadFile(filepath.Join(bundleRoot, "proj-1", "a.txt")) + if err != nil || string(got) != "v2" { + t.Fatalf("after re-upload a.txt = %q, err %v, want %q", got, err, "v2") + } +} + +// A crashed extract and a live one look identical on disk: the backup is aside +// and dest is briefly absent. A second daemon starting in that window must not +// take the running extract's tree. +func TestRecoverBundleDirLeavesALiveExtractAlone(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + inPublish := make(chan struct{}) + real := stagingFS.rename + var once sync.Once + stagingFS.rename = func(from, to string) error { + // Stall only the publish (clone into dest), not the restore. + if filepath.Base(from) == "repo" { + once.Do(func() { close(inPublish) }) + time.Sleep(300 * time.Millisecond) + } + return real(from, to) + } + t.Cleanup(func() { stagingFS.rename = real }) + + var wg sync.WaitGroup + var extractErr error + wg.Add(1) + go func() { + defer wg.Done() + extractErr = extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil) + }() + + <-inPublish + time.Sleep(20 * time.Millisecond) // the second daemon starts mid-swap + recoverBundleDir(dir, nil) + wg.Wait() + + if extractErr != nil { + t.Errorf("recovery interfered with a live extract: %v", extractErr) + } + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil { + t.Fatalf("dest holds no tree after the live extract: %v", err) + } + if string(got) != "v1" { + t.Fatalf("a.txt = %q, want the published %q", got, "v1") + } +} + +// stageBackup plants a staging dir holding a backup tree for linkID, named the +// way extractBundle names one so recovery can order it. A stamp above 0 gets the +// allocator's exact grammar, because that is now the only shape recovery orders; +// the name argument then only distinguishes unstamped fixtures, which are the +// shape recovery must refuse to order. Two stamped fixtures in one directory +// need distinct stamps, as two extracts do. +func stageBackup(t *testing.T, dir, name, linkID, content string, stamp int64) string { + t.Helper() + if stamp > 0 { + name = fmt.Sprintf("%020d%s", stamp, stagingSeqSuffix) + } + staging := filepath.Join(dir, stagingPrefix+name) + backup := filepath.Join(staging, "backup") + if err := os.MkdirAll(filepath.Join(backup, ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(backup, "a.txt"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + seq, _ := stagingStamp(filepath.Base(staging)) + if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: linkID, Seq: seq}); err != nil { + t.Fatal(err) + } + return staging +} + +// One link can end up with more than one staged backup: a staging cleanup that +// could not finish leaves one behind, and a later crash adds another. Recovery +// must bring back the newest, and must not delete the newer one as superseded +// just because directory order put the older one first. +func TestRecoverBundleDirRestoresTheNewestOfSeveralBackups(t *testing.T) { + dir := t.TempDir() + // Lexically first, so directory order alone would pick the older tree. + stageBackup(t, dir, "aaa-old", "proj-1", "v0", 100) + newer := stageBackup(t, dir, "zzz-new", "proj-1", "v1", 200) + + recoverBundleDir(dir, nil) + + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil { + t.Fatalf("nothing was restored: %v", err) + } + if string(got) != "v1" { + t.Errorf("restored a.txt = %q, want the newest tree %q", got, "v1") + if _, err := os.Stat(filepath.Join(newer, "backup", "a.txt")); err != nil { + t.Errorf("and the newest tree was deleted as superseded: %v", err) + } + } +} + +// Dropping a backup rests on the live tree having been published over it, which +// is not true of a tree recovery itself just put back. A second backup that +// carries no order recovery can compare against that restore may be the newer +// copy, so it is kept rather than dropped. +// parkedStaging is where recovery moves a staging dir it kept, so a later pass +// does not read the restored tree at dest as a later extract publishing over it. +func parkedStaging(staging string) string { + base := filepath.Base(staging) + return filepath.Join(filepath.Dir(staging), keptPrefix+strings.TrimPrefix(base, stagingPrefix)) +} + +// A name this code never wrote carries no order at all, so it is not a candidate +// for anything: not restored, not parked, not deleted. Moving it would be acting +// on a directory nothing attributes to this code. +func TestRecoverBundleDirKeepsAnUnorderableBackupAgainstATreeItRestored(t *testing.T) { + dir := t.TempDir() + stamped := stageBackup(t, dir, "current", "proj-1", "v1", 200) + unordered := stageBackup(t, dir, "leftover", "proj-1", "v2", 0) + + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }) + + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("the stamped backup should be restored: got %q err %v", got, err) + } + if _, err := os.Stat(filepath.Join(stamped, "backup")); !os.IsNotExist(err) { + t.Errorf("the restored staging dir should be cleared, got %v", err) + } + if _, err := os.Stat(filepath.Join(unordered, "backup", "a.txt")); err != nil { + t.Errorf("a backup under a name this code never wrote must stay where it is: %v", err) + } + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "does not carry a name this code writes") }) { + t.Errorf("an unowned backup should be reported, got %v", logged) + } +} + +// Recovery runs on every start. A backup the pass above deliberately kept must +// still be there after the next one, where the restore is already at dest and +// nothing records that this pass, not a later extract, put it there. +func TestRecoverBundleDirKeepsAnUnorderableBackupAcrossRestarts(t *testing.T) { + dir := t.TempDir() + stageBackup(t, dir, "current", "proj-1", "v1", 200) + unordered := stageBackup(t, dir, "leftover", "proj-1", "v2", 0) + + recoverBundleDir(dir, nil) + if _, err := os.Stat(filepath.Join(unordered, "backup", "a.txt")); err != nil { + t.Fatalf("the first pass should keep the unorderable backup: %v", err) + } + + // The daemon restarts: same dir, a fresh recovery pass with no memory of the + // first. + recoverBundleDir(dir, nil) + + if !keptBackupSurvives(t, dir, "v2") { + t.Error("a backup kept as unorderable must survive the next recovery pass") + } + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil || string(got) != "v1" { + t.Errorf("the restored tree must be left alone across restarts: got %q err %v", got, err) + } + + // And it must not be re-restored or re-parked on every start after that. + recoverBundleDir(dir, nil) + if !keptBackupSurvives(t, dir, "v2") { + t.Error("the kept backup must survive a third pass too") + } +} + +// keptBackupSurvives reports whether a backup holding content is still somewhere +// under dir, wherever recovery decided to keep it. +func keptBackupSurvives(t *testing.T, dir, content string) bool { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + got, err := os.ReadFile(filepath.Join(dir, entry.Name(), "backup", "a.txt")) + if err == nil && string(got) == content { + return true + } + } + return false +} + +// Parking is a rename, and a rename onto a directory that already holds +// something must fail rather than replace it. Recovery then leaves the copy +// where it is, which is still a copy, and never eats the occupant. +func TestRecoverBundleDirDoesNotClobberAnOccupiedParkedName(t *testing.T) { + dir := t.TempDir() + plantUsableDest(t, dir, "proj-1", "live") + keepable := stageBackup(t, dir, "", "proj-1", "v2", 200) + + occupied := parkedStaging(keepable) + if err := os.MkdirAll(occupied, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(occupied, "keep.txt"), []byte("not mine"), 0o600); err != nil { + t.Fatal(err) + } + + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }) + + got, err := os.ReadFile(filepath.Join(occupied, "keep.txt")) + if err != nil || string(got) != "not mine" { + t.Errorf("the occupant of the parked name must be untouched: got %q err %v", got, err) + } + if _, err := os.Stat(filepath.Join(keepable, "backup", "a.txt")); err != nil { + t.Errorf("a backup that could not be parked must stay where it is: %v", err) + } + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, "could not park") }) { + t.Errorf("a failed park should be reported, got %v", logged) + } + + // The next pass finds it under the scanned prefix again. It still carries no + // commit flag, so it is parked again rather than reclaimed: a park that + // failed must never turn into a delete one pass later. + recoverBundleDir(dir, nil) + if _, err := os.Stat(filepath.Join(keepable, "backup", "a.txt")); err != nil { + t.Errorf("a copy that could not be parked must survive the next pass: %v", err) + } +} + +// An older backup is moved out of the scanned prefix once a newer one has been +// restored, so the next pass does not reconsider it. It is not deleted: being +// older than the restored copy is not evidence that anything published over it, +// and only that evidence licenses a delete. +func TestRecoverBundleDirDropsAnOlderBackupAfterRestoringANewerOne(t *testing.T) { + dir := t.TempDir() + stale := stageBackup(t, dir, "stale", "proj-1", "v0", 100) + stageBackup(t, dir, "current", "proj-1", "v1", 200) + + recoverBundleDir(dir, nil) + + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("the newest backup should be restored: got %q err %v", got, err) + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Errorf("the older backup should leave the scanned prefix, got %v", err) + } + if _, err := os.Stat(filepath.Join(parkedStaging(stale), "backup", "a.txt")); err != nil { + t.Errorf("the older backup must be kept, not deleted: %v", err) + } +} + +// Link ids starting with '.' used to be accepted, so a work tree may already be +// published under a name that now looks like a staging dir. The reaper must not +// mistake it for an abandoned extract and delete it on the first start. +func TestRecoverBundleDirKeepsAWorkTreePublishedUnderAReservedName(t *testing.T) { + dir := t.TempDir() + published := filepath.Join(dir, stagingPrefix+"foo") + if err := os.MkdirAll(filepath.Join(published, ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(published, "a.txt"), []byte("someones repo"), 0o644); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-3 * gitTimeout) + if err := os.Chtimes(published, old, old); err != nil { + t.Fatal(err) + } + + recoverBundleDir(dir, nil) + + if _, err := os.Stat(filepath.Join(published, "a.txt")); err != nil { + t.Errorf("a published work tree was reaped on upgrade: %v", err) + } +} + +// A coarse-resolution filesystem can stamp the backup and the live tree with the +// same directory mtime, which is why recovery must not decide staleness from +// timestamps at all: a backup only ever holds the tree that was live BEFORE the +// one at dest, so a link that has a tree supersedes it either way. +func TestRecoverBundleDirDropsBackupWhenMtimesAreEqual(t *testing.T) { + dir := t.TempDir() + live := plantUsableDest(t, dir, "proj-1", "live") + staging := plantInterruptedExtract(t, dir, "proj-1", "stale") + markCommitted(t, staging) + // What a one-second-granularity filesystem produces for a backup and a + // publish that happen in the same second. + tie := time.Unix(1700000000, 0) + for _, path := range []string{filepath.Join(staging, "backup"), live} { + if err := os.Chtimes(path, tie, tie); err != nil { + t.Fatal(err) + } + } + + recoverBundleDir(dir, nil) + + got, err := os.ReadFile(filepath.Join(live, "a.txt")) + if err != nil || string(got) != "live" { + t.Fatalf("the live tree must win: got %q err %v", got, err) + } + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Errorf("a superseded backup should be removed, got %v", err) + } +} + +// End to end over the real transaction. A real upload publishes a tree, a second +// upload is interrupted exactly the way a killed daemon interrupts it (the +// publish rename and the restore behind it both fail, so the prior tree is left +// in staging), and a fresh bridge on the same directory puts it back. Nothing +// here plants a fixture: the staging name recovery has to order by is the one +// extractBundle writes, which is the half a hand-built fixture cannot check. +func TestBridgeRecoversARealInterruptedExtractOnStart(t *testing.T) { + srv := newBridgeServer(t, staticLauncher()) + auth, _ := NewTokenAuthenticator("tok") + bundleRoot := t.TempDir() + addr, ca := startBridge(t, srv, BridgeOptions{Authenticator: auth, BundleDir: bundleRoot}) + cfg := RemoteConfig{Address: addr, Token: "tok", CACertFile: ca} + dest := filepath.Join(bundleRoot, "proj-1") + + if _, err := UploadRepoBundle(cfg, initTestRepo(t, "a.txt", "v1"), "proj-1"); err != nil { + t.Fatalf("first upload: %v", err) + } + if got, err := os.ReadFile(filepath.Join(dest, "a.txt")); err != nil || string(got) != "v1" { + t.Fatalf("first upload did not publish: got %q err %v", got, err) + } + + // Both renames fail, which is the one path that leaves the tree in staging + // with dest absent -- what a process killed between the two renames leaves. + real := stagingFS.rename + stagingFS.rename = func(from, to string) error { + if to == dest { + return errors.New("injected rename failure") + } + return real(from, to) + } + _, err := UploadRepoBundle(cfg, initTestRepo(t, "a.txt", "v2"), "proj-1") + stagingFS.rename = real + if err == nil { + t.Fatal("an upload whose publish and restore both fail must report an error") + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatalf("the interrupted swap should leave dest absent, got %v", err) + } + + // The name extractBundle wrote must be one recovery can order by. If the + // writer and the reader ever disagree on the format, this is where it shows. + entries, err := os.ReadDir(bundleRoot) + if err != nil { + t.Fatal(err) + } + stagedNames := []string{} + for _, e := range entries { + if strings.HasPrefix(e.Name(), stagingPrefix) { + stagedNames = append(stagedNames, e.Name()) + } + } + if len(stagedNames) != 1 { + t.Fatalf("want exactly one staging dir left behind, got %v", stagedNames) + } + if _, ok := stagingStamp(stagedNames[0]); !ok { + t.Fatalf("recovery cannot order the name extractBundle wrote: %q", stagedNames[0]) + } + + // A fresh daemon over the same directory repairs it before serving. + addr2, ca2 := startBridge(t, newBridgeServer(t, staticLauncher()), BridgeOptions{Authenticator: auth, BundleDir: bundleRoot}) + + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("the interrupted extract was not restored: got %q err %v", got, err) + } + for _, name := range stagedNames { + if _, err := os.Stat(filepath.Join(bundleRoot, name)); !os.IsNotExist(err) { + t.Errorf("staging %s should be cleared after recovery, got %v", name, err) + } + } + // And the repaired link still serves. + if _, err := UploadRepoBundle(RemoteConfig{Address: addr2, Token: "tok", CACertFile: ca2}, initTestRepo(t, "a.txt", "v3"), "proj-1"); err != nil { + t.Fatalf("upload to a recovered link: %v", err) + } + if got, err := os.ReadFile(filepath.Join(dest, "a.txt")); err != nil || string(got) != "v3" { + t.Fatalf("after re-upload a.txt = %q, err %v, want %q", got, err, "v3") + } +} + +// A backward clock leaves an OLDER backup carrying a LARGER stamp. Recovery must +// still restore the tree that was actually live last, and must not delete it as +// superseded on the way. +// +// Fixture order is load-bearing: NewBridge repairs the directory before it +// serves, so a backup planted before the bridge starts is consumed by that first +// pass and this test would pass without proving anything. Publish first, plant +// second. +func TestExtractBundleOutOrdersALeftoverStampedInTheFuture(t *testing.T) { + // A nanosecond stamp decades ahead, which is what a clock correction + // backward leaves behind on the earlier of two writes. + const farFuture = int64(4_000_000_000_000_000_000) + + srv := newBridgeServer(t, staticLauncher()) + auth, _ := NewTokenAuthenticator("tok") + bundleRoot := t.TempDir() + addr, ca := startBridge(t, srv, BridgeOptions{Authenticator: auth, BundleDir: bundleRoot}) + cfg := RemoteConfig{Address: addr, Token: "tok", CACertFile: ca} + dest := filepath.Join(bundleRoot, "proj-1") + + if _, err := UploadRepoBundle(cfg, initTestRepo(t, "a.txt", "v1"), "proj-1"); err != nil { + t.Fatalf("first upload: %v", err) + } + + // Only now, with the bridge already up and its recovery pass behind us. + stale := stageBackup(t, bundleRoot, "old", "proj-1", "v-old", farFuture) + + real := stagingFS.rename + stagingFS.rename = func(from, to string) error { + if to == dest { + return errors.New("injected rename failure") + } + return real(from, to) + } + _, err := UploadRepoBundle(cfg, initTestRepo(t, "a.txt", "v2"), "proj-1") + stagingFS.rename = real + if err == nil { + t.Fatal("an upload whose publish and restore both fail must report an error") + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatalf("the interrupted swap should leave dest absent, got %v", err) + } + + recoverBundleDir(bundleRoot, nil) + + // The interrupted extract set the LIVE tree aside, so v1 is what recovery + // owes back. The interrupted clone of v2 never reached dest. + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v1" { + t.Errorf("recovery restored %q (err %v), want the tree that was live last, %q", got, err, "v1") + // Distinguish "restored the wrong one" from "restored the wrong one AND + // destroyed the right one", which is the data loss this test exists for. + if !liveTreeSurvivesSomewhere(t, bundleRoot, "v1") { + t.Error("the tree that was live last was deleted as superseded; no copy of it is left on disk") + } + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Errorf("the genuinely older backup should leave the scanned prefix once the newer one is restored, got %v", err) + } + if _, err := os.Stat(filepath.Join(parkedStaging(stale), "backup", "a.txt")); err != nil { + t.Errorf("it must be kept, not deleted: %v", err) + } +} + +// The allocator and the parser must agree, and a number already spoken for by a +// kept backup must never come round again. +func TestStagingNamesAllocateInOrderAndParse(t *testing.T) { + t.Run("counts up and parses", func(t *testing.T) { + dir := t.TempDir() + for want := int64(1); want <= 2; want++ { + seq, err := nextStagingSeq(dir) + if err != nil { + t.Fatal(err) + } + path, err := createSequencedStagingDir(dir, seq) + if err != nil { + t.Fatal(err) + } + stamp, ok := stagingStamp(filepath.Base(path)) + if !ok { + t.Fatalf("the allocator wrote a name recovery cannot order: %q", filepath.Base(path)) + } + if stamp != want { + t.Errorf("stamp = %d, want %d", stamp, want) + } + } + }) + + // The upgrade case. v0.8.0 staged under os.MkdirTemp(parent, ".staging-*"), + // whose suffix is a decimal uint32 with no second '-', and this branch's + // intermediate commits wrote a random one. Neither shape is a sequence this + // package ever handed out, so the allocator reads none of them and starts at + // one; the residue is left where it is for recovery to report. + t.Run("ignores a legacy MkdirTemp-shaped name", func(t *testing.T) { + dir := t.TempDir() + const legacy = int64(1_700_000_000_000_000_000) + name := fmt.Sprintf("%s%020d-x7Kq3", stagingPrefix, legacy) + if err := os.Mkdir(filepath.Join(dir, name), 0o700); err != nil { + t.Fatal(err) + } + seq, err := nextStagingSeq(dir) + if err != nil { + t.Fatal(err) + } + if seq != 1 { + t.Errorf("next sequence = %d, want 1: a name this package never wrote is not a sequence", seq) + } + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + t.Errorf("the legacy entry must be left alone: %v", err) + } + }) + + // A parked backup owns its number for as long as it exists. + t.Run("skips a number already parked", func(t *testing.T) { + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, fmt.Sprintf("%s%020d%s", keptPrefix, 7, stagingSeqSuffix)), 0o700); err != nil { + t.Fatal(err) + } + seq, err := nextStagingSeq(dir) + if err != nil { + t.Fatal(err) + } + if seq <= 7 { + t.Errorf("next sequence = %d, want strictly greater than the parked 7", seq) + } + }) +} + +// The grammar is the first ownership filter: a name that is not exactly what +// createSequencedStagingDir writes was written by something else, and reading a +// sequence out of it is how a legacy work tree gets a say in the ordering. +func TestStagingStampRequiresTheExactGrammar(t *testing.T) { + // No '*', '?' or ':' in any name here: a table entry Windows cannot name + // would fail its own setup rather than test the parser. + cases := []struct { + name string + seq int64 + ok bool + }{ + {stagingPrefix + "00000000000000000042" + stagingSeqSuffix, 42, true}, + {stagingPrefix + "42" + stagingSeqSuffix, 0, false}, + // The shape this branch's intermediate commits wrote before the suffix + // was fixed, and the shape os.MkdirTemp writes. + {stagingPrefix + "00000000000000000042-x7Kq3", 0, false}, + // The v0.8.0 shape: MkdirTemp's decimal suffix with no second '-'. + {stagingPrefix + "1234567890", 0, false}, + {stagingPrefix + "00000000000000000042" + stagingSeqSuffix + "-extra", 0, false}, + {stagingPrefix + "0000000000000000004a" + stagingSeqSuffix, 0, false}, + // Twenty characters, and ParseInt would take the sign. A link id may + // contain '-', so this is a name a legacy work tree can carry. + {stagingPrefix + "-0000000000000000042" + stagingSeqSuffix, 0, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + seq, ok := stagingStamp(tc.name) + if ok != tc.ok { + t.Fatalf("stagingStamp(%q) ok = %v, want %v", tc.name, ok, tc.ok) + } + if ok && seq != tc.seq { + t.Errorf("stagingStamp(%q) = %d, want %d", tc.name, seq, tc.seq) + } + }) + } +} + +// Link ids starting with '.' used to be accepted, so a published work tree can +// sit under a name that now reads as a staging sequence. One at the maximum +// would make the allocator refuse to allocate anything, permanently, and no +// extract for any link in the directory could run again. +func TestNextStagingSeqIgnoresALegacyWorkTreeAtTheMaximum(t *testing.T) { + dir := t.TempDir() + // Twenty digits, so it passes the grammar and only the .git veto keeps it + // out of the ordering. + planted := []string{ + fmt.Sprintf("%s%020d%s", stagingPrefix, int64(math.MaxInt64), stagingSeqSuffix), + // Nineteen digits, the name from the review: the grammar alone rejects it. + stagingPrefix + "9223372036854775807" + stagingSeqSuffix, + } + for _, name := range planted { + if err := os.MkdirAll(filepath.Join(dir, name, ".git"), 0o700); err != nil { + t.Fatal(err) + } + } + // A legacy staging dir at the maximum with no work tree in it: the grammar + // is the only thing that can keep this one out, so loosening the grammar is + // visible here even with the veto in place. + if err := os.Mkdir(filepath.Join(dir, fmt.Sprintf("%s%020d-x7Kq3", stagingPrefix, int64(math.MaxInt64))), 0o700); err != nil { + t.Fatal(err) + } + + seq, err := nextStagingSeq(dir) + if err != nil { + t.Fatalf("a legacy entry must not stop the allocator: %v", err) + } + if seq != 1 { + t.Errorf("next sequence = %d, want 1: none of these names is one this package wrote", seq) + } + for _, name := range planted { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + t.Errorf("the allocator must not touch %s: %v", name, err) + } + } +} + +// The veto is keyed on .git at the entry's own root. A live extract's staging +// dir holds its clone one level down, in repo/, so its number still counts; +// handing it out again would put two transactions on one name. +func TestNextStagingSeqCountsAStagingDirHoldingAClone(t *testing.T) { + dir := t.TempDir() + staging := filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, 4, stagingSeqSuffix)) + if err := os.MkdirAll(filepath.Join(staging, "repo", ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(staging, "backup", ".git"), 0o700); err != nil { + t.Fatal(err) + } + + seq, err := nextStagingSeq(dir) + if err != nil { + t.Fatal(err) + } + if seq != 5 { + t.Errorf("next sequence = %d, want 5: a live extract's number is still spoken for", seq) + } +} + +// Kept backups are named off the staging name, so they are counted, and they are +// counted under the same grammar: a loose read of one is the same defect as a +// loose read of a staging name, one prefix over. +func TestNextStagingSeqCountsKeptNamesUnderTheSameGrammar(t *testing.T) { + dir := t.TempDir() + kept := filepath.Join(dir, fmt.Sprintf("%s%020d%s", keptPrefix, 7, stagingSeqSuffix)) + if err := os.Mkdir(kept, 0o700); err != nil { + t.Fatal(err) + } + if err := writeMarker(kept, txnMarker{Kind: txnKindBundleExtract, Dest: "proj-1", Seq: 7}); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(dir, fmt.Sprintf("%s%020d-x", keptPrefix, 99)), 0o700); err != nil { + t.Fatal(err) + } + + seq, err := nextStagingSeq(dir) + if err != nil { + t.Fatal(err) + } + if seq != 8 { + t.Errorf("next sequence = %d, want 8: the parked 7 is spoken for and the loose 99 is not a sequence", seq) + } +} + +// A publish that succeeded but could not clear its staging dir reports success to +// the client while a whole copy of the prior tree stays on disk. The next +// recovery pass reclaims it, and must say so: with no bridge logger configured +// nothing else ever mentions the space that was being held. +func TestRecoverBundleDirReportsReclaimingASupersededTree(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + // The publish landed, so dest holds the new tree. + if err := os.MkdirAll(filepath.Join(dest, ".git"), 0o700); err != nil { + t.Fatal(err) + } + // The cleanup did not, so the old tree is still staged beside it, carrying + // the flag that proves the publish landed over it. + staging := stageBackup(t, dir, "leftover", "proj-1", "v-old", 100) + markCommitted(t, staging) + + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }) + + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Errorf("a staged tree the live one superseded should be reclaimed, got %v", err) + } + if !slices.ContainsFunc(logged, func(m string) bool { return strings.Contains(m, staging) }) { + t.Errorf("reclaiming a superseded staged tree should name it, got %v", logged) + } +} + +// A bundle dir's ordinary contents are the per-link work trees, and a link id is +// whatever the uploading client sent. The allocator must read only the names it +// owns: a link id is not a sequence, however much it looks like one. +func TestStagingSeqIgnoresNamesItDoesNotOwn(t *testing.T) { + // sanitizeLinkID permits digits, '-' and letters, so every id here is one a + // client can actually upload under. + for _, id := range []string{"12345-abc", "2024-project", "09223372036854775807-seq"} { + t.Run(id, func(t *testing.T) { + if _, err := sanitizeLinkID(id); err != nil { + t.Fatalf("fixture is not a link id a client could send: %v", err) + } + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, id), 0o700); err != nil { + t.Fatal(err) + } + + seq, err := nextStagingSeq(dir) + if err != nil { + t.Fatalf("a link work tree must not stop the allocator: %v", err) + } + if seq != 1 { + t.Errorf("next sequence = %d, want 1: a link name is not a staging name", seq) + } + }) + } +} + +// The retry branch: another writer already holds the number this one started +// from, so it walks up rather than failing or reusing. +func TestCreateSequencedStagingDirSkipsAnOccupiedNumber(t *testing.T) { + dir := t.TempDir() + taken := fmt.Sprintf("%s%020d%s", stagingPrefix, 1, stagingSeqSuffix) + if err := os.Mkdir(filepath.Join(dir, taken), 0o700); err != nil { + t.Fatal(err) + } + + got, err := createSequencedStagingDir(dir, 1) + if err != nil { + t.Fatal(err) + } + want := filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, 2, stagingSeqSuffix)) + if got != want { + t.Errorf("claimed %q, want %q", got, want) + } + if _, err := os.Stat(filepath.Join(dir, taken)); err != nil { + t.Errorf("the occupied name must be left alone: %v", err) + } +} + +// A name at the maximum would make the seeding addition wrap negative, and a +// negative renders a '-' the parser reads as empty digits, so the entry would +// leave the ordering silently. Refusing is the loud version, and the refusal +// belongs where the addition is. +func TestNextStagingSeqRefusesOverflow(t *testing.T) { + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, int64(math.MaxInt64), stagingSeqSuffix)), 0o700); err != nil { + t.Fatal(err) + } + if seq, err := nextStagingSeq(dir); err == nil { + t.Errorf("nextStagingSeq returned %d, want an error rather than a wrapped value", seq) + } +} + +func TestCreateSequencedStagingDirRefusesOutOfRange(t *testing.T) { + dir := t.TempDir() + for _, n := range []int64{0, -1, math.MinInt64} { + if _, err := createSequencedStagingDir(dir, n); err == nil { + t.Errorf("createSequencedStagingDir(%d) succeeded, want an error", n) + } + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Errorf("a refused allocation must create nothing, got %v", entries) + } +} + +// Exclusive creation is the whole concurrency story, so it gets executed rather +// than argued: no two allocators may come away with the same name or number. +func TestStagingSeqAllocatesDistinctValuesConcurrently(t *testing.T) { + dir := t.TempDir() + // 128, not a smaller number: against a check-then-create allocator this + // detects the duplicate in every run, where 16 caught it about half the time. + const writers = 128 + + var wg sync.WaitGroup + paths := make([]string, writers) + errs := make([]error, writers) + for i := 0; i < writers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + seq, err := nextStagingSeq(dir) + if err != nil { + errs[i] = err + return + } + paths[i], errs[i] = createSequencedStagingDir(dir, seq) + }(i) + } + wg.Wait() + + seen := map[string]bool{} + stamps := map[int64]bool{} + for i, path := range paths { + if errs[i] != nil { + t.Fatalf("writer %d: %v", i, errs[i]) + } + if seen[path] { + t.Errorf("two writers claimed the same name %q", path) + } + seen[path] = true + stamp, ok := stagingStamp(filepath.Base(path)) + if !ok { + t.Errorf("writer %d wrote an unorderable name %q", i, filepath.Base(path)) + continue + } + if stamps[stamp] { + t.Errorf("two writers claimed sequence %d", stamp) + } + stamps[stamp] = true + } +} + +// The staging dir holds the only copy of a link's tree mid-swap, so it keeps the +// owner-only mode it has always had. os.Mkdir takes +// a mode where os.MkdirTemp did not, so the allocator has to name it. +func TestStagingDirKeepsOwnerOnlyPermissions(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Go permission bits do not map to Windows ACLs") + } + dir := t.TempDir() + path, err := createSequencedStagingDir(dir, 1) + if err != nil { + t.Fatal(err) + } + requireUmaskAllowsWiderThan0700(t, dir) + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o700 { + t.Errorf("staging dir mode = %o, want 0700", got) + } +} + +// requireUmaskAllowsWiderThan0700 skips when the process umask would strip the +// group and other bits anyway. Without it this assertion is vacuous under a +// umask of 077: a wrongly widened 0o755 comes back as 0700 and passes, so the +// one test guarding the mode would silently stop guarding it. +func requireUmaskAllowsWiderThan0700(t *testing.T, parent string) { + t.Helper() + control := filepath.Join(parent, ".umask-control") + if err := os.Mkdir(control, 0o755); err != nil { + t.Fatal(err) + } + defer func() { _ = os.RemoveAll(control) }() + info, err := os.Stat(control) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o755 { + t.Skipf("umask masks a 0755 request down to %o, so this assertion cannot tell 0700 from a widened mode", info.Mode().Perm()) + } +} + +// liveTreeSurvivesSomewhere reports whether any staging or kept dir under root +// still holds a backup with the given content. +func liveTreeSurvivesSomewhere(t *testing.T, root, content string) bool { + t.Helper() + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + got, err := os.ReadFile(filepath.Join(root, entry.Name(), "backup", "a.txt")) + if err == nil && string(got) == content { + return true + } + } + return false +} + +// Recovery decides each link on its own: restoring one link's tree must not make +// another link's superseded backup look unorderable, and vice versa. +func TestRecoverBundleDirHandlesLinksIndependently(t *testing.T) { + dir := t.TempDir() + // proj-1 has no tree, so its backup is restored. + restorable := stageBackup(t, dir, "one", "proj-1", "v1", 100) + // proj-2 has a usable live tree and a copy whose flag proves that tree + // published over it, so that copy is dropped. + live := plantUsableDest(t, dir, "proj-2", "live") + superseded := stageBackup(t, dir, "two", "proj-2", "old", 200) + markCommitted(t, superseded) + + recoverBundleDir(dir, nil) + + if got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")); err != nil || string(got) != "v1" { + t.Errorf("proj-1 should be restored: got %q err %v", got, err) + } + if _, err := os.Stat(restorable); !os.IsNotExist(err) { + t.Errorf("proj-1 staging should be cleared, got %v", err) + } + if got, err := os.ReadFile(filepath.Join(live, "a.txt")); err != nil || string(got) != "live" { + t.Errorf("proj-2's live tree must win: got %q err %v", got, err) + } + if _, err := os.Stat(superseded); !os.IsNotExist(err) { + t.Errorf("proj-2's superseded staging should be removed, got %v", err) + } +} + +// Recovery runs on every start, so a second pass over an already-repaired +// directory must be a no-op rather than treating the tree it restored last time +// as something to move again. +func TestRecoverBundleDirIsIdempotent(t *testing.T) { + dir := t.TempDir() + stageBackup(t, dir, "one", "proj-1", "v1", 100) + + recoverBundleDir(dir, nil) + first, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + recoverBundleDir(dir, nil) + second, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + if got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")); err != nil || string(got) != "v1" { + t.Fatalf("the restored tree must survive a second pass: got %q err %v", got, err) + } + names := func(es []os.DirEntry) []string { + out := []string{} + for _, e := range es { + out = append(out, e.Name()) + } + return out + } + if !slices.Equal(names(first), names(second)) { + t.Errorf("a second recovery pass changed the directory: %v then %v", names(first), names(second)) + } +} + +// Two backups can both carry names this code never wrote: v0.8.0 residue, or a +// crash before the marker. Neither is a candidate, so neither is restored and +// neither is touched. Restoring one would be picking between two copies with +// nothing on disk to order them by, and the loser is then a copy sitting beside +// a destination the next pass reads as having superseded it. +func TestRecoverBundleDirKeepsABackupItCannotTellApartFromTheRestore(t *testing.T) { + dir := t.TempDir() + first := stageBackup(t, dir, "one", "proj-1", "v1", 0) + second := stageBackup(t, dir, "two", "proj-1", "v2", 0) + + for pass := 1; pass <= 2; pass++ { + recoverBundleDir(dir, nil) + + if _, err := os.Stat(filepath.Join(dir, "proj-1")); !os.IsNotExist(err) { + t.Fatalf("pass %d: nothing may be restored from a name this code never wrote: %v", pass, err) + } + for _, staging := range []string{first, second} { + if _, err := os.Stat(filepath.Join(staging, "backup", "a.txt")); err != nil { + t.Errorf("pass %d: %s must be retained in place: %v", pass, staging, err) + } + } + } +} + +// ---- filesystem seam ------------------------------------------------------- + +// injectFault swaps one field of stagingFS for a wrapper that fails the call +// whose arguments satisfy match and passes every other call through to the real +// primitive. The call ordinal is appended to the arguments handed to match, so +// nthCall can select by call order without a second counter, and it is counted +// with an atomic because the racing scenarios drive one seam from two +// goroutines. The whole struct is restored in t.Cleanup, so a test that injects +// twice unwinds in reverse. +func injectFault(t *testing.T, step string, match func(args ...string) bool, err error) { + t.Helper() + var calls atomic.Int64 + real := stagingFS + t.Cleanup(func() { stagingFS = real }) + fire := func(args ...string) bool { + n := calls.Add(1) + if match == nil { + return true + } + return match(append(args, strconv.FormatInt(n, 10))...) + } + switch step { + case "rename": + stagingFS.rename = func(from, to string) error { + if fire(from, to) { + return err + } + return real.rename(from, to) + } + case "removeAll": + stagingFS.removeAll = func(path string) error { + if fire(path) { + return err + } + return real.removeAll(path) + } + case "stat": + stagingFS.stat = func(name string) (os.FileInfo, error) { + if fire(name) { + return nil, err + } + return real.stat(name) + } + case "lstat": + stagingFS.lstat = func(name string) (os.FileInfo, error) { + if fire(name) { + return nil, err + } + return real.lstat(name) + } + case "readDir": + stagingFS.readDir = func(name string) ([]os.DirEntry, error) { + if fire(name) { + return nil, err + } + return real.readDir(name) + } + case "readFile": + stagingFS.readFile = func(name string) ([]byte, error) { + if fire(name) { + return nil, err + } + return real.readFile(name) + } + case "mkdir": + stagingFS.mkdir = func(name string, perm os.FileMode) error { + if fire(name) { + return err + } + return real.mkdir(name, perm) + } + case "writeFile": + stagingFS.writeFile = func(name string, data []byte, perm os.FileMode) error { + if fire(name) { + return err + } + return real.writeFile(name, data, perm) + } + case "create": + stagingFS.create = func(name string, flag int, perm os.FileMode) (*os.File, error) { + if fire(name) { + return nil, err + } + return real.create(name, flag, perm) + } + case "createTemp": + stagingFS.createTemp = func(dir, pattern string) (*os.File, error) { + if fire(dir, pattern) { + return nil, err + } + return real.createTemp(dir, pattern) + } + default: + t.Fatalf("injectFault: unknown step %q", step) + } +} + +// nthCall selects the nth call through an injected seam. It reads the ordinal +// injectFault appends to the arguments rather than counting for itself, so the +// two share one counter and a concurrent test has one place to be correct. +func nthCall(n int) func(args ...string) bool { + return func(args ...string) bool { + return len(args) > 0 && args[len(args)-1] == strconv.Itoa(n) + } +} + +// blockStep holds every call to step until the returned release runs, which is +// how a test parks one transaction mid-step and lets another reach the same +// destination. Releasing twice is safe, so a test can release on the happy path +// and still defer it. +func blockStep(t *testing.T, step string) (release func()) { + t.Helper() + gate := make(chan struct{}) + var once sync.Once + release = func() { once.Do(func() { close(gate) }) } + t.Cleanup(release) + injectFaultGate(t, step, gate) + return release +} + +// injectFaultGate is blockStep's half of the swap, split out so the wrapper it +// installs sits beside the fault wrappers above. +func injectFaultGate(t *testing.T, step string, gate <-chan struct{}) { + t.Helper() + real := stagingFS + t.Cleanup(func() { stagingFS = real }) + switch step { + case "rename": + stagingFS.rename = func(from, to string) error { + <-gate + return real.rename(from, to) + } + case "removeAll": + stagingFS.removeAll = func(path string) error { + <-gate + return real.removeAll(path) + } + case "stat": + stagingFS.stat = func(name string) (os.FileInfo, error) { + <-gate + return real.stat(name) + } + default: + t.Fatalf("blockStep: unknown step %q", step) + } +} + +// The seam has to be the real filesystem until a test swaps a field. A nil field +// is a panic in whatever production path reaches it first, and a field that no +// longer behaves like its os function makes every test above it prove nothing. +func TestStagingFSDefaultsAreTheRealPrimitives(t *testing.T) { + for name, missing := range map[string]bool{ + "rename": stagingFS.rename == nil, + "removeAll": stagingFS.removeAll == nil, + "stat": stagingFS.stat == nil, + "lstat": stagingFS.lstat == nil, + "readDir": stagingFS.readDir == nil, + "readFile": stagingFS.readFile == nil, + "mkdir": stagingFS.mkdir == nil, + "writeFile": stagingFS.writeFile == nil, + "create": stagingFS.create == nil, + "createTemp": stagingFS.createTemp == nil, + } { + if missing { + t.Errorf("stagingFS.%s is nil", name) + } + } + + root := t.TempDir() + dir := filepath.Join(root, "one") + if err := stagingFS.mkdir(dir, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + file := filepath.Join(dir, "a.txt") + if err := stagingFS.writeFile(file, []byte("v0"), 0o600); err != nil { + t.Fatalf("writeFile: %v", err) + } + if _, err := stagingFS.stat(file); err != nil { + t.Fatalf("stat: %v", err) + } + if _, err := stagingFS.lstat(file); err != nil { + t.Fatalf("lstat: %v", err) + } + if got, err := stagingFS.readFile(file); err != nil || string(got) != "v0" { + t.Fatalf("readFile = %q, %v, want %q", got, err, "v0") + } + entries, err := stagingFS.readDir(dir) + if err != nil || len(entries) != 1 || entries[0].Name() != "a.txt" { + t.Fatalf("readDir = %v, %v, want one entry a.txt", entries, err) + } + tmp, err := stagingFS.createTemp(dir, "seam-*") + if err != nil { + t.Fatalf("createTemp: %v", err) + } + tmpName := tmp.Name() + if err := tmp.Close(); err != nil { + t.Fatalf("close temp: %v", err) + } + opened, err := stagingFS.create(tmpName, os.O_WRONLY, 0o600) + if err != nil { + t.Fatalf("create: %v", err) + } + if err := opened.Close(); err != nil { + t.Fatalf("close opened: %v", err) + } + moved := filepath.Join(root, "two") + if err := stagingFS.rename(dir, moved); err != nil { + t.Fatalf("rename: %v", err) + } + if _, err := stagingFS.stat(dir); !os.IsNotExist(err) { + t.Fatalf("stat of the old name = %v, want not-exist", err) + } + if err := stagingFS.removeAll(moved); err != nil { + t.Fatalf("removeAll: %v", err) + } + if _, err := stagingFS.stat(moved); !os.IsNotExist(err) { + t.Fatalf("stat after removeAll = %v, want not-exist", err) + } +} + +// The matrix needs both selections: "fail the rename whose source is seq 2's +// backup" is an argument match, and "fail the second remove" is an ordinal one. +// An ordinal alone cannot express the first, and an argument match cannot +// express a step that runs twice on the same path. +func TestInjectFaultMatchesByArgumentAndByOrdinal(t *testing.T) { + injected := errors.New("injected seam failure") + root := t.TempDir() + + t.Run("by argument", func(t *testing.T) { + injectFault(t, "rename", func(args ...string) bool { + return strings.HasSuffix(args[0], filepath.Join("seq", "backup")) + }, injected) + for _, name := range []string{"one", "seq", "three"} { + from := filepath.Join(root, name, "backup") + if err := os.MkdirAll(from, 0o700); err != nil { + t.Fatal(err) + } + err := stagingFS.rename(from, filepath.Join(root, name, "moved")) + if name == "seq" { + if !errors.Is(err, injected) { + t.Fatalf("rename of %s = %v, want the injected failure", from, err) + } + continue + } + if err != nil { + t.Fatalf("rename of %s = %v, want it to pass through", from, err) + } + } + }) + + t.Run("by ordinal", func(t *testing.T) { + injectFault(t, "removeAll", nthCall(2), injected) + for i := 1; i <= 3; i++ { + dir := filepath.Join(root, fmt.Sprintf("remove-%d", i)) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + err := stagingFS.removeAll(dir) + if i == 2 { + if !errors.Is(err, injected) { + t.Fatalf("removeAll call %d = %v, want the injected failure", i, err) + } + if _, statErr := os.Stat(dir); statErr != nil { + t.Fatalf("the failed call must not have removed %s: %v", dir, statErr) + } + continue + } + if err != nil { + t.Fatalf("removeAll call %d = %v, want it to pass through", i, err) + } + if _, statErr := os.Stat(dir); !os.IsNotExist(statErr) { + t.Fatalf("removeAll call %d left %s behind: %v", i, dir, statErr) + } + } + }) + + if got, want := reflect.ValueOf(stagingFS.removeAll).Pointer(), reflect.ValueOf(os.RemoveAll).Pointer(); got != want { + t.Fatal("stagingFS.removeAll was not restored after the subtest's cleanup") + } + if got, want := reflect.ValueOf(stagingFS.rename).Pointer(), reflect.ValueOf(os.Rename).Pointer(); got != want { + t.Fatal("stagingFS.rename was not restored after the subtest's cleanup") + } + + // One seam, two goroutines: the ordinal must come from a counter that is + // safe to increment concurrently, or the racing rows this seam exists for + // report a race instead of a result. + t.Run("concurrently", func(t *testing.T) { + injectFault(t, "stat", nthCall(1), injected) + start := make(chan struct{}) + errs := make([]error, 2) + var wg sync.WaitGroup + for i := range errs { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + _, errs[i] = stagingFS.stat(root) + }(i) + } + close(start) + wg.Wait() + failed := 0 + for _, err := range errs { + if errors.Is(err, injected) { + failed++ + } else if err != nil { + t.Fatalf("the call that was not selected = %v, want it to pass through", err) + } + } + if failed != 1 { + t.Fatalf("%d of 2 concurrent calls failed, want exactly 1", failed) + } + }) +} + +// A blocked step has to stay blocked: the racing rows park one transaction +// inside a step and only then let the second one reach the same destination. +func TestBlockStepHoldsTheCallUntilRelease(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "blocked") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + release := blockStep(t, "removeAll") + done := make(chan error, 1) + go func() { done <- stagingFS.removeAll(dir) }() + select { + case err := <-done: + t.Fatalf("removeAll returned %v before release", err) + case <-time.After(20 * time.Millisecond): + } + release() + release() + if err := <-done; err != nil { + t.Fatalf("removeAll after release: %v", err) + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Fatalf("the released call did not run: %v", err) + } +} + +// ---- recovery: reconcile per destination ----------------------------------- + +// markCommitted writes the flag extractBundle writes after a publish lands. It +// is the only evidence a staged copy was superseded, so a fixture that omits it +// is a copy recovery is not allowed to delete. +func markCommitted(t *testing.T, staging string) { + t.Helper() + f, err := os.OpenFile(filepath.Join(staging, committedFile), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } +} + +// stageScratch plants an owned staging dir that holds no copy of any tree: the +// state a crash between the marker and the set-aside leaves. It is the one shape +// besides a committed copy that recovery may delete. +func stageScratch(t *testing.T, dir, linkID string, stamp int64) string { + t.Helper() + staging := filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, stamp, stagingSeqSuffix)) + if err := os.MkdirAll(filepath.Join(staging, "repo"), 0o700); err != nil { + t.Fatal(err) + } + if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: linkID, Seq: stamp}); err != nil { + t.Fatal(err) + } + return staging +} + +// plantUsableDest writes the shape the bundle site's usability predicate looks +// for: a work tree, which is dest/.git present. Nothing here shells out to git, +// because the predicate does not either. +func plantUsableDest(t *testing.T, dir, id, content string) string { + t.Helper() + dest := filepath.Join(dir, id) + if err := os.MkdirAll(filepath.Join(dest, ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dest, "a.txt"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return dest +} + +// plantUnusableDest writes a destination that exists and cannot serve: a husk +// with no .git, which is what a partial restore or a half-finished swap leaves. +func plantUnusableDest(t *testing.T, dir, id, content string) string { + t.Helper() + dest := filepath.Join(dir, id) + if err := os.MkdirAll(dest, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dest, "husk.txt"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return dest +} + +func recoverAndLog(t *testing.T, dir string) []string { + t.Helper() + var logged []string + recoverBundleDir(dir, func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }) + return logged +} + +func logged(logs []string, want string) bool { + return slices.ContainsFunc(logs, func(m string) bool { return strings.Contains(m, want) }) +} + +// The commit flag is the only evidence a staged copy was published over. The +// predicate this replaces was "the destination exists", which is not evidence of +// anything: a destination can exist because an outside hand put it there, or +// because a crash left a husk, and deleting the last copy of a tree on that +// reading is the central defect of the old pass. +func TestRecoverBundleDirDeletesOnlyWithACommitFlag(t *testing.T) { + t.Run("beside a usable dest only the flagged copy goes", func(t *testing.T) { + dir := t.TempDir() + plantUsableDest(t, dir, "proj-1", "live") + committed := stageBackup(t, dir, "", "proj-1", "superseded", 100) + markCommitted(t, committed) + uncommitted := stageBackup(t, dir, "", "proj-1", "unproven", 200) + + logs := recoverAndLog(t, dir) + + if _, err := os.Stat(committed); !os.IsNotExist(err) { + t.Errorf("a copy the destination provably published over should be reclaimed, got %v", err) + } + got, err := os.ReadFile(filepath.Join(parkedStaging(uncommitted), "backup", "a.txt")) + if err != nil || string(got) != "unproven" { + t.Errorf("a copy with no commit flag must be kept: got %q err %v", got, err) + } + if _, err := os.Stat(uncommitted); !os.IsNotExist(err) { + t.Errorf("the kept copy should be parked out of the scanned prefix, got %v", err) + } + if !logged(logs, uncommitted) { + t.Errorf("the retained copy should be named in the report, got %v", logs) + } + }) + + t.Run("beside an unusable dest the flagged copy is restored", func(t *testing.T) { + dir := t.TempDir() + dest := plantUnusableDest(t, dir, "proj-1", "husk") + committed := stageBackup(t, dir, "", "proj-1", "last-copy", 100) + markCommitted(t, committed) + + recoverAndLog(t, dir) + + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "last-copy" { + t.Errorf("the only usable copy should be restored over a husk: got %q err %v", got, err) + } + if _, err := os.Stat(committed); !os.IsNotExist(err) { + t.Errorf("the restored copy's directory should be gone, got %v", err) + } + if !keptHuskSurvives(t, dir, "husk") { + t.Error("the husk that was at the destination must be kept, not deleted") + } + }) +} + +// keptHuskSurvives reports whether the husk a set-aside moved out of dest is +// still under a Kept name. keptBackupSurvives reads a.txt; a husk has none. +func keptHuskSurvives(t *testing.T, dir, content string) bool { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), keptPrefix) { + continue + } + got, err := os.ReadFile(filepath.Join(dir, entry.Name(), "backup", "husk.txt")) + if err == nil && string(got) == content { + return true + } + } + return false +} + +// A copy recovery parked is retained permanently. A later successful publish is +// not evidence about it: the commit flag is written into the copy the publishing +// transaction set aside, and a Kept backup is never that copy. +func TestRecoverBundleDirKeepsAnUncommittedCopyAcrossALaterPublish(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + kept := parkedStaging(stageBackup(t, dir, "", "proj-1", "unproven", 500)) + + recoverAndLog(t, dir) + if _, err := os.Stat(filepath.Join(kept, "backup", "a.txt")); err != nil { + t.Fatalf("the uncommitted copy should have been parked: %v", err) + } + + // A real publish over the same destination, then another pass. + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v2"), dest, nil); err != nil { + t.Fatalf("second extract: %v", err) + } + recoverAndLog(t, dir) + + got, err := os.ReadFile(filepath.Join(kept, "backup", "a.txt")) + if err != nil || string(got) != "unproven" { + t.Errorf("a Kept backup must survive a later publish: got %q err %v", got, err) + } +} + +// If the newest usable copy cannot be put back, recovery stops for that +// destination. Falling through to an older copy is what manufactures the +// provenance the next pass then reads as supersession of the newer one. +func TestRecoverBundleDirStopsWhenTheNewestRestoreFails(t *testing.T) { + dir := t.TempDir() + older := stageBackup(t, dir, "", "proj-1", "v1", 100) + newest := stageBackup(t, dir, "", "proj-1", "v2", 200) + dest := filepath.Join(dir, "proj-1") + + fail := func() { + injected := errors.New("injected restore failure") + injectFault(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(newest, "backup") + }, injected) + } + + for pass := 1; pass <= 2; pass++ { + var logs []string + func() { + saved := stagingFS + defer func() { stagingFS = saved }() + fail() + logs = recoverAndLog(t, dir) + }() + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatalf("pass %d: nothing may be installed when the newest copy cannot be: %v", pass, err) + } + for _, staging := range []string{older, newest} { + if _, err := os.Stat(filepath.Join(staging, "backup", "a.txt")); err != nil { + t.Fatalf("pass %d: %s must be retained in place: %v", pass, staging, err) + } + } + if !logged(logs, newest) { + t.Errorf("pass %d: the failed restore should be reported, got %v", pass, logs) + } + } + + // Once the fault is gone the newest comes back and the older is parked. + recoverAndLog(t, dir) + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v2" { + t.Fatalf("the newest copy should be restored once the fault is gone: got %q err %v", got, err) + } + if _, err := os.Stat(filepath.Join(parkedStaging(older), "backup", "a.txt")); err != nil { + t.Errorf("the older copy should be parked, not deleted: %v", err) + } +} + +// Selection is by sequence, and the loser is a copy of a tree like any other, so +// it is parked rather than deleted. +func TestRecoverBundleDirRestoresTheNewestUsableAndParksTheOlder(t *testing.T) { + dir := t.TempDir() + older := stageBackup(t, dir, "", "proj-1", "v1", 100) + newest := stageBackup(t, dir, "", "proj-1", "v2", 200) + + recoverAndLog(t, dir) + + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil || string(got) != "v2" { + t.Fatalf("restored a.txt = %q err %v, want the newest %q", got, err, "v2") + } + if _, err := os.Stat(newest); !os.IsNotExist(err) { + t.Errorf("the restored copy's directory should be gone, got %v", err) + } + if _, err := os.Stat(filepath.Join(parkedStaging(older), "backup", "a.txt")); err != nil { + t.Errorf("the older copy must be parked, not deleted: %v", err) + } +} + +// A copy that cannot serve is skipped in selection rather than installed. The +// old pass selected on "a backup directory exists", which installs a partial +// tree over an absent destination and calls it recovery. +func TestRecoverBundleDirSkipsAnUnusableNewestCandidate(t *testing.T) { + dir := t.TempDir() + older := stageBackup(t, dir, "", "proj-1", "v1", 100) + newest := stageBackup(t, dir, "", "proj-1", "v2", 200) + if err := os.RemoveAll(filepath.Join(newest, "backup", ".git")); err != nil { + t.Fatal(err) + } + + recoverAndLog(t, dir) + + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("restored a.txt = %q err %v, want the newest USABLE %q", got, err, "v1") + } + if _, err := os.Stat(older); !os.IsNotExist(err) { + t.Errorf("the restored copy's directory should be gone, got %v", err) + } + if _, err := os.Stat(filepath.Join(parkedStaging(newest), "backup", "a.txt")); err != nil { + t.Errorf("the unusable copy must be parked, not deleted: %v", err) + } +} + +// Unusable and unreadable are different facts. A durable "this copy will never +// pass the predicate" is a selection decision; a filesystem error is not a fact +// about the copy at all, and acting on it turns a fault into a delete. +func TestRecoverBundleDirStopsOnAnUnreadableCandidate(t *testing.T) { + for _, tc := range []struct { + name string + // break makes one stat under the staging dir fail with EACCES without + // making the marker unreadable: a candidate whose marker cannot be read + // lands in the unowned branch, where every mutation below passes. + breakIt func(t *testing.T, staging string) + // portable rows run everywhere. The chmod rows do not, and this + // distinction is the axis the no-fallback rule rests on, so at least one + // row has to hold on the Windows leg and under root. + portable bool + }{ + { + name: "the usability probe fails at the seam", + portable: true, + breakIt: func(t *testing.T, staging string) { + injectFault(t, "stat", func(args ...string) bool { + return args[0] == filepath.Join(staging, "backup", ".git") + }, os.ErrPermission) + }, + }, + { + name: "the set-aside copy cannot be stat'd", + breakIt: func(t *testing.T, staging string) { + locked := filepath.Join(staging, "locked") + if err := os.Rename(filepath.Join(staging, "backup"), filepath.Join(staging, "locked-backup")); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(locked, 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(filepath.Join(staging, "locked-backup"), filepath.Join(locked, "backup")); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(locked, "backup"), filepath.Join(staging, "backup")); err != nil { + t.Fatal(err) + } + if err := os.Chmod(locked, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(locked, 0o700) }) + }, + }, + { + name: "the usability probe cannot run", + breakIt: func(t *testing.T, staging string) { + backup := filepath.Join(staging, "backup") + if err := os.Chmod(backup, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(backup, 0o700) }) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if !tc.portable { + if os.Geteuid() == 0 { + t.Skip("root ignores the directory permissions this row relies on") + } + if runtime.GOOS == "windows" { + t.Skip("POSIX directory permissions") + } + } + dir := t.TempDir() + older := stageBackup(t, dir, "", "proj-1", "v1", 100) + newest := stageBackup(t, dir, "", "proj-1", "v2", 200) + tc.breakIt(t, newest) + + logs := recoverAndLog(t, dir) + + if _, err := os.Stat(filepath.Join(dir, "proj-1")); !os.IsNotExist(err) { + t.Errorf("recovery must stop for a destination it cannot read, got %v", err) + } + if _, err := os.Lstat(newest); err != nil { + t.Errorf("the unreadable copy must be retained: %v", err) + } + if _, err := os.Stat(filepath.Join(older, "backup", "a.txt")); err != nil { + t.Errorf("every copy for that destination must be retained: %v", err) + } + if !logged(logs, newest) { + t.Errorf("the unreadable copy should be reported, got %v", logs) + } + }) + } +} + +// The reap is licensed by ownership plus the destination's lock, never by a +// prefix and a clock. A live extract sits between its marker write and its +// set-aside with exactly this shape, and the lock is the only thing that tells +// the two apart. +func TestRecoverBundleDirReapsOwnedScratchOnlyUnderTheLock(t *testing.T) { + t.Run("another process holds the destination", func(t *testing.T) { + dir := t.TempDir() + scratch := stageScratch(t, dir, "proj-1", 100) + lockDir := filepath.Join(dir, lockDirName) + if err := os.MkdirAll(lockDir, 0o700); err != nil { + t.Fatal(err) + } + held, err := lockutil.TryAcquireFileLockAt(dir, filepath.Join(lockDir, "proj-1.lock")) + if err != nil { + t.Fatal(err) + } + + logs := recoverAndLog(t, dir) + if _, err := os.Stat(scratch); err != nil { + t.Errorf("a destination another process owns must be left alone: %v", err) + } + if !logged(logs, "proj-1") { + t.Errorf("the skipped destination should be reported, got %v", logs) + } + + if err := held.Release(); err != nil { + t.Fatal(err) + } + recoverAndLog(t, dir) + if _, err := os.Stat(scratch); !os.IsNotExist(err) { + t.Errorf("owned scratch should be reaped once the lock is free, got %v", err) + } + }) + + t.Run("a goroutine in this process holds the destination", func(t *testing.T) { + dir := t.TempDir() + scratch := stageScratch(t, dir, "proj-1", 100) + release := lockExtract(filepath.Join(dir, "proj-1")) + defer release() + + done := make(chan []string, 1) + go func() { done <- recoverAndLog(t, dir) }() + select { + case logs := <-done: + if !logged(logs, "proj-1") { + t.Errorf("the skipped destination should be reported, got %v", logs) + } + case <-time.After(time.Second): + t.Fatal("recovery blocked on a destination a live extract holds; it must skip it") + } + if _, err := os.Stat(scratch); err != nil { + t.Errorf("a destination a live extract holds must be left alone: %v", err) + } + }) +} + +// A directory with no marker names no destination, so no lock excludes the live +// allocation that may be sitting inside it right now. It is retained and named, +// and the cost of that is one empty directory per crash between the mkdir and +// the marker write. +func TestRecoverBundleDirRetainsAnUnmarkedStagingDir(t *testing.T) { + dir := t.TempDir() + unmarked := filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, 3, stagingSeqSuffix)) + if err := os.MkdirAll(unmarked, 0o700); err != nil { + t.Fatal(err) + } + // The v0.8.0 shape: os.MkdirTemp's decimal suffix, which carries no marker + // and may hold a partial clone. + legacy := filepath.Join(dir, stagingPrefix+"1234567890") + if err := os.MkdirAll(filepath.Join(legacy, "repo"), 0o700); err != nil { + t.Fatal(err) + } + + for pass := 1; pass <= 2; pass++ { + logs := recoverAndLog(t, dir) + for _, path := range []string{unmarked, legacy} { + if _, err := os.Stat(path); err != nil { + t.Errorf("pass %d: %s must be retained: %v", pass, path, err) + } + if !logged(logs, path) { + t.Errorf("pass %d: %s should be reported, got %v", pass, path, logs) + } + } + } +} + +// No decision reads a clock. The old pass reaped on mtime age, which makes the +// outcome depend on a forward clock jump or a suspended host rather than on what +// is on disk. +func TestRecoverBundleDirUsesNoClock(t *testing.T) { + for _, tc := range []struct { + name string + when time.Time + }{ + {name: "fresh", when: time.Now()}, + {name: "a year old", when: time.Now().Add(-365 * 24 * time.Hour)}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + scratch := stageScratch(t, dir, "proj-1", 100) + kept := stageBackup(t, dir, "", "proj-1", "v1", 200) + plantUsableDest(t, dir, "proj-1", "live") + for _, path := range []string{scratch, kept} { + if err := os.Chtimes(path, tc.when, tc.when); err != nil { + t.Fatal(err) + } + } + + recoverAndLog(t, dir) + + if _, err := os.Stat(scratch); !os.IsNotExist(err) { + t.Errorf("owned scratch is reaped whatever its age, got %v", err) + } + if _, err := os.Stat(filepath.Join(parkedStaging(kept), "backup", "a.txt")); err != nil { + t.Errorf("an uncommitted copy is kept whatever its age: %v", err) + } + }) + } +} + +// A destination that exists and cannot serve is a stuck state: the old pass saw +// "dest exists" and deleted the copy that could have replaced it. The husk moves +// only once a candidate has been selected, so a destination nothing can replace +// is never taken apart. +func TestRecoverBundleDirSetsAnUnusableDestinationAside(t *testing.T) { + dir := t.TempDir() + // Inside an enclosing checkout, which is what makes the predicate's shape + // falsifiable: git rev-parse --is-inside-work-tree discovers upward and + // answers true for a husk with no .git of its own, so a predicate that + // shells out reads this destination as usable and parks the copy that could + // have replaced it. The structural check does not discover upward. + if out, err := exec.Command("git", "init", filepath.Dir(dir)).CombinedOutput(); err != nil { + t.Fatalf("git init: %v: %s", err, out) + } + dest := plantUnusableDest(t, dir, "proj-1", "partial") + candidate := stageBackup(t, dir, "", "proj-1", "v1", 100) + + logs := recoverAndLog(t, dir) + + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("the usable copy should be live at the destination: got %q err %v", got, err) + } + if _, err := os.Stat(candidate); !os.IsNotExist(err) { + t.Errorf("the restored copy's directory should be gone, got %v", err) + } + if !keptHuskSurvives(t, dir, "partial") { + t.Error("the husk must be kept under the Kept prefix, not deleted") + } + if !logged(logs, candidate) || !logged(logs, dest) { + t.Errorf("both the restore and the set-aside should be reported, got %v", logs) + } +} + +// A legacy work tree can carry the exact generated name AND a file called txn at +// its root, so the .git veto sits ahead of the marker read and ahead of every +// delete. +func TestRecoverBundleDirNeverTrustsAMarkerInsideAWorkTree(t *testing.T) { + dir := t.TempDir() + tree := filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, 5, stagingSeqSuffix)) + if err := os.MkdirAll(filepath.Join(tree, ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tree, "a.txt"), []byte("someones repo"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeMarker(tree, txnMarker{Kind: txnKindBundleExtract, Dest: "proj-1", Seq: 5}); err != nil { + t.Fatal(err) + } + + logs := recoverAndLog(t, dir) + + if got, err := os.ReadFile(filepath.Join(tree, "a.txt")); err != nil || string(got) != "someones repo" { + t.Errorf("a published work tree must be left exactly as it is: got %q err %v", got, err) + } + if !logged(logs, "work tree") { + t.Errorf("a work tree under a reserved name should be reported, got %v", logs) + } +} + +// The scan runs before the lock is held, so everything it saw could have changed +// by the time the lock is taken. Classifying from the scan is a read of state +// nobody was holding. +func TestRecoverBundleDirRevalidatesUnderTheLock(t *testing.T) { + dir := t.TempDir() + scratch := stageScratch(t, dir, "proj-1", 100) + marker := filepath.Join(scratch, stagingMarkerFile) + + // The marker is there for the scan and gone by the time the lock is held, + // which is what a concurrent hand between the two produces. Under the lock + // the directory is unowned, and an unowned directory is never deleted. + real := stagingFS + var reads atomic.Int64 + stagingFS.readFile = func(name string) ([]byte, error) { + out, err := real.readFile(name) + if name == marker && reads.Add(1) == 1 { + if rmErr := os.Remove(name); rmErr != nil { + t.Error(rmErr) + } + } + return out, err + } + t.Cleanup(func() { stagingFS = real }) + + logs := recoverAndLog(t, dir) + + if _, err := os.Stat(scratch); err != nil { + t.Errorf("a directory that is unowned under the lock must be retained: %v", err) + } + if !logged(logs, scratch) { + t.Errorf("the retained directory should be reported, got %v", logs) + } +} + +// With no uncommitted copy to prefer, a committed one is still the last usable +// copy of the tree, so an absent destination gets it back. The commit flag +// licenses a delete only when a usable destination is there to supersede it. +func TestRecoverBundleDirRestoresACommittedCopyWhoseDestIsGone(t *testing.T) { + dir := t.TempDir() + committed := stageBackup(t, dir, "", "proj-1", "v1", 100) + markCommitted(t, committed) + + recoverAndLog(t, dir) + + got, err := os.ReadFile(filepath.Join(dir, "proj-1", "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("a committed copy beside an absent destination should be restored: got %q err %v", got, err) + } + if _, err := os.Stat(committed); !os.IsNotExist(err) { + t.Errorf("the restored copy's directory should be gone, got %v", err) + } +} + +// The husk moves only after a candidate is selected. With nothing to put in its +// place, taking it apart would leave the destination absent and the operator +// with strictly less than they started with. +func TestRecoverBundleDirLeavesAnUnusableDestinationWithNoCandidateAlone(t *testing.T) { + dir := t.TempDir() + dest := plantUnusableDest(t, dir, "proj-1", "partial") + unusable := stageBackup(t, dir, "", "proj-1", "v1", 100) + if err := os.RemoveAll(filepath.Join(unusable, "backup", ".git")); err != nil { + t.Fatal(err) + } + unowned := filepath.Join(dir, fmt.Sprintf("%s%020d%s", stagingPrefix, 200, stagingSeqSuffix)) + if err := os.MkdirAll(unowned, 0o700); err != nil { + t.Fatal(err) + } + + for pass := 1; pass <= 2; pass++ { + logs := recoverAndLog(t, dir) + got, err := os.ReadFile(filepath.Join(dest, "husk.txt")) + if err != nil || string(got) != "partial" { + t.Fatalf("pass %d: the destination must be left exactly as planted: got %q err %v", pass, got, err) + } + if _, err := os.Stat(filepath.Join(dest, "a.txt")); !os.IsNotExist(err) { + t.Errorf("pass %d: nothing may be installed over the husk: %v", pass, err) + } + if _, err := os.Stat(filepath.Join(parkedStaging(unusable), "backup", "a.txt")); err != nil { + t.Errorf("pass %d: the unusable copy must be kept: %v", pass, err) + } + if _, err := os.Stat(unowned); err != nil { + t.Errorf("pass %d: the unowned directory must stay in place: %v", pass, err) + } + if pass == 1 && (!logged(logs, unusable) || !logged(logs, unowned) || !logged(logs, "proj-1")) { + t.Errorf("pass %d: all three should be reported, got %v", pass, logs) + } + } +} + +// The marker goes in before the first destructive rename on recovery's own +// set-aside too. Only extractBundle's copy of this window was pinned; a husk +// moved into a staging dir before the marker lands is the only copy of whatever +// was at the destination, sitting in a directory nothing on disk attributes, +// which recovery can then neither restore from nor ever reclaim. +func TestSetAsideHuskWritesTheMarkerBeforeMovingTheDestination(t *testing.T) { + dir := t.TempDir() + // A destination that exists and cannot serve, plus a usable copy beside it: + // the one state that reaches the set-aside, since the husk moves only after + // a candidate has been selected. + dest := plantUnusableDest(t, dir, "proj-1", "partial") + candidate := stageBackup(t, dir, "", "proj-1", "v1", 100) + + injectFault(t, "rename", func(args ...string) bool { + return args[0] == dest + }, errors.New("injected set-aside failure")) + // The staging dir is reaped on this failure path, which is right and would + // also erase what this test is asserting on. + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + recoverBundleDir(dir, discardLog) + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + husk := "" + for _, entry := range entries { + path := filepath.Join(dir, entry.Name()) + if !strings.HasPrefix(entry.Name(), stagingPrefix) || path == candidate { + continue + } + if husk != "" { + t.Fatalf("want one staging dir beside the candidate, got %s and %s", husk, path) + } + husk = path + } + if husk == "" { + t.Fatal("the set-aside allocated no staging dir, so this test proves nothing about the order") + } + m, err := readMarker(husk) + if err != nil { + t.Fatalf("the staging dir was moved into before it carried a marker: %v", err) + } + seq, ok := stagingStamp(filepath.Base(husk)) + if !ok { + t.Fatalf("the allocator wrote a name the grammar rejects: %q", filepath.Base(husk)) + } + if m.Kind != txnKindBundleExtract || m.Dest != "proj-1" || m.Seq != seq { + t.Errorf("marker = %+v, want kind %q dest %q seq %d", m, txnKindBundleExtract, "proj-1", seq) + } + if _, err := os.Lstat(filepath.Join(husk, "backup")); !os.IsNotExist(err) { + t.Errorf("the set-aside failed, so nothing should have moved into the staging dir: %v", err) + } + if got, err := os.ReadFile(filepath.Join(dest, "husk.txt")); err != nil || string(got) != "partial" { + t.Errorf("the destination must be left exactly as found: %q err %v", got, err) + } +} + +// A failed restore after the husk was set aside must leave the destination +// exactly as it was found. Anything else is recovery that made the state worse +// than the crash did. +func TestRecoverBundleDirPutsTheHuskBackWhenTheRestoreFails(t *testing.T) { + dir := t.TempDir() + dest := plantUnusableDest(t, dir, "proj-1", "partial") + candidate := stageBackup(t, dir, "", "proj-1", "v1", 100) + + for pass := 1; pass <= 2; pass++ { + var logs []string + func() { + saved := stagingFS + defer func() { stagingFS = saved }() + injectFault(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(candidate, "backup") + }, errors.New("injected restore failure")) + logs = recoverAndLog(t, dir) + }() + + got, err := os.ReadFile(filepath.Join(dest, "husk.txt")) + if err != nil || string(got) != "partial" { + t.Fatalf("pass %d: the husk must be back at the destination: got %q err %v", pass, got, err) + } + if _, err := os.Stat(filepath.Join(candidate, "backup", "a.txt")); err != nil { + t.Errorf("pass %d: the candidate must be retained at its staging name: %v", pass, err) + } + if keptHuskSurvives(t, dir, "partial") { + t.Errorf("pass %d: no Kept entry may exist while the restore has not landed", pass) + } + if !logged(logs, candidate) { + t.Errorf("pass %d: the failed restore should be reported, got %v", pass, logs) + } + } + + recoverAndLog(t, dir) + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("once the fault is gone the candidate should be restored: got %q err %v", got, err) + } + if !keptHuskSurvives(t, dir, "partial") { + t.Error("the husk should be parked once the restore lands") + } +} + +// The reap boundary of a live extract: between its marker write and its +// set-aside a staging dir is owned and holds nothing, which is exactly the shape +// the reap deletes. Only the destination's lock separates that from a crash, so +// a recovering pass must find the destination held and leave the whole thing +// alone. blockStep gates every call to a step, and the marker write is a rename +// too, so this gates the set-aside by its arguments instead. +func TestRecoverBundleDirLeavesALiveExtractsScratchAlone(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + if err := extractBundle(context.Background(), testBundle(t, "a.txt", "v0"), dest, nil); err != nil { + t.Fatalf("seed extract: %v", err) + } + + inSetAside := make(chan struct{}) + gate := make(chan struct{}) + real := stagingFS + var once sync.Once + stagingFS.rename = func(from, to string) error { + if filepath.Base(to) == "backup" { + once.Do(func() { close(inSetAside) }) + <-gate + } + return real.rename(from, to) + } + t.Cleanup(func() { stagingFS = real }) + + var wg sync.WaitGroup + var extractErr error + wg.Add(1) + go func() { + defer wg.Done() + extractErr = extractBundle(context.Background(), testBundle(t, "a.txt", "v1"), dest, nil) + }() + + <-inSetAside + staging := soleStaging(t, dir) + if _, err := os.Stat(filepath.Join(staging, "backup")); !os.IsNotExist(err) { + t.Fatalf("the fixture must be an owned staging dir with no set-aside copy yet, got %v", err) + } + recoverBundleDir(dir, nil) + if _, err := os.Stat(filepath.Join(staging, stagingMarkerFile)); err != nil { + t.Errorf("a live extract's scratch must survive a recovering pass: %v", err) + } + close(gate) + wg.Wait() + + if extractErr != nil { + t.Errorf("recovery interfered with a live extract: %v", extractErr) + } + got, err := os.ReadFile(filepath.Join(dest, "a.txt")) + if err != nil || string(got) != "v1" { + t.Fatalf("a.txt = %q err %v, want the published %q", got, err, "v1") + } +} + +// ---- the reclaim surface --------------------------------------------------- + +// plantKeptBackup writes what recovery leaves under the Kept prefix: a staged +// copy, named and marked for linkID, moved out of the prefix the scan +// enumerates. Going through stageBackup and the real park name is what keeps the +// fixture from testing a shape production never writes. +func plantKeptBackup(t *testing.T, dir, linkID, content string, seq int64) string { + t.Helper() + staging := stageBackup(t, dir, "", linkID, content, seq) + kept := filepath.Join(dir, keptPrefix+strings.TrimPrefix(filepath.Base(staging), stagingPrefix)) + if err := os.Rename(staging, kept); err != nil { + t.Fatal(err) + } + return kept +} + +// keptBytes is what the listing should report for a backup plantKeptBackup +// wrote: the marker and the one file in the copy. Summed from the two names the +// fixture created rather than by walking, so the assertion is not the +// implementation restated. +func keptBytes(t *testing.T, kept, content string) int64 { + t.Helper() + info, err := os.Stat(filepath.Join(kept, stagingMarkerFile)) + if err != nil { + t.Fatal(err) + } + return info.Size() + int64(len(content)) +} + +func findKept(t *testing.T, list []KeptBackup, path string) KeptBackup { + t.Helper() + for _, b := range list { + if b.Path == path { + return b + } + } + t.Fatalf("%s is missing from the listing %v", path, list) + return KeptBackup{} +} + +// A Kept backup an operator cannot find is one that can never be reclaimed, and +// this listing is the only thing that finds them: recovery deliberately never +// enumerates the prefix. The destination comes off the marker, because the +// bundle site's Kept name carries no link id at all, and anything that merely +// carries the name is reported unowned so recovery's own residue is visible +// beside the real backups. +func TestListKeptBackupsReportsDestSeqAndSize(t *testing.T) { + dir := t.TempDir() + first := plantKeptBackup(t, dir, "proj-1", "one", 1) + second := plantKeptBackup(t, dir, "proj-2", "second-copy", 2) + + // Kept grammar, nothing attributing it: the empty directory a crash between + // Mkdir and the marker write leaves, parked by a later pass. + bare := filepath.Join(dir, fmt.Sprintf("%s%020d%s", keptPrefix, 3, stagingSeqSuffix)) + if err := os.Mkdir(bare, 0o700); err != nil { + t.Fatal(err) + } + // A marker that contradicts the name it sits in proves nothing about who + // wrote either of them. + skewed := plantKeptBackup(t, dir, "proj-3", "skew", 4) + if err := writeMarker(skewed, txnMarker{Kind: txnKindBundleExtract, Dest: "proj-3", Seq: 99}); err != nil { + t.Fatal(err) + } + // A sibling that fails the grammar is not a Kept backup and is not listed. + if err := os.Mkdir(filepath.Join(dir, keptPrefix+"notanumber-seq"), 0o700); err != nil { + t.Fatal(err) + } + + list, err := ListKeptBackups(dir) + if err != nil { + t.Fatalf("ListKeptBackups: %v", err) + } + if len(list) != 4 { + t.Fatalf("listed %d kept backups, want 4: %v", len(list), list) + } + + got := findKept(t, list, first) + if !got.Owned || got.Dest != "proj-1" || got.Seq != 1 || got.Bytes != keptBytes(t, first, "one") { + t.Errorf("first = %+v, want owned proj-1 seq 1 bytes %d", got, keptBytes(t, first, "one")) + } + got = findKept(t, list, second) + if !got.Owned || got.Dest != "proj-2" || got.Seq != 2 || got.Bytes != keptBytes(t, second, "second-copy") { + t.Errorf("second = %+v, want owned proj-2 seq 2 bytes %d", got, keptBytes(t, second, "second-copy")) + } + for _, path := range []string{bare, skewed} { + got := findKept(t, list, path) + if got.Owned { + t.Errorf("%s is attributed by nothing on disk and must be listed unowned, got %+v", path, got) + } + // The destination has to be empty too: naming one the marker does not + // support tells the operator this copy belongs to a link it may not. + if got.Dest != "" { + t.Errorf("%s: unowned entries carry no destination, got %q", path, got.Dest) + } + } +} + +// Removal carries the same ownership proof recovery's own deletes carry. It is +// the one thing on this machine that deletes a Kept backup, so an entry nothing +// attributes has to survive it: the operator is told to remove that one by hand. +func TestRemoveKeptBackupRefusesAnUnownedEntry(t *testing.T) { + t.Run("no marker", func(t *testing.T) { + dir := t.TempDir() + kept := plantKeptBackup(t, dir, "proj-1", "one", 1) + if err := os.Remove(filepath.Join(kept, stagingMarkerFile)); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err == nil { + t.Fatal("a directory with no marker must not be removed") + } + if _, err := os.Stat(filepath.Join(kept, "backup", "a.txt")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) + t.Run("marker disagrees with the name", func(t *testing.T) { + dir := t.TempDir() + kept := plantKeptBackup(t, dir, "proj-1", "one", 1) + if err := writeMarker(kept, txnMarker{Kind: txnKindBundleExtract, Dest: "proj-1", Seq: 42}); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err == nil { + t.Fatal("a marker for another sequence must not license a removal") + } + if _, err := os.Stat(filepath.Join(kept, "backup", "a.txt")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) + // Link ids beginning with '.' used to be accepted, so a published work tree + // can sit under the exact Kept name and carry a file called txn at its root. + // The work-tree veto has to run ahead of the marker read, or that file + // licenses removing a user's checkout. + t.Run("work tree beside a valid-looking marker", func(t *testing.T) { + dir := t.TempDir() + kept := plantKeptBackup(t, dir, "proj-1", "one", 1) + if err := os.MkdirAll(filepath.Join(kept, ".git"), 0o700); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err == nil { + t.Fatal("a work tree must not be removed whatever its marker says") + } + if _, err := os.Stat(filepath.Join(kept, ".git")); err != nil { + t.Errorf("the work tree must be left intact: %v", err) + } + }) +} + +// The lock is what separates a Kept backup from one a live extract is about to +// restore from: on disk the two are the same directory, and removing one +// mid-transaction takes the copy the transaction rolls back to. +func TestRemoveKeptBackupRefusesWhileTheDestinationIsLocked(t *testing.T) { + t.Run("in process", func(t *testing.T) { + dir := t.TempDir() + kept := plantKeptBackup(t, dir, "proj-1", "one", 1) + unlock := lockExtract(filepath.Join(dir, "proj-1")) + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err == nil { + unlock() + t.Fatal("a removal must not run while an extract in this process holds the link") + } + if _, err := os.Stat(kept); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + unlock() + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err != nil { + t.Fatalf("the same removal must go through once the lock is free: %v", err) + } + if _, err := os.Stat(kept); !os.IsNotExist(err) { + t.Errorf("the copy should be gone, got %v", err) + } + }) + t.Run("another process", func(t *testing.T) { + dir := t.TempDir() + kept := plantKeptBackup(t, dir, "proj-1", "one", 1) + lockDir := filepath.Join(dir, lockDirName) + if err := os.MkdirAll(lockDir, 0o700); err != nil { + t.Fatal(err) + } + held, err := lockutil.TryAcquireFileLockAt(dir, filepath.Join(lockDir, "proj-1.lock")) + if err != nil { + t.Fatalf("take the lock as the other daemon would: %v", err) + } + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err == nil { + _ = held.Release() + t.Fatal("a removal must not run while another process holds the link") + } + if _, err := os.Stat(kept); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + if err := held.Release(); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err != nil { + t.Fatalf("the same removal must go through once the lock is free: %v", err) + } + }) +} + +// The attribution runs twice: once before the lock and once under it. The first +// read happens with nothing excluding a live extract, so a directory that +// stopped being this link's in between must not be deleted on the strength of +// that reading. Both halves of the second check are driven here, because +// deleting the whole block leaves the package green. +func TestRemoveKeptBackupRefusesWhenAttributionChangesUnderTheLock(t *testing.T) { + t.Run("the marker goes unreadable", func(t *testing.T) { + dir := t.TempDir() + kept := plantKeptBackup(t, dir, "proj-1", "one", 1) + marker := filepath.Join(kept, stagingMarkerFile) + + var read atomic.Bool + injectFault(t, "readFile", func(args ...string) bool { + // The pre-lock attribution reads the marker first; the re-read under + // the lock is the one this fault is for. + return args[0] == marker && !read.CompareAndSwap(false, true) + }, errors.New("injected marker read failure")) + + err := RemoveKeptBackup(dir, filepath.Base(kept)) + if err == nil || !strings.Contains(err.Error(), "no longer attributable") { + t.Fatalf("RemoveKeptBackup = %v, want a refusal naming the attribution that changed", err) + } + if _, err := os.Stat(filepath.Join(kept, "backup", "a.txt")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) + t.Run("the marker names another link", func(t *testing.T) { + dir := t.TempDir() + kept := plantKeptBackup(t, dir, "proj-1", "one", 1) + marker := filepath.Join(kept, stagingMarkerFile) + // A readable marker for a different link is the other half: the re-read + // succeeds and answers with an id the first one did not, which is what a + // directory reused by another extract looks like from here. + other, err := json.Marshal(txnMarker{Kind: txnKindBundleExtract, Dest: "proj-2", Seq: 1}) + if err != nil { + t.Fatal(err) + } + + real := stagingFS + t.Cleanup(func() { stagingFS = real }) + var read atomic.Bool + stagingFS.readFile = func(name string) ([]byte, error) { + if name != marker || read.CompareAndSwap(false, true) { + return real.readFile(name) + } + return other, nil + } + + if err := RemoveKeptBackup(dir, filepath.Base(kept)); err == nil || !strings.Contains(err.Error(), "no longer attributable") { + t.Fatalf("RemoveKeptBackup = %v, want a refusal naming the attribution that changed", err) + } + if _, err := os.Stat(filepath.Join(kept, "backup", "a.txt")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) +} + +// The command takes a name, never a path. A separator in the argument would join +// to a directory outside the bundle dir, which turns an operator's reclaim into +// a remote rm, so the name is refused before any filesystem call happens at all. +func TestRemoveKeptBackupRejectsANameThatIsNotABaseName(t *testing.T) { + dir := t.TempDir() + injectFault(t, "removeAll", func(args ...string) bool { + t.Errorf("a rejected name must not reach the filesystem, got removeAll(%v)", args) + return false + }, nil) + injectFault(t, "stat", func(args ...string) bool { + t.Errorf("a rejected name must not reach the filesystem, got stat(%v)", args) + return false + }, nil) + injectFault(t, "readFile", func(args ...string) bool { + t.Errorf("a rejected name must not reach the filesystem, got readFile(%v)", args) + return false + }, nil) + for _, name := range []string{"../x", "a/b", filepath.Join(dir, ".kept-x"), ".", "..", ""} { + if err := RemoveKeptBackup(dir, name); err == nil { + t.Errorf("RemoveKeptBackup(%q) = nil, want a refusal", name) + } + } +} + +// The scanned prefix is recovery's, not the operator's: a directory under it may +// be the staging dir of an extract running right now, and the next pass has a +// disposition for it either way. Only the Kept prefix is this command's to touch. +func TestRemoveKeptBackupNeverTouchesTheScannedPrefix(t *testing.T) { + dir := t.TempDir() + staging := stageBackup(t, dir, "", "proj-1", "one", 1) + if err := RemoveKeptBackup(dir, filepath.Base(staging)); err == nil { + t.Fatal("a directory under the scanned prefix must not be removed by name") + } + if _, err := os.Stat(filepath.Join(staging, "backup", "a.txt")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } +} + +// A stat error on the work-tree probe is not proof there is no work tree. R1 +// puts the veto ahead of the marker precisely because a published tree can +// carry a file named txn at its root, so an unreadable probe has to retain +// rather than fall through to a marker the tree itself may have supplied. +func TestAttributeStagingDirRetainsWhenTheWorkTreeProbeFails(t *testing.T) { + dir := t.TempDir() + staging := filepath.Join(dir, ".staging-00000000000000000001-seq") + if err := os.MkdirAll(filepath.Join(staging, ".git"), 0o755); err != nil { + t.Fatal(err) + } + if err := writeMarker(staging, txnMarker{Kind: txnKindBundleExtract, Dest: "proj-1", Seq: 1}); err != nil { + t.Fatal(err) + } + injectFault(t, "stat", func(args ...string) bool { + return filepath.Base(args[0]) == ".git" + }, errors.New("injected work-tree probe failure")) + + if _, ok, _ := attributeStagingDir(dir, staging, 1, func(string, ...any) {}); ok { + t.Fatal("a directory whose work-tree probe could not be read must not be attributed") + } +} + +// A marker that read cleanly during the scan and fails when it is re-read under +// the lock stops that destination. The scan already named the destination, so +// the stop can be scoped to it; an entry whose marker never read at all carries +// no destination and is retained by the scan instead, since blocking every link +// on this host would cost more than the one wrong version it prevents. +func TestRecoverBundleDirStopsWhenAMarkerGoesUnreadableUnderTheLock(t *testing.T) { + dir := t.TempDir() + newest := stageBackup(t, dir, "", "proj-1", "v2", 2) + older := stageBackup(t, dir, "", "proj-1", "v1", 1) + + var scanned atomic.Bool + injectFault(t, "readFile", func(args ...string) bool { + if !strings.HasPrefix(args[0], newest) { + return false + } + // The scan reads every marker first; the re-read under the lock is the + // one this fault is for. + return !scanned.CompareAndSwap(false, true) + }, errors.New("injected marker read failure")) + + recoverBundleDir(dir, func(string, ...any) {}) + + if _, err := os.Lstat(filepath.Join(dir, "proj-1")); !os.IsNotExist(err) { + t.Fatalf("recovery published a copy while a candidate went unreadable under the lock: %v", err) + } + for _, staging := range []string{newest, older} { + if _, err := os.Stat(filepath.Join(staging, "backup", ".git")); err != nil { + t.Errorf("every copy must be retained until the unreadable one can be read: %v", err) + } + } +} + +// The in-process lock is taken before the one carrying a budget, so an upload +// queued behind a live extract used to wait on a bare mutex with no bound at +// all. A client that keeps sending to one link id then pins a connection slot +// per upload and nothing ever gives the slot back. +func TestExtractBundleGivesUpWaitingForTheInProcessLock(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "proj-1") + release := lockExtract(dest) + defer release() + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + done := make(chan error, 1) + go func() { done <- extractBundle(ctx, filepath.Join(dir, "absent.bundle"), dest, nil) }() + + select { + case err := <-done: + if err == nil { + t.Fatal("a queued extract must not report success while another holds the link") + } + case <-time.After(5 * time.Second): + t.Fatal("a queued extract waited on the in-process lock with no bound") + } +} + +// Every not-exist probe in this file runs on a seam result, and a seam can hand +// back a wrapped error. os.IsNotExist does not unwrap, so a wrapped ENOENT took +// the opposite branch from the one the same syscall takes unwrapped: the +// classifier read "no backup here" as "this copy cannot be read" and stopped a +// destination that had nothing wrong with it. +func TestClassifierTreatsAWrappedNotExistAsAbsent(t *testing.T) { + dir := t.TempDir() + staging := stageBackup(t, dir, "", "proj-1", "v1", 1) + if err := os.RemoveAll(filepath.Join(staging, "backup")); err != nil { + t.Fatal(err) + } + injectFault(t, "stat", func(args ...string) bool { + return args[0] == filepath.Join(staging, "backup") + }, fmt.Errorf("seam wrapped: %w", fs.ErrNotExist)) + + recoverBundleDir(dir, func(string, ...any) {}) + + if _, err := os.Stat(staging); !os.IsNotExist(err) { + t.Errorf("a marked copy holding nothing is owned-and-empty and should be reaped, got %v", err) + } +} + +// Windows strips a trailing dot and a trailing space when it resolves a path, +// so two link ids that differ only there name one directory on that platform. +// The two extract locks key on the id, not on the resolved path, so the pair +// would take different locks and clone into the same tree at the same time. +// Reserved device names resolve away from the directory entirely. +func TestSanitizeLinkIDRefusesNamesThatAreNotDistinctOnWindows(t *testing.T) { + for _, id := range []string{"proj.", "proj..", "CON", "con", "nul", "COM1", "lpt9", "AUX.txt"} { + if got, err := sanitizeLinkID(id); err == nil { + t.Errorf("sanitizeLinkID(%q) = %q, want a refusal: it is not one distinct directory on every platform", id, got) + } + } + for _, id := range []string{"proj-1", "proj.git", "console", "com10", "auxiliary", "nulls"} { + if _, err := sanitizeLinkID(id); err != nil { + t.Errorf("sanitizeLinkID(%q) = %v, want it accepted", id, err) + } + } + // Surrounding space is not aliasing: it is trimmed before any path is + // built, so both spellings become one id and take one lock. + trimmed, err := sanitizeLinkID("proj-1 ") + if err != nil || trimmed != "proj-1" { + t.Errorf(`sanitizeLinkID("proj-1 ") = %q, %v; want it normalized to "proj-1"`, trimmed, err) + } +} diff --git a/internal/dictation/download.go b/internal/dictation/download.go index b1343327e..c444b5877 100644 --- a/internal/dictation/download.go +++ b/internal/dictation/download.go @@ -2,18 +2,29 @@ package dictation import ( "archive/tar" + "cmp" "compress/bzip2" "context" "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" + "io/fs" + "math" "net/http" "os" "path/filepath" "runtime" + "slices" + "strconv" "strings" + "sync/atomic" + "time" + + "github.com/Gitlawb/zero/internal/fsutil" + "github.com/Gitlawb/zero/internal/lockutil" ) // Auto-download of the local engine + a default model (opt-in, behind a confirm @@ -483,25 +494,40 @@ func EnsureLocalEngine(ctx context.Context, opts DownloadOptions) (EngineCompone apiBase = defaultAPIBase } - engineDir := filepath.Join(opts.DestRoot, "engine-"+version+"-"+key) + engineName := "engine-" + version + "-" + key + engineDir := filepath.Join(opts.DestRoot, engineName) targetWindows := strings.HasPrefix(key, "windows-") // Resolve through the tarball's flattened subdir so an ALREADY-extracted // engine is found and not needlessly re-downloaded (the idempotency check). - binPath, serverPath := resolveEnginePaths(engineDir, targetWindows) - if !fileExists(binPath) { + enginePublished := func(dir string) bool { + bin, _ := resolveEnginePaths(dir, targetWindows) + return fileExists(bin) + } + // The engine lock covers recovery, the decision to download, and the + // promotion, and it is released before the model lock is taken. Holding + // both would give two concurrent installs of different models an ordering + // to get wrong for no gain. + if err := withDestinationLock(ctx, opts.DestRoot, engineName, enginePublished, func(txn *destTxn) error { + // A previous run may have been stopped mid-promotion, leaving the only + // install in a holder beside engineDir. Put it back before deciding + // whether anything needs downloading. + restoreInterruptedPromotion(txn, engineDir, enginePublished, progress) + if enginePublished(engineDir) { + return nil + } pinned := "" if version == DefaultSherpaVersion && !opts.skipPinned { pinned = pinnedEngineDigest[key] } asset, err := resolveAsset(ctx, client, apiBase, version, "sherpa-onnx-", suffix) if err != nil { - return EngineComponents{}, err - } - if err := downloadVerifyExtract(ctx, client, asset, pinned, false, "Engine", engineDir, progress); err != nil { - return EngineComponents{}, err + return err } - binPath, serverPath = resolveEnginePaths(engineDir, targetWindows) + return downloadVerifyExtract(ctx, client, asset, pinned, false, "Engine", engineDir, txn, progress) + }); err != nil { + return EngineComponents{}, err } + binPath, serverPath := resolveEnginePaths(engineDir, targetWindows) if !fileExists(binPath) { return EngineComponents{}, fmt.Errorf("dictation download: engine binary not found after extraction in %s", engineDir) } @@ -515,10 +541,18 @@ func EnsureLocalEngine(ctx context.Context, opts DownloadOptions) (EngineCompone modelDirName = "model-moonshine-tiny-en-int8" } modelDir := filepath.Join(opts.DestRoot, modelDirName) - if !dirHasModel(modelDir) { + if err := withDestinationLock(ctx, opts.DestRoot, modelDirName, dirHasModel, func(txn *destTxn) error { + // promoteStagedDir is shared with the model, so a stop mid-promotion + // leaves the model in a holder too. Put it back before deciding + // anything is missing: without this an offline user has no download to + // fall back on. + restoreInterruptedPromotion(txn, modelDir, dirHasModel, progress) + if dirHasModel(modelDir) { + return nil + } asset, err := resolveAsset(ctx, client, apiBase, modelReleaseTag, modelName, "") if err != nil { - return EngineComponents{}, err + return err } modelPinned := opts.ModelPinnedDigest // Only fall back to the built-in digest for the DEFAULT model — a different @@ -538,9 +572,9 @@ func EnsureLocalEngine(ctx context.Context, opts DownloadOptions) (EngineCompone // download (the model release predates GitHub's per-asset digests). The // engine BINARY above never allows this — a native executable is always // digest-verified. - if err := downloadVerifyExtract(ctx, client, asset, modelPinned, true, modelLabel, modelDir, progress); err != nil { - return EngineComponents{}, err - } + return downloadVerifyExtract(ctx, client, asset, modelPinned, true, modelLabel, modelDir, txn, progress) + }); err != nil { + return EngineComponents{}, err } resolvedModel := modelDir if !hasTokensFile(resolvedModel) { @@ -615,7 +649,12 @@ func resolveAsset(ctx context.Context, client *http.Client, apiBase, tag, namePr // (against the API digest, and — when pinned is set — a cross-check that the // resolved digest equals the audited value), and extracts the tar.bz2. A // mismatch aborts before anything is extracted or run. -func downloadVerifyExtract(ctx context.Context, client *http.Client, asset resolvedAsset, pinned string, allowUnverified bool, label, destDir string, progress func(string)) error { +func downloadVerifyExtract(ctx context.Context, client *http.Client, asset resolvedAsset, pinned string, allowUnverified bool, label, destDir string, txn *destTxn, progress func(string)) error { + // Refused here as well as in promoteStagedDir, so a caller that forgot the + // lock does not get a full download and extraction before finding out. + if !txn.holds(destDir) { + return fmt.Errorf("dictation download: refusing to install %s: no install lock is held for %s", label, destDir) + } // Determine the digest to verify against: the API digest and the pinned digest // must agree when both exist; at least one must exist unless allowUnverified // (models are data files from the official release — TLS-only is acceptable; @@ -695,13 +734,8 @@ func downloadVerifyExtract(ctx context.Context, client *http.Client, asset resol if err := extractTarBz2(tmpPath, stageDir, "Extracting "+label, progress); err != nil { return err } - // Remove any pre-existing destDir so Rename is atomic on the same - // filesystem (os.Rename refuses to overwrite a non-empty dir). - if err := os.RemoveAll(destDir); err != nil { - return fmt.Errorf("clearing previous %s install: %w", label, err) - } - if err := os.Rename(stageDir, destDir); err != nil { - return fmt.Errorf("promoting staged %s: %w", label, err) + if err := promoteStagedDir(txn, stageDir, destDir, label, progress); err != nil { + return err } cleanupStage = false return nil @@ -757,6 +791,991 @@ func resolveEnginePaths(engineDir string, targetWindows bool) (bin, server strin return bin, server } +// holderSuffix is what promoteStagedDir appends to an install's own name for the +// holder it sets that install aside in. An ordering number goes in the name +// because recovery has to pick the NEWEST holder when a failed cleanup left an +// older one behind, and nothing else records that order: Glob sorts lexically +// and a directory mtime tracks the install's contents, not its promotion. The +// number is a per-install sequence, one past the highest already beside this +// install, so no decision here reads a clock. There is no older shape to accept: +// released versions staged under .stage-* and published with a plain rename, +// and none of them ever created a holder. +// fsOps is the filesystem seam the promotion path, the holder allocator, and +// recovery take every step through. A step that calls os directly cannot be made +// to fail, and the crash each of these steps exists to survive is then only +// reasoned about. Every field is one call, so a test can fail the second remove +// or the rename whose source is one particular holder and leave the rest real. +type fsOps struct { + rename func(from, to string) error + removeAll func(path string) error + stat func(name string) (os.FileInfo, error) + lstat func(name string) (os.FileInfo, error) + readDir func(name string) ([]os.DirEntry, error) + readFile func(name string) ([]byte, error) + mkdir func(name string, perm os.FileMode) error + writeFile func(name string, data []byte, perm os.FileMode) error + create func(name string, flag int, perm os.FileMode) (*os.File, error) + createTemp func(dir, pattern string) (*os.File, error) +} + +// holderFS is the seam every filesystem step in the promotion path goes through. +// Tests swap a field and restore it; nothing else writes to it. +var holderFS = fsOps{ + rename: os.Rename, + removeAll: os.RemoveAll, + stat: os.Stat, + lstat: os.Lstat, + readDir: os.ReadDir, + readFile: os.ReadFile, + mkdir: os.Mkdir, + writeFile: os.WriteFile, + create: os.OpenFile, + createTemp: os.CreateTemp, +} + +// errInstallInProgress reports that another process held this destination's +// Install lock for the whole wait budget and the destination is still not +// usable. It is not an install failure: nothing was attempted, and the caller +// reports it and moves on. +var errInstallInProgress = errors.New("another install of this component is already running") + +// installLockWait bounds how long a caller waits for another process to finish +// with a destination. It covers a download, so a budget shorter than the +// critical section it guards is a scheduled failure. A var so a test can shorten +// it; nothing else assigns to it. +var installLockWait = 2 * time.Minute + +// installLockPoll is the retry interval while waiting. The lock is advisory and +// held by an open file handle, so there is nothing to wake us and polling is how +// the wait ends. +const installLockPoll = 50 * time.Millisecond + +// installLockDir holds one lock file per destination. It is a sibling of the +// destinations rather than a file beside each one, so a lock file is never +// mistaken for part of an install and never moves with one. +const installLockDir = ".install-locks" + +// destTxn is the handle proving its holder owns the Install lock for one +// destination. Recovery and promotion move and delete whole installs, so both +// take one: without it a second process can delete the copy this one is about +// to roll back to. +type destTxn struct { + root string + dest string + lock *lockutil.FileLock + // released is atomic because holds is the guard every locked entry point + // consults, and a handle can be released on one goroutine while another is + // still asking whether it is safe to act. + released atomic.Bool +} + +// destDir is the destination this handle locks. Callers keep passing their own +// destDir alongside the handle so a handle for a DIFFERENT destination is +// representable, which is what makes the mismatch testable rather than assumed. +func (t *destTxn) destDir() string { return filepath.Join(t.root, t.dest) } + +// holds reports whether this handle actually locks destDir. A nil handle and a +// handle for another destination both fail closed. +func (t *destTxn) holds(destDir string) bool { + return t != nil && t.lock != nil && !t.released.Load() && t.destDir() == destDir +} + +// release drops the Install lock. Idempotent, so a caller can release early and +// still defer it. +func (t *destTxn) release() { + if t == nil || t.lock == nil { + return + } + if t.released.Swap(true) { + return + } + _ = t.lock.Release() +} + +// isInstallDestName reports whether name is one path component naming an install +// destination. A destination is one path component: any other shape puts the +// lock file somewhere else, and two callers for the same destination would then +// lock different inodes and both proceed. Recovery runs the same check over the +// destination name it reads out of a marker, so a name off disk is never trusted +// further than one the caller supplied. +func isInstallDestName(name string) bool { + return name != "" && name == filepath.Base(name) && name != "." && name != ".." +} + +// lockDestination takes the Install lock for one destination under destRoot, +// waiting out another process that holds it until ctx ends or the budget runs +// out. The budget expiring is errInstallInProgress, which the caller answers by +// re-checking the destination rather than by failing. +func lockDestination(ctx context.Context, destRoot, dest string) (*destTxn, error) { + deadline := time.Now().Add(installLockWait) + for { + txn, held, err := tryLockDestination(destRoot, dest) + if err != nil { + return nil, err + } + if !held { + return txn, nil + } + // Waiting is the normal case: the holder is almost always another + // process installing this very component, and failing on the first + // contended try would turn that into a user-visible install failure. + if !time.Now().Before(deadline) { + return nil, fmt.Errorf("%w: %s is locked by another process", errInstallInProgress, dest) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(installLockPoll): + } + } +} + +// tryLockDestination takes one destination's Install lock without waiting. Held +// is not an error: a caller either waits the holder out, as lockDestination +// does, or leaves that destination alone, as the operator's reclaim does. An +// operator command cannot use the waiting form, because the wait budget covers a +// download and stalling a terminal for two minutes to report a refusal is not a +// refusal anyone reads. +func tryLockDestination(destRoot, dest string) (txn *destTxn, held bool, err error) { + if !isInstallDestName(dest) { + return nil, false, fmt.Errorf("dictation download: %q is not an install destination name", dest) + } + // destRoot is otherwise created lazily by the download this lock covers, and + // lockutil opens the root with O_DIRECTORY, so on a first run there would be + // no directory to take the lock in. + if err := os.MkdirAll(destRoot, 0o755); err != nil { + return nil, false, err + } + lockDir := filepath.Join(destRoot, installLockDir) + if err := holderFS.mkdir(lockDir, 0o700); err != nil && !errors.Is(err, fs.ErrExist) { + return nil, false, err + } + lock, err := lockutil.TryAcquireFileLockAt(destRoot, filepath.Join(lockDir, dest+".lock")) + if err != nil { + if errors.Is(err, lockutil.ErrLockHeld) { + return nil, true, nil + } + return nil, false, err + } + return &destTxn{root: destRoot, dest: dest, lock: lock}, false, nil +} + +// withDestinationLock runs fn under dest's Install lock. A wait that runs out is +// not by itself a failure: the likeliest reason is that the other process +// finished this exact install, so usable re-checks the destination before the +// caller is told anything is wrong. +func withDestinationLock(ctx context.Context, destRoot, dest string, usable func(string) bool, fn func(*destTxn) error) error { + txn, err := lockDestination(ctx, destRoot, dest) + if err != nil { + if errors.Is(err, errInstallInProgress) && usable(filepath.Join(destRoot, dest)) { + return nil + } + return err + } + defer txn.release() + return fn(txn) +} + +const holderSuffix = ".previous-" + +// keptSuffix names a holder recovery decided to retain rather than restore or +// delete. It is a different prefix from holderSuffix so recovery's scan never +// enumerates a copy it has already ruled on, while the allocator still counts +// both and cannot reissue a number a Kept backup already carries. +const keptSuffix = ".kept-" + +// holderSeqDigits is how many digits a sequenced name carries. Fixed width is +// what makes the grammar exact: a name one digit short is a name this code did +// not write, whatever else it looks like. +const holderSeqDigits = 20 + +// holderStamp reads back the ordering number in a holder name, reporting false +// for any name this code did not write. The grammar is exact and it is the +// first ownership filter: the install's own base name, then .previous- or +// .kept-, then twenty digits, then holderSeqSuffix, and nothing else. Loose +// parsing is what let a sibling merely NAMED like a holder be attributed to the +// install and moved on its account. No released version wrote a holder at all, +// so there is no older shape to accept. +func holderStamp(destDir, holder string) (int64, bool) { + name := filepath.Base(holder) + base := filepath.Base(destDir) + var rest string + switch { + case strings.HasPrefix(name, base+holderSuffix): + rest = name[len(base)+len(holderSuffix):] + case strings.HasPrefix(name, base+keptSuffix): + rest = name[len(base)+len(keptSuffix):] + default: + return 0, false + } + digits, ok := strings.CutSuffix(rest, holderSeqSuffix) + if !ok || len(digits) != holderSeqDigits { + return 0, false + } + for _, c := range []byte(digits) { + // ParseInt would take a leading '+' or '-', and a signed number renders + // back through %020d as a different string, so the writer and the reader + // would disagree about a name they both accept. + if c < '0' || c > '9' { + return 0, false + } + } + stamp, err := strconv.ParseInt(digits, 10, 64) + if err != nil { + return 0, false + } + return stamp, true +} + +// holderSeqSuffix closes a sequenced holder name. It is what the grammar ends +// on, so a name that stops at the digits, or carries anything after the suffix, +// is not one this code wrote. +const holderSeqSuffix = "-seq" + +// holderSeqAttempts bounds the walk up from a taken number, in the spirit of the +// retry limit os.MkdirTemp applies to its own random names. +const holderSeqAttempts = 10000 + +// holderMarkerFile names the transaction marker written inside every holder +// before anything is moved into it. The name alone is not ownership: any +// directory can be named to look like a holder, and moving or deleting one on +// its name is how a sibling that merely collides with the prefix loses its +// contents. +const holderMarkerFile = "txn" + +// committedFile is the flag promoteStagedDir creates inside a holder after the +// publish rename succeeds. It is the only evidence that the copy in the holder +// was superseded by an install that actually landed, and so the only thing that +// licenses removing it. +const committedFile = "committed" + +// holderMarkerKind is what this site writes in a marker's kind field. The two +// install sites use one marker format and different kinds, so a directory +// written by the other one is never read as this one's. +const holderMarkerKind = "dictation-promote" + +// txnMarker is the ownership proof inside a holder: the transaction that created +// it, the destination it was created for, and the sequence in its name. All +// three have to agree with what recovery already knows before the holder is +// this code's to touch. +type txnMarker struct { + Kind string `json:"kind"` + Dest string `json:"dest"` + Seq int64 `json:"seq"` +} + +// errMarkerMissing separates "there is no marker" from "the marker could not be +// read". The first is a directory this code cannot claim; the second is a +// filesystem fault, and treating them alike would let a transient read error +// silently reclassify a copy. +var errMarkerMissing = errors.New("no transaction marker") + +// writeHolderMarker writes the marker atomically: a temp file in the same +// directory, then a rename. A torn write cannot then leave a partial marker that +// parses, which is what makes "the marker is missing" and "the marker is ours" +// the only two answers a crash can produce. +func writeHolderMarker(dir string, m txnMarker) error { + data, err := json.Marshal(m) + if err != nil { + return err + } + f, err := holderFS.createTemp(dir, holderMarkerFile+"-*") + if err != nil { + return err + } + tmp := f.Name() + if err := writeAndSync(f, data); err != nil { + _ = holderFS.removeAll(tmp) + return err + } + if err := fsutil.RenameWithRetry(tmp, filepath.Join(dir, holderMarkerFile), holderFS.rename); err != nil { + _ = holderFS.removeAll(tmp) + return err + } + return nil +} + +// writeAndSync fills the marker's temp file and gets it to disk before the +// rename that publishes it. Without the Sync the rename can be visible after a +// crash while the bytes behind it are not. +func writeAndSync(f *os.File, data []byte) error { + if _, err := f.Write(data); err != nil { + f.Close() + return err + } + if err := f.Sync(); err != nil { + f.Close() + return err + } + return f.Close() +} + +// readHolderMarker reads a holder's marker. A missing one is errMarkerMissing; +// anything else is returned as it came back, so a caller can tell a directory it +// cannot claim from one it cannot read. +func readHolderMarker(dir string) (txnMarker, error) { + data, err := holderFS.readFile(filepath.Join(dir, holderMarkerFile)) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return txnMarker{}, fmt.Errorf("%w in %s", errMarkerMissing, dir) + } + return txnMarker{}, err + } + var m txnMarker + if err := json.Unmarshal(data, &m); err != nil { + return txnMarker{}, fmt.Errorf("unreadable transaction marker in %s: %w", dir, err) + } + return m, nil +} + +// writeCommitFlag records that the publish this holder's copy was set aside for +// actually landed. O_EXCL because a flag that is already there was written by a +// different transaction, and overwriting it would move the evidence. +func writeCommitFlag(dir string) error { + f, err := holderFS.create(filepath.Join(dir, committedFile), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + return f.Close() +} + +// holderCandidate is one holder recovery may act on, with the sequence its name +// and its marker agree on. +type holderCandidate struct { + path string + seq int64 +} + +// ownedHoldersBeside splits the entries beside destDir into the holders this +// code can prove it wrote for THIS destination and the ones it cannot. Owned +// means the exact name grammar and a marker whose kind, destination, and +// sequence all agree with it; unowned means a name this code would have written +// with no such marker behind it, which is retained and reported, never moved. +// An entry that only collides with the prefix is neither: it was never this +// install's, so it is not even reported on its account. +func ownedHoldersBeside(destDir string) (owned []holderCandidate, unowned []string, unreadable string) { + // ReadDir and a prefix rather than filepath.Glob: destDir is a real path, + // and a '[' anywhere in it opens a character class to Glob, which then + // matches nothing and strands the install this exists to put back. + parent := filepath.Dir(destDir) + base := filepath.Base(destDir) + entries, err := holderFS.readDir(parent) + if err != nil { + return nil, nil, "" + } + prefix := base + holderSuffix + for _, entry := range entries { + name := entry.Name() + if !entry.IsDir() || !strings.HasPrefix(name, prefix) { + continue + } + path := filepath.Join(parent, name) + seq, ok := holderStamp(destDir, name) + if !ok { + continue + } + m, err := readHolderMarker(path) + if err != nil { + if !errors.Is(err, errMarkerMissing) { + // A marker this pass could not read is not a marker proving the + // holder is someone else's. Walking past it would let recovery + // install an older copy and let the next pass read that copy as + // proof this one was superseded, which is the provenance loss + // the no-fallback rule exists to prevent. + return nil, nil, path + } + unowned = append(unowned, path) + continue + } + // Everything read off disk goes back through the check the write path + // applied, and the whole marker has to agree: a marker naming another + // destination or another sequence is evidence this directory belongs to + // a transaction that is not the one being recovered. + if m.Kind != holderMarkerKind || m.Seq != seq || m.Dest != base || !isInstallDestName(m.Dest) { + continue + } + owned = append(owned, holderCandidate{path: path, seq: seq}) + } + return owned, unowned, "" +} + +// parkKeptHolder renames a holder under the Kept prefix, which is how a copy +// recovery will not restore and cannot prove superseded leaves the scan without +// leaving the disk. A plain rename: the Kept name is a distinct sequence nothing +// else claims, and a park that would have to clear something first is a park +// onto a copy that is not ours to remove. +func parkKeptHolder(holder string) error { + kept, ok := keptHolderPath(holder) + if !ok { + return fmt.Errorf("%q is not a holder name", filepath.Base(holder)) + } + return fsutil.RenameWithRetry(holder, kept, holderFS.rename) +} + +// keptHolderPath is the Kept name a holder parks under: the same sequence, the +// other prefix. Recovery names it in its report before the rename as well as +// after, so the operator is told the same path whether the park lands or not. +func keptHolderPath(holder string) (string, bool) { + dir, name := filepath.Split(holder) + cut := strings.LastIndex(name, holderSuffix) + if cut < 0 { + return "", false + } + destDir := filepath.Join(dir, name[:cut]) + if _, ok := holderStamp(destDir, holder); !ok { + return "", false + } + return filepath.Join(dir, name[:cut]+keptSuffix+name[cut+len(holderSuffix):]), true +} + +// keptHolderStamp splits a Kept backup name into the install it was set aside +// for and its sequence. The install's own name is the leading part, so the +// grammar alone does NOT bound the name to one path component: "../x.kept-N-seq" +// parses as destination "../x". Callers check both halves are base names before +// joining either of them to anything. +func keptHolderStamp(name string) (base string, seq int64, ok bool) { + cut := strings.LastIndex(name, keptSuffix) + if cut <= 0 { + return "", 0, false + } + base = name[:cut] + seq, ok = holderStamp(base, name) + if !ok { + return "", 0, false + } + return base, seq, true +} + +// KeptBackup is one copy recovery moved under the Kept prefix rather than +// deleted, as an operator sees it. Recovery never enumerates that prefix and +// nothing reclaims one on its own, so this listing is the only way a retained +// copy is found again. Owned false is a directory carrying the Kept name that +// nothing on disk attributes, listed beside the real backups so recovery's +// residue is visible rather than silent until a disk fills. +type KeptBackup struct { + Path string + Dest string + Seq int64 + Bytes int64 + Owned bool +} + +// ListKeptBackups reports every Kept backup under an install root. The +// destination is reported only when the marker inside the copy agrees with the +// name it sits in: a name is what anything that can write in this directory +// says, and a copy attributed on its name alone is one an operator would remove +// believing it belonged to an install it never came from. +func ListKeptBackups(destRoot string) ([]KeptBackup, error) { + entries, err := holderFS.readDir(destRoot) + if err != nil { + return nil, err + } + var out []KeptBackup + for _, entry := range entries { + if !entry.IsDir() { + continue + } + base, seq, ok := keptHolderStamp(entry.Name()) + if !ok { + continue + } + path := filepath.Join(destRoot, entry.Name()) + backup := KeptBackup{Path: path, Seq: seq, Bytes: dirBytes(path)} + if ownedKeptHolder(path, base, seq) == nil { + backup.Dest = base + backup.Owned = true + } + out = append(out, backup) + } + slices.SortFunc(out, func(a, b KeptBackup) int { return cmp.Compare(a.Path, b.Path) }) + return out, nil +} + +// ownedKeptHolder is the whole ownership proof for a Kept backup: a marker this +// site wrote, for this destination, carrying the sequence its name carries. It +// is the same agreement ownedHoldersBeside requires before recovery touches a +// holder, applied here so an operator's delete is licensed by no less. +func ownedKeptHolder(path, base string, seq int64) error { + m, err := readHolderMarker(path) + if err != nil { + return err + } + if m.Kind != holderMarkerKind || m.Seq != seq || m.Dest != base || !isInstallDestName(m.Dest) { + return fmt.Errorf("the transaction marker in %s does not match its name", path) + } + return nil +} + +// RemoveKeptBackup deletes the one Kept backup an operator named. It is not an +// rm: the name has to be a single component under destRoot, pass the Kept +// grammar, and carry a marker whose kind, destination, and sequence all agree, +// and the destination's Install lock has to be free. Anything else is refused +// and left for the operator to remove by hand, because a Kept backup here is +// often the only offline copy of an engine or a model. +func RemoveKeptBackup(destRoot, name string) error { + if !isInstallDestName(name) { + // Ahead of every filesystem call: the Kept grammar accepts a leading + // install name with a separator in it, so without this the join reaches + // outside destRoot. + return fmt.Errorf("dictation download: %q is not the name of an entry in %s", name, destRoot) + } + base, seq, ok := keptHolderStamp(name) + if !ok || !isInstallDestName(base) { + return fmt.Errorf("dictation download: %s is not a kept backup; recovery may still act on it, so remove it by hand", name) + } + path := filepath.Join(destRoot, name) + if err := ownedKeptHolder(path, base, seq); err != nil { + return fmt.Errorf("dictation download: nothing on disk attributes %s to an install (%w); remove it by hand", name, err) + } + txn, held, err := tryLockDestination(destRoot, base) + if err != nil { + return err + } + if held { + return fmt.Errorf("dictation download: %s is being installed by another process; try again once it finishes", base) + } + defer txn.release() + // Re-read under the lock. The check above ran with nothing excluding a live + // promotion, and a directory that stopped being this install's in between is + // one that must not be deleted on the strength of the earlier reading. + if err := ownedKeptHolder(path, base, seq); err != nil { + return fmt.Errorf("dictation download: %s is no longer attributable to %s (%w); leaving it in place", name, base, err) + } + return holderFS.removeAll(path) +} + +// dirBytes sums the regular files under path. Unreadable entries are skipped +// rather than failing the listing: a size an operator cannot see is a worse +// answer than a size that is short, and the entry itself is still reported. +func dirBytes(path string) int64 { + var total int64 + _ = filepath.WalkDir(path, func(_ string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return nil + } + total += info.Size() + return nil + }) + return total +} + +// nextHolderSeq is the number a new holder should claim: one past the highest +// already set aside for this install, counting BOTH the scanned prefix and the +// Kept one. Recovery parks a holder by renaming it under .kept-, and a sequence +// that stopped counting there would reissue a number a parked copy already +// carries. Holders for a different install are a separate sequence and are never +// compared against this one, and a name the grammar rejects raises nothing. +func nextHolderSeq(destDir string) (int64, error) { + entries, err := holderFS.readDir(filepath.Dir(destDir)) + if err != nil { + return 0, err + } + var high int64 + for _, entry := range entries { + if !entry.IsDir() { + continue + } + if stamp, ok := holderStamp(destDir, entry.Name()); ok && stamp > high { + high = stamp + } + } + if high == math.MaxInt64 { + // The addition below would wrap negative, and %020d of a negative + // renders a '-' the parser reads as empty digits, so the holder would + // drop out of the ordering without saying so. Refuse instead. + return 0, fmt.Errorf("a previous install of %s is named at the maximum sequence; remove it before installing again", filepath.Base(destDir)) + } + return high + 1, nil +} + +// createSequencedHolder claims the first free holder name from n upward. Nothing +// locks this directory, so exclusive creation is what arbitrates: two promotions +// racing cannot both win a name, and the loser walks up above the winner. +func createSequencedHolder(destDir string, n int64) (string, error) { + if n < 1 { + return "", fmt.Errorf("refusing to allocate holder sequence %d", n) + } + for i := 0; i < holderSeqAttempts; i++ { + path := fmt.Sprintf("%s%s%020d%s", destDir, holderSuffix, n, holderSeqSuffix) + // The writer and the reader agree on the format or nothing is written. + if stamp, ok := holderStamp(destDir, path); !ok || stamp != n { + return "", fmt.Errorf("holder name %q does not read back as sequence %d", filepath.Base(path), n) + } + err := holderFS.mkdir(path, 0o700) + if err == nil { + return path, nil + } + if !errors.Is(err, fs.ErrExist) { + return "", err + } + if n == math.MaxInt64 { + return "", fmt.Errorf("holder sequence exhausted for %s", filepath.Base(destDir)) + } + n++ + } + return "", fmt.Errorf("could not claim a holder name for %s after %d attempts", filepath.Base(destDir), holderSeqAttempts) +} + +// holderState is one owned holder after classification. Recovery decides what +// every candidate IS before it moves anything: a ruling made while the disk is +// half-read is a ruling made on a state that no longer exists by the time it is +// acted on, which is how a copy that was the last usable one gets deleted. +type holderState struct { + holderCandidate + // empty means the marker is there and the set-aside content is not, so the + // holder provably holds no copy of any install. + empty bool + // committed means the publish this copy was set aside for actually landed. + // It is the only evidence that licenses removing a copy. + committed bool + // usable is the CALLER's predicate applied to the copy itself. Recovery has + // no opinion about what an install looks like; the consumer does. + usable bool +} + +// classifyHolder answers what one owned holder is, or reports that it cannot be +// read. Unusable and unreadable are deliberately different answers: the first is +// durable and the copy is skipped and kept, the second is a filesystem fault, +// and turning a fault into a classification is how a transient error becomes a +// permanent ruling about a copy nobody can get back. +func classifyHolder(c holderCandidate, published func(string) bool) (holderState, error) { + st := holderState{holderCandidate: c} + install := filepath.Join(c.path, "install") + // ReadDir rather than Stat: a copy whose contents cannot be listed is one + // whose usability cannot be decided either, and the predicate below would + // answer "unusable" for it without ever having seen it. + if _, err := holderFS.readDir(install); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return holderState{}, err + } + st.empty = true + return st, nil + } + if _, err := holderFS.stat(filepath.Join(c.path, committedFile)); err == nil { + st.committed = true + } else if !errors.Is(err, fs.ErrNotExist) { + return holderState{}, err + } + st.usable = published(install) + return st, nil +} + +// retainHolder gets one copy recovery will not restore out of the scan without +// getting it off the disk. An owned holder that provably holds nothing is +// removed; everything else moves under the Kept prefix, where the next pass does +// not look and the operator can still find it by name. A park that fails leaves +// the copy exactly where it is: the next pass reaches the same branch and tries +// again, and nothing here ever turns a retain into a delete. +func retainHolder(st holderState, report func(string)) { + if st.empty { + if err := holderFS.removeAll(st.path); err != nil { + report(fmt.Sprintf("Could not remove the empty holder %s (%v)", st.path, err)) + } + return + } + kept, ok := keptHolderPath(st.path) + if !ok { + report(fmt.Sprintf("Keeping a copy of this install in %s", st.path)) + return + } + if err := parkKeptHolder(st.path); err != nil { + report(fmt.Sprintf("Keeping a copy of this install in %s; it could not be moved to %s (%v)", st.path, kept, err)) + return + } + report(fmt.Sprintf("Keeping a copy of this install in %s", kept)) +} + +// setAsideUnusableDest moves a destination that exists but holds no usable +// install into a fresh sequenced holder, so a usable copy can be restored over +// it. Fresh rather than the candidate's own holder: nesting one set-aside copy +// inside another is how a copy stops being findable by its own name. Only ever +// called once a candidate has been selected, so a destination nothing can +// replace is never taken apart. +func setAsideUnusableDest(destDir string) (string, error) { + seq, err := nextHolderSeq(destDir) + if err != nil { + return "", err + } + holder, err := createSequencedHolder(destDir, seq) + if err != nil { + return "", err + } + claimed, ok := holderStamp(destDir, holder) + if !ok { + _ = holderFS.removeAll(holder) + return "", fmt.Errorf("holder name %q is not one recovery can order", filepath.Base(holder)) + } + // Same order the write path uses: the marker goes in before the first + // destructive rename, so a stop in this window leaves a holder recovery can + // prove is its own rather than one nothing on disk claims. + if err := writeHolderMarker(holder, txnMarker{Kind: holderMarkerKind, Dest: filepath.Base(destDir), Seq: claimed}); err != nil { + _ = holderFS.removeAll(holder) + return "", err + } + if err := fsutil.RenameWithRetry(destDir, filepath.Join(holder, "install"), holderFS.rename); err != nil { + // Owned and holding nothing, which is the one shape that costs nothing + // to remove. Leaving it would accumulate scratch on every failed pass. + _ = holderFS.removeAll(holder) + return "", err + } + return holder, nil +} + +// restoreInterruptedPromotion reconciles one destination with every copy of it +// beside it. A process stopped between promoteStagedDir's two renames leaves the +// only usable install in a .previous-* holder nothing else looks at, and an +// offline caller has no download to fall back on; a failed cleanup leaves a copy +// that IS superseded; and recovery itself can leave a destination holding a tree +// the caller cannot use. So every candidate is classified before anything moves, +// and the ruling is a function of on-disk state alone: a second pass with no +// memory of the first reaches the same answer. +// +// published reports whether a directory holds an install this caller can +// actually use. It is the whole basis of every decision here, because "there is +// something at destDir" and "a promotion published there" are different claims, +// and only the second makes a copy beside it superseded. +func restoreInterruptedPromotion(txn *destTxn, destDir string, published func(string) bool, report func(string)) { + if report == nil { + report = func(string) {} + } + // Recovery moves and deletes whole installs. Without destDir's Install lock + // it can restore a holder a promotion in another process is parked on and + // remove it, leaving that promotion's rollback nothing to put back. + if !txn.holds(destDir) { + report(fmt.Sprintf("Skipping recovery of %s: no install lock is held for it", filepath.Base(destDir))) + return + } + // Every ruling below rests on the predicate. With none there is no evidence + // for any of them, and acting anyway would delete or publish a copy on the + // strength of a directory merely existing. + if published == nil { + report(fmt.Sprintf("Skipping recovery of %s: no usability check was supplied", filepath.Base(destDir))) + return + } + // Only holders this code can prove it wrote for THIS destination. A sibling + // that merely collides with the prefix is not a copy of this install and is + // never restored from, removed, or reported on its account. + owned, unowned, unreadable := ownedHoldersBeside(destDir) + if unreadable != "" { + report(fmt.Sprintf("dictation: leaving every retained copy for %s in place until %s can be read", filepath.Base(destDir), unreadable)) + return + } + for _, path := range unowned { + report(fmt.Sprintf("Keeping %s: nothing in it identifies it as a copy of %s", path, filepath.Base(destDir))) + } + states := make([]holderState, 0, len(owned)) + for _, candidate := range owned { + st, err := classifyHolder(candidate, published) + if err != nil { + // One unreadable candidate stops the whole destination. Ruling on + // the rest would mean choosing between copies while one of them is + // unread, and the copy nobody could read may be the newest. + report(fmt.Sprintf("Stopping recovery of %s: %s could not be read (%v)", filepath.Base(destDir), candidate.path, err)) + return + } + states = append(states, st) + } + // Newest first: the sequence in the name, which the marker agrees with, is + // the only record of which copy was live last. No decision here reads a + // clock, so a backward clock cannot reorder them. + slices.SortStableFunc(states, func(a, b holderState) int { + return cmp.Compare(b.seq, a.seq) + }) + + destPresent := false + if _, err := holderFS.lstat(destDir); err == nil { + destPresent = true + } else if !errors.Is(err, fs.ErrNotExist) { + report(fmt.Sprintf("Stopping recovery of %s: it could not be read (%v)", filepath.Base(destDir), err)) + return + } + + if destPresent && published(destDir) { + // The destination holds a real install, so every copy beside it was set + // aside by some earlier transaction. Only the ones carrying the commit + // flag are provably superseded by it; the rest may still be the last + // usable copy of something and are kept. + for _, st := range states { + if st.committed && !st.empty { + if err := holderFS.removeAll(st.path); err != nil { + report(fmt.Sprintf("Could not remove the superseded copy in %s (%v)", st.path, err)) + } + continue + } + retainHolder(st, report) + } + return + } + + // The destination is absent or holds nothing this caller can use, so a + // usable copy beside it is what should be live. Uncommitted first: it was + // set aside by the transaction that never finished, which makes it the most + // recent state anything can prove. + selected := -1 + for i, st := range states { + if !st.empty && !st.committed && st.usable { + selected = i + break + } + } + if selected < 0 { + for i, st := range states { + if !st.empty && st.committed && st.usable { + selected = i + break + } + } + } + if selected < 0 { + if destPresent { + // Nothing beside it can replace it, so taking it apart would leave + // the caller with no destination at all instead of an unusable one. + report(fmt.Sprintf("%s holds no usable install and no copy beside it can replace it; it is left as it is", destDir)) + } + for _, st := range states { + retainHolder(st, report) + } + return + } + + // Classify, select, THEN set aside. A husk moved before a candidate is + // chosen is a destination taken apart for a restore that may never happen, + // and a failed restore would strand it under a prefix recovery never reads. + husk := "" + if destPresent { + var err error + husk, err = setAsideUnusableDest(destDir) + if err != nil { + report(fmt.Sprintf("Leaving %s as it is: it could not be set aside (%v)", destDir, err)) + return + } + } + winner := states[selected] + if err := fsutil.RenameWithRetry(filepath.Join(winner.path, "install"), destDir, holderFS.rename); err != nil { + // No fallback to an older copy: installing one would put a tree at the + // destination that the next pass reads as evidence the newer copy was + // superseded, which is exactly how provenance is lost. + if husk != "" { + if backErr := fsutil.RenameWithRetry(filepath.Join(husk, "install"), destDir, holderFS.rename); backErr != nil { + report(fmt.Sprintf("%s could not be put back and is kept in %s (%v)", destDir, husk, backErr)) + } else if rmErr := holderFS.removeAll(husk); rmErr != nil { + report(fmt.Sprintf("Could not remove the empty holder %s (%v)", husk, rmErr)) + } + } + report(fmt.Sprintf("Keeping the copy of %s in %s: it could not be moved back into place (%v)", filepath.Base(destDir), winner.path, err)) + return + } + // The winner's holder is owned and now holds nothing. + if err := holderFS.removeAll(winner.path); err != nil { + report(fmt.Sprintf("Could not remove the empty holder %s (%v)", winner.path, err)) + } + if husk != "" { + retainHolder(holderState{holderCandidate: holderCandidate{path: husk}}, report) + } + for i, st := range states { + if i != selected { + retainHolder(st, report) + } + } +} + +// promoteStagedDir moves stageDir into place at destDir. os.Rename refuses to +// overwrite a non-empty directory, so any previous install has to move out of +// the way first; it is set aside rather than deleted, and put back if the +// promotion fails, so destDir is never left holding no install at all. If the +// restore fails too, the set-aside copy is kept and named in the error. +func promoteStagedDir(txn *destTxn, stageDir, destDir, label string, report func(string)) error { + // Normalized once so every later report is one call rather than a nil check + // a new one can forget. A caller with nowhere to report to still gets the + // same behavior; only the message goes nowhere. + if report == nil { + report = func(string) {} + } + // Same reason recovery refuses: the window between the two renames has the + // only copy of the install in a holder, and nothing else keeps a concurrent + // recovery out of it. + if !txn.holds(destDir) { + { + report(fmt.Sprintf("Refusing to install %s: no install lock is held for %s", label, filepath.Base(destDir))) + } + return fmt.Errorf("promoting staged %s: no install lock is held for %s", label, destDir) + } + holder := "" + cleanupHolder := true + defer func() { + if holder != "" && cleanupHolder { + // A failure here strands a whole copy of the previous install under + // a name recovery no longer enumerates, so say so rather than + // leaking it silently: the operator's reclaim command is the only + // thing that removes it afterwards. + if err := holderFS.removeAll(holder); err != nil { + report(fmt.Sprintf("dictation: could not remove the superseded %s copy in %s: %v", label, holder, err)) + } + } + }() + + restore := func() error { return nil } + if _, err := holderFS.lstat(destDir); err == nil { + var seq int64 + seq, err = nextHolderSeq(destDir) + if err != nil { + return fmt.Errorf("setting aside previous %s install: %w", label, err) + } + holder, err = createSequencedHolder(destDir, seq) + if err != nil { + return fmt.Errorf("setting aside previous %s install: %w", label, err) + } + // createSequencedHolder walks up from seq when a name is taken, so the + // marker has to carry the number actually claimed. A marker that + // disagrees with the name it sits in is not ownership, and recovery + // would leave the copy in place forever. + claimed, ok := holderStamp(destDir, holder) + if !ok { + return fmt.Errorf("setting aside previous %s install: holder name %q is not one recovery can order", label, filepath.Base(holder)) + } + // The marker goes in BEFORE the first destructive rename. A holder that + // takes the only copy of an install before it can be proven ours is one + // nothing on disk lets recovery claim. + if err := writeHolderMarker(holder, txnMarker{Kind: holderMarkerKind, Dest: filepath.Base(destDir), Seq: claimed}); err != nil { + return fmt.Errorf("setting aside previous %s install: %w", label, err) + } + previous := filepath.Join(holder, "install") + if err := fsutil.RenameWithRetry(destDir, previous, holderFS.rename); err != nil { + return fmt.Errorf("setting aside previous %s install: %w", label, err) + } + restore = func() error { return fsutil.RenameWithRetry(previous, destDir, holderFS.rename) } + } else if !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("checking previous %s install: %w", label, err) + } + + if err := fsutil.RenameWithRetry(stageDir, destDir, holderFS.rename); err != nil { + if restoreErr := restore(); restoreErr != nil { + cleanupHolder = false + return fmt.Errorf("promoting staged %s: %w (previous install left in %s: %v)", label, err, holder, restoreErr) + } + return fmt.Errorf("promoting staged %s: %w", label, err) + } + if holder != "" { + // The flag follows the publish and precedes the cleanup, so the only + // thing a crash in that window costs is a retained copy. Writing it any + // earlier would license deleting a copy that is still the only one. + if err := writeCommitFlag(holder); err != nil { + // The install itself landed. Deleting a whole previous install on + // the word of a step that just failed is the one outcome worse than + // leaving it on disk, so the copy stays and the report says where. + cleanupHolder = false + if report != nil { + report(fmt.Sprintf("Installed %s, but the previous install could not be marked superseded and is kept in %s (%v)", label, holder, err)) + } + } + } + return nil +} + // extractTarBz2 unpacks a bzip2-compressed tar into destDir, guarding against // path-traversal entries. It reports extraction progress by how much of the // (compressed) archive it has consumed — the same MB scale as the download, so diff --git a/internal/dictation/download_matrix_test.go b/internal/dictation/download_matrix_test.go new file mode 100644 index 000000000..327c29272 --- /dev/null +++ b/internal/dictation/download_matrix_test.go @@ -0,0 +1,1337 @@ +package dictation + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// The crash rows of the recovery plan's Acceptance Examples, as code. Each row +// drives the REAL promotion into the on-disk state a stop at one step leaves, +// then runs recovery twice and asserts both halves the table asserts: which +// install is live, and where every other copy of an install ended up. Nothing +// here plants a holder by hand, so the fixture is whatever the write path +// actually writes. +// +// A second pass is asserted because recovery keeps no memory: a copy retained on +// pass one must not be reclassified on pass two, and a copy deleted on pass one +// must not come back. + +// copyDisposition is where one copy of an install ends up. The five values are +// the only terminal states the table uses, so a row that means "retained" has to +// say which kind of retention, and a delete can never be written down as +// anything else. +type copyDisposition int + +const ( + // copyDeleted: gone from disk, under neither prefix. + copyDeleted copyDisposition = iota + // copyAtHolder: still under the name the promotion allocated for it. + copyAtHolder + // copyAtKept: moved under the Kept prefix, where the scan does not look. + copyAtKept + // copyInPlaceUnowned: still at its name and carrying no marker, so nothing + // on disk attributes it to this code. + copyInPlaceUnowned + // copyAtDest: restored, so the copy's content is the live install and its + // holder is gone. + copyAtDest +) + +// dictationCrashRow is one row of the table. Each of these rows leaves exactly +// one holder, and the assertion compares the WHOLE set of copies beside the +// destination against that one, so a row can never pass while a second copy +// nobody expected sits beside it. +type dictationCrashRow struct { + id string + // arrange runs the real promotion with faults injected to stop it at one + // step. The faults are unwound before recovery runs, so recovery sees a + // crashed install and not a failing filesystem. + arrange func(t *testing.T, txn *destTxn, stage, dest string) + wantLive string + wantCopy copyDisposition + // report1 and report2 assert the copy is named in the recovery report on + // that pass. A copy recovery retains but never names is one an operator + // cannot find, which the table counts as a failure of the row. + report1 bool + report2 bool +} + +// arrangeInterruptedPromotion stops the promotion between its two renames: the +// previous install is aside in the holder, the publish failed, and the restore +// failed too, so the destination is absent and the holder has the only copy of +// it. D2 and D5 differ in how the writer got here and are identical on disk, +// which is why they share one arrangement. +func arrangeInterruptedPromotion(t *testing.T, txn *destTxn, stage, dest string) { + t.Helper() + injectFault(t, "rename", func(args ...string) bool { + return filepath.Base(args[1]) == filepath.Base(dest) + }, errors.New("injected publish and restore failure")) + if err := promoteStagedDir(txn, stage, dest, "engine", nil); err == nil { + t.Fatal("a promotion whose publish and restore both fail must report an error") + } + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Fatalf("the interrupted promotion should leave %s absent, got %v", dest, err) + } +} + +func dictationCrashRows() []dictationCrashRow { + return []dictationCrashRow{ + { + // A stop between the holder and its marker leaves a directory + // nothing on disk attributes. Deleting it would need proof this code + // wrote it, and the marker is that proof, so it is retained and + // named rather than reaped on the strength of its name. + id: "D1", + arrange: func(t *testing.T, txn *destTxn, stage, dest string) { + injectFault(t, "createTemp", func(args ...string) bool { + return args[1] == holderMarkerFile+"-*" + }, errors.New("injected marker temp failure")) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := promoteStagedDir(txn, stage, dest, "engine", nil); err == nil { + t.Fatal("a promotion whose marker cannot be written must fail") + } + }, + wantLive: "old", + wantCopy: copyInPlaceUnowned, + report1: true, + report2: true, + }, + { + // The marker landed and the set-aside did not, so the holder is + // owned and holds no copy of any install. Nothing can be lost by + // removing it, and leaving it is what accumulates scratch forever. + id: "D1b", + arrange: func(t *testing.T, txn *destTxn, stage, dest string) { + injected := errors.New("injected set-aside failure") + injectFault(t, "rename", func(args ...string) bool { + return filepath.Base(args[1]) == "install" + }, injected) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := promoteStagedDir(txn, stage, dest, "engine", nil); !errors.Is(err, injected) { + t.Fatalf("promote = %v, want the injected set-aside failure", err) + } + }, + wantLive: "old", + wantCopy: copyDeleted, + }, + { + // The previous install is aside and the destination is absent, so + // the holder has the only copy of it and it has to come back. An + // offline caller has no download to fall back on. + id: "D2", + arrange: arrangeInterruptedPromotion, + wantLive: "old", + wantCopy: copyAtDest, + }, + { + // The publish landed and the flag did not. Without the flag there is + // no evidence the copy in the holder was superseded, so it may not + // be deleted; parking it is what keeps it out of the next pass's way + // without dropping the last copy of an install. + id: "D3", + arrange: func(t *testing.T, txn *destTxn, stage, dest string) { + injectFault(t, "create", func(args ...string) bool { + return filepath.Base(args[0]) == committedFile + }, errors.New("injected commit flag failure")) + if err := promoteStagedDir(txn, stage, dest, "engine", nil); err != nil { + t.Fatalf("a published install must be reported as success: %v", err) + } + }, + wantLive: "new", + wantCopy: copyAtKept, + report1: true, + }, + { + // The publish failed and the restore put the install back, so the + // holder holds no copy of anything: owned and empty. + id: "D4", + arrange: func(t *testing.T, txn *destTxn, stage, dest string) { + injected := errors.New("injected publish failure") + injectFault(t, "rename", func(args ...string) bool { + return filepath.Base(args[0]) == filepath.Base(stage) + }, injected) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := promoteStagedDir(txn, stage, dest, "engine", nil); !errors.Is(err, injected) { + t.Fatalf("promote = %v, want the injected publish failure", err) + } + }, + wantLive: "old", + wantCopy: copyDeleted, + }, + { + // The writer's own retain branch: publish failed, restore failed, + // the copy was kept and named in the error. On disk this is D2, and + // recovery must not tell them apart, because nothing on disk does. + id: "D5", + arrange: arrangeInterruptedPromotion, + wantLive: "old", + wantCopy: copyAtDest, + }, + { + // The flag is there, so the copy in the holder is provably + // superseded by the install now live at the destination. This is the + // one and only shape that licenses deleting a copy of an install. + id: "D6", + arrange: func(t *testing.T, txn *destTxn, stage, dest string) { + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + if err := promoteStagedDir(txn, stage, dest, "engine", nil); err != nil { + t.Fatalf("a cleanup failure must not fail the install: %v", err) + } + }, + wantLive: "new", + wantCopy: copyDeleted, + }, + } +} + +func TestDictationCrashMatrix(t *testing.T) { + for _, row := range dictationCrashRows() { + t.Run(row.id, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + stagedTree(t, dest, "old") + // One handle for the whole row: the lock is per open file + // description, so a second acquire in this process would contend + // with the first rather than nest. + txn := lockFor(t, dest) + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + // The faults belong to the crash, not to recovery: restoring the + // seam here is what makes the next lines a recovery over a real + // filesystem rather than over a failing one. + func() { + saved := holderFS + defer func() { holderFS = saved }() + row.arrange(t, txn, stage, dest) + }() + holder := soleHolder(t, dest) + + for pass, wantReport := range []bool{row.report1, row.report2} { + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, func(m string) { + reported = append(reported, m) + }) + assertDictationTerminalState(t, dest, holder, row.wantCopy, row.wantLive, pass+1) + if wantReport { + assertCopyReported(t, reported, holderSeqDigitsOf(t, dest, holder), pass+1) + } + } + }) + } +} + +// assertDictationTerminalState asserts both halves of a row: the live install, +// and the full set of copies still beside the destination under either prefix. +// The set is compared whole rather than one path at a time, so an extra copy +// nobody expected fails the row instead of going unnoticed. +func assertDictationTerminalState(t *testing.T, dest, holder string, want copyDisposition, wantLive string, pass int) { + t.Helper() + got, err := os.ReadFile(filepath.Join(dest, "engine")) + switch { + case wantLive == "": + if err == nil { + t.Errorf("pass %d: the destination should be absent, it holds %q", pass, got) + } + case err != nil || string(got) != wantLive: + t.Errorf("pass %d: dest engine = %q (err %v), want %q", pass, got, err, wantLive) + } + + var wantPaths []string + switch want { + case copyAtHolder, copyInPlaceUnowned: + wantPaths = []string{holder} + case copyAtKept: + wantPaths = []string{keptHolderName(t, dest, holder)} + } + base := filepath.Base(dest) + assertCopySet(t, filepath.Dir(dest), []string{base + holderSuffix, base + keptSuffix}, wantPaths, pass) + + switch want { + case copyInPlaceUnowned: + // Retained is not the whole claim: the reason it is retained is that + // nothing on disk attributes it, and a row that stopped checking that + // would pass over a copy recovery could have proved was its own. + if _, err := readHolderMarker(holder); !errors.Is(err, errMarkerMissing) { + t.Errorf("pass %d: %s should carry no marker, readHolderMarker = %v", pass, holder, err) + } + case copyAtDest: + if _, err := os.Stat(dest); err != nil { + t.Errorf("pass %d: the copy should have been restored to the destination: %v", pass, err) + } + } +} + +// keptHolderName is the name a parked copy takes, derived the way parkKeptHolder +// derives it: the same sequence under the Kept prefix. +func keptHolderName(t *testing.T, dest, holder string) string { + t.Helper() + name := filepath.Base(holder) + cut := strings.LastIndex(name, holderSuffix) + if cut < 0 { + t.Fatalf("holder name %q carries no holder suffix", name) + } + return filepath.Join(filepath.Dir(holder), name[:cut]+keptSuffix+name[cut+len(holderSuffix):]) +} + +// holderSeqDigitsOf is the sequence a copy carries in its name. It survives a +// park, so a report assertion keyed on it holds whether the copy is still under +// the holder prefix or has moved under the Kept one. +func holderSeqDigitsOf(t *testing.T, dest, holder string) string { + t.Helper() + seq, ok := holderStamp(dest, holder) + if !ok { + t.Fatalf("holder name %q carries no sequence", filepath.Base(holder)) + } + return fmt.Sprintf("%020d", seq) +} + +// A copy recovery keeps and never names is one an operator cannot find, so the +// report is half of every retain row rather than a nicety. +func assertCopyReported(t *testing.T, reported []string, seq string, pass int) { + t.Helper() + if !slices.ContainsFunc(reported, func(m string) bool { return strings.Contains(m, seq) }) { + t.Errorf("pass %d: the retained copy (sequence %s) should be named in the report, got %v", pass, seq, reported) + } +} + +// assertCopySet compares every directory under the given prefixes against the +// paths the row expects, resolving both sides through EvalSymlinks because the +// temp root is a symlink on macOS and a path built from t.TempDir would then +// never equal one read back out of the directory. +func assertCopySet(t *testing.T, parent string, prefixes, want []string, pass int) { + t.Helper() + entries, err := os.ReadDir(parent) + if err != nil { + t.Fatal(err) + } + var got []string + for _, entry := range entries { + if !entry.IsDir() { + continue + } + for _, prefix := range prefixes { + if strings.HasPrefix(entry.Name(), prefix) { + got = append(got, resolvedPath(t, filepath.Join(parent, entry.Name()))) + break + } + } + } + wantResolved := make([]string, 0, len(want)) + for _, path := range want { + wantResolved = append(wantResolved, resolvedPath(t, path)) + } + slices.Sort(got) + slices.Sort(wantResolved) + if !slices.Equal(got, wantResolved) { + t.Errorf("pass %d: copies on disk = %v, want %v", pass, got, wantResolved) + } +} + +// resolvedPath resolves symlinks so the two sides of a path comparison are the +// same path. A path that does not exist cannot be resolved, and its cleaned form +// is what the comparison then reports. +func resolvedPath(t *testing.T, path string) string { + t.Helper() + if resolved, err := filepath.EvalSymlinks(path); err == nil { + return resolved + } + return filepath.Clean(path) +} + +// Ownership is a name plus a marker that agrees with it, and neither half alone. +// A directory that only carries the generated name is a sibling someone renamed +// or an unrelated install, and recovery restoring from one, or reaping one, is +// the whole class of defect the marker exists to close. Each case here is a +// single check dropped: the marker, its kind, its sequence, its destination. +func TestOwnershipCannotBeForgedBySiblingNames(t *testing.T) { + const forgedSeq = 7 + for _, tc := range []struct { + name string + marker *txnMarker + }{ + {name: "no marker at all"}, + {name: "another site's kind", marker: &txnMarker{Kind: "bundle-extract", Dest: "engine-dir", Seq: forgedSeq}}, + {name: "sequence disagrees with the name", marker: &txnMarker{Kind: holderMarkerKind, Dest: "engine-dir", Seq: forgedSeq + 1}}, + {name: "destination is another install", marker: &txnMarker{Kind: holderMarkerKind, Dest: "model-dir", Seq: forgedSeq}}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + // Built outside and renamed in, so nothing about it was ever written + // by this code: the name is the only thing it shares with a holder, + // and it is shaped like a restorable one. + unrelated := filepath.Join(t.TempDir(), "unrelated") + if err := os.MkdirAll(filepath.Join(unrelated, "install"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(unrelated, "install", "engine"), []byte("forged"), 0o644); err != nil { + t.Fatal(err) + } + forged := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, forgedSeq, holderSeqSuffix) + if err := os.Rename(unrelated, forged); err != nil { + t.Fatal(err) + } + if tc.marker != nil { + if err := writeHolderMarker(forged, *tc.marker); err != nil { + t.Fatal(err) + } + } + txn := lockFor(t, dest) + + for pass := 1; pass <= 2; pass++ { + restoreInterruptedPromotion(txn, dest, testPublished, nil) + got, err := os.ReadFile(filepath.Join(forged, "install", "engine")) + if err != nil || string(got) != "forged" { + t.Fatalf("pass %d: the forged directory must be left exactly as it was, got %q (err %v)", pass, got, err) + } + // A restore, a park, or a reap all show up here: any of them + // either creates a destination or moves the directory. + assertNoOtherEntries(t, root, filepath.Base(forged), installLockDir) + } + }) + } +} + +// assertNoOtherEntries fails when anything but the named entries is in dir. It +// is how "nothing was owned" is asserted without naming every way ownership +// could have been acted on: a restore, a park, and a reap all change this set. +func assertNoOtherEntries(t *testing.T, dir string, allowed ...string) { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !slices.Contains(allowed, entry.Name()) { + t.Errorf("%s should hold only %v, found %s", dir, allowed, entry.Name()) + } + } +} + +// ---- the regression net: recovery's own steps, and a live promotion -------- + +// deleteWatch records every removeAll recovery asks for, and flags the ones no +// classification licenses: a holder that still holds a copy of an install with +// no commit flag beside it. It is asserted AT THE SEAM rather than off the disk +// because a fault-injected removeAll leaves the copy exactly where a removeAll +// that was never requested does, so a row reading the surviving files could not +// tell a recovery that did the right thing from one that tried to do the wrong +// thing and was stopped by the injected fault. +type deleteWatch struct { + mu sync.Mutex + all []string + denied []string +} + +// install swaps removeAll for a recorder, restoring the seam in t.Cleanup. The +// order matters: only a watch installed AFTER a fault sees the calls that fault +// is failing, and cleanups unwind in reverse, so the seam still ends the test as +// the real filesystem. +func (w *deleteWatch) install(t *testing.T) { + t.Helper() + real := holderFS + t.Cleanup(func() { holderFS = real }) + holderFS.removeAll = func(path string) error { + w.mu.Lock() + w.all = append(w.all, path) + if holdsUncommittedCopy(path) { + w.denied = append(w.denied, path) + } + w.mu.Unlock() + return real.removeAll(path) + } +} + +func (w *deleteWatch) counts() (all, denied []string) { + w.mu.Lock() + defer w.mu.Unlock() + return slices.Clone(w.all), slices.Clone(w.denied) +} + +// holdsUncommittedCopy reports whether path still holds a copy of an install +// that nothing proves was published over. Recovery may park one of these +// forever; it may never delete one, whatever step just failed. At this site that +// copy is the only offline copy of an engine or a model. +func holdsUncommittedCopy(path string) bool { + if _, err := os.Stat(filepath.Join(path, "install")); err != nil { + return false + } + _, err := os.Stat(filepath.Join(path, committedFile)) + return os.IsNotExist(err) +} + +// The watch is the whole regression net below, so it gets its own proof: a net +// that cannot see the delete it exists to catch would make every row under it +// pass for the wrong reason. The three shapes are the two deletes recovery is +// licensed to make and the one it is not. +func TestDeleteWatchSeesOnlyTheDeleteRecoveryMayNotMake(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + uncommitted := plantHolder(t, dest, 100, "v1", false) + committed := plantHolder(t, dest, 200, "v2", true) + scratch := plantScratchHolder(t, dest, 50) + + var watch deleteWatch + func() { + saved := holderFS + defer func() { holderFS = saved }() + watch.install(t) + for _, path := range []string{committed, scratch, uncommitted} { + if err := holderFS.removeAll(path); err != nil { + t.Fatal(err) + } + } + }() + + all, denied := watch.counts() + if len(all) != 3 { + t.Errorf("the watch should have seen all three deletes, got %v", all) + } + if !slices.Equal(denied, []string{uncommitted}) { + t.Errorf("only the uncommitted copy is a violation, got %v", denied) + } +} + +// plantScratchHolder plants an owned holder that holds no copy of any install: +// the state a crash between the marker and the set-aside leaves. It is the one +// shape besides a committed copy that recovery may delete. +func plantScratchHolder(t *testing.T, destDir string, seq int64) string { + t.Helper() + holder := holderNamed(destDir, seq) + if err := os.MkdirAll(holder, 0o755); err != nil { + t.Fatal(err) + } + if err := writeHolderMarker(holder, txnMarker{Kind: holderMarkerKind, Dest: filepath.Base(destDir), Seq: seq}); err != nil { + t.Fatal(err) + } + return holder +} + +func holderNamed(destDir string, seq int64) string { + return fmt.Sprintf("%s%s%020d%s", destDir, holderSuffix, seq, holderSeqSuffix) +} + +func keptNamed(destDir string, seq int64) string { + return fmt.Sprintf("%s%s%020d%s", destDir, keptSuffix, seq, holderSeqSuffix) +} + +// recoveryStep is one filesystem step of a recovery pass, with the fault that +// fails it. Each is matched where it is reached rather than by call ordinal, so +// a change in how many times recovery stats a holder cannot silently move which +// call the row is failing. +type recoveryStep struct { + name string + // inject installs the row's fault and returns whether that fault was ever + // actually called. The return is not bookkeeping: a row that injects a + // failure into a step the pass never reaches asserts only that a clean run + // works, which every other row already proves. + inject func(t *testing.T, newest string) (fired func() bool) +} + +// firedFault wraps a row's match so the row can report whether its fault was +// reached. A nil match means every call of the step, matching injectFault's own +// contract. +func firedFault(match func(args ...string) bool) (wrapped func(args ...string) bool, fired func() bool) { + var hit atomic.Bool + return func(args ...string) bool { + if match != nil && !match(args...) { + return false + } + hit.Store(true) + return true + }, func() bool { return hit.Load() } +} + +func dictationRecoverySteps() []recoveryStep { + failed := errors.New("injected recovery failure") + simple := func(step string) func(*testing.T, string) func() bool { + return func(t *testing.T, _ string) func() bool { + match, fired := firedFault(nil) + injectFault(t, step, match, failed) + return fired + } + } + return []recoveryStep{ + {name: "readDir", inject: simple("readDir")}, + {name: "stat", inject: simple("stat")}, + {name: "marker read", inject: simple("readFile")}, + { + name: "restore rename", + inject: func(t *testing.T, newest string) func() bool { + match, fired := firedFault(func(args ...string) bool { + return args[0] == filepath.Join(newest, "install") + }) + injectFault(t, "rename", match, failed) + return fired + }, + }, + {name: "removeAll", inject: simple("removeAll")}, + { + name: "park rename", + inject: func(t *testing.T, _ string) func() bool { + match, fired := firedFault(func(args ...string) bool { + return strings.Contains(filepath.Base(args[1]), keptSuffix) + }) + injectFault(t, "rename", match, failed) + return fired + }, + }, + } +} + +// Every recovery step, failed on pass one, on pass two, and on both. Two claims +// per case, and the first is the one the whole branch exists for: no failure of +// any step ever produces a delete of a copy nothing proves was superseded. The +// second is that a fault only DELAYS recovery: a later pass with the fault gone +// reaches the same terminal state a clean run reaches, so a step that failed +// once has not left the destination in a state recovery can no longer reason +// about. +func TestDictationRecoveryStepFailures(t *testing.T) { + for _, step := range dictationRecoverySteps() { + // A standalone "pass two" variant used to sit between these two. It ran + // pass one clean, which reaches the terminal state, so pass two had no + // candidate left and the fault it injected was never called: the row + // asserted a clean run and nothing else. The both-passes row is the one + // that exercises a fault on pass two, because pass one fails first and + // leaves work behind for it. + for _, when := range []struct { + name string + faulty [2]bool + }{ + {name: "pass one", faulty: [2]bool{true, false}}, + {name: "both passes", faulty: [2]bool{true, true}}, + } { + t.Run(step.name+"/"+when.name, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + // Two uncommitted usable copies and one owned holder that holds + // nothing, with the destination gone: the state that drives + // every step in the list through a real decision. One copy is + // restored, one is parked, and the empty holder is the only + // delete on the happy path, so a step that fails has something + // to corrupt. + older := plantHolder(t, dest, 100, "v1", false) + newest := plantHolder(t, dest, 200, "v2", false) + scratch := plantScratchHolder(t, dest, 50) + txn := lockFor(t, dest) + + var watch deleteWatch + pass := func(faulty bool) func() bool { + saved := holderFS + defer func() { holderFS = saved }() + fired := func() bool { return true } + if faulty { + fired = step.inject(t, newest) + } + // After the fault, so the watch sees the calls the fault is + // failing rather than only the ones it lets through. + watch.install(t) + restoreInterruptedPromotion(txn, dest, testPublished, nil) + return fired + } + firedOne := pass(when.faulty[0]) + if when.faulty[0] { + if !firedOne() { + // The pass never called the step this row fails, so the + // row ran a clean recovery under a fault that does not + // exist and every assertion below it is vacuous. + t.Fatalf("the injected %s failure was never called on pass one, so this row proves nothing", step.name) + } + if dictationStepTerminal(dest, older, newest, scratch) { + // Same defect from the other side: a fault that was + // called and changed nothing left the pass free to + // finish. + t.Errorf("the injected %s failure left the pass free to finish, so this row proves nothing", step.name) + } + } + firedTwo := pass(when.faulty[1]) + if when.faulty[1] && !firedTwo() { + t.Fatalf("the injected %s failure was never called on pass two, so this row proves nothing", step.name) + } + // The fault is gone, and recovery has to be able to finish from + // wherever the failed passes left the destination. + pass(false) + + all, denied := watch.counts() + if len(denied) > 0 { + t.Errorf("no step failure may produce a delete of an uncommitted copy, got %v", denied) + } + if len(all) == 0 { + t.Fatal("the watch saw no delete at all, so it proves nothing about this pass") + } + assertDictationStepTerminalState(t, dest, older, newest, scratch) + }) + } + } +} + +// dictationStepTerminal reports whether the fixture reached the state a clean +// run reaches. It is a predicate rather than an assertion because it is asked +// twice for opposite reasons: a faulted pass must NOT have got here, and the +// pass after the fault is gone must have. +func dictationStepTerminal(dest, older, newest, scratch string) bool { + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "v2" { + return false + } + kept, ok := keptHolderPath(older) + if !ok { + return false + } + if _, err := os.Stat(filepath.Join(kept, "install", "engine")); err != nil { + return false + } + for _, gone := range []string{newest, scratch} { + if _, err := os.Stat(gone); !os.IsNotExist(err) { + return false + } + } + return true +} + +// assertDictationStepTerminalState is where the fixture above has to end up once +// nothing is failing: the newest copy live, its own holder gone, the older one +// kept under the Kept prefix with its install intact, and the empty holder +// reaped. Content is read back rather than names checked, because a park that +// moved a name and lost the install passes every name assertion. +func assertDictationStepTerminalState(t *testing.T, dest, older, newest, scratch string) { + t.Helper() + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "v2" { + t.Errorf("dest engine = %q (err %v), want the newest copy %q", got, err, "v2") + } + parked := keptName(t, dest, older) + kept, err := os.ReadFile(filepath.Join(parked, "install", "engine")) + if err != nil || string(kept) != "v1" { + t.Errorf("the parked copy's engine = %q (err %v), want %q", kept, err, "v1") + } + for _, gone := range []string{newest, scratch} { + if _, err := os.Stat(gone); !os.IsNotExist(err) { + t.Errorf("%s should be gone once recovery finished: %v", gone, err) + } + } + base := filepath.Base(dest) + assertCopySet(t, filepath.Dir(dest), []string{base + holderSuffix, base + keptSuffix}, []string{parked}, 3) +} + +// gateAt parks the write path at one named boundary: it signals when the call +// whose arguments match is reached, holds that call until release, and passes +// every other call through. blockStep gates EVERY call to a step, which for a +// rename would park a promotion at the rename inside its marker write rather +// than at the set-aside or the publish, so a matched gate is what puts the +// writer at the boundary a row is actually about. +func gateAt(t *testing.T, match func(from, to string) bool) (reached <-chan struct{}, release func()) { + t.Helper() + real := holderFS + t.Cleanup(func() { holderFS = real }) + arrived := make(chan struct{}) + gate := make(chan struct{}) + var arrive, open sync.Once + release = func() { open.Do(func() { close(gate) }) } + t.Cleanup(release) + holderFS.rename = func(from, to string) error { + if match(from, to) { + arrive.Do(func() { close(arrived) }) + <-gate + } + return real.rename(from, to) + } + return arrived, release +} + +// signalRemoveAll reports when a removeAll of a matching path is reached. It +// wraps whatever is already installed, so layering it over blockStep signals +// BEFORE the call parks rather than after it is released. +func signalRemoveAll(t *testing.T, match func(path string) bool) <-chan struct{} { + t.Helper() + installed := holderFS + t.Cleanup(func() { holderFS = installed }) + arrived := make(chan struct{}) + var arrive sync.Once + holderFS.removeAll = func(path string) error { + if match(path) { + arrive.Do(func() { close(arrived) }) + } + return installed.removeAll(path) + } + return arrived +} + +// A recovery pass and a live promotion for one destination, running at the same +// time, at each of the three boundaries where the destination's only copy is in +// a holder. Nothing here asserts that recovery wins: the claim is that neither +// side destroys the other's copy, that the promotion that was already running +// completes, and that the recovering side either waits the lock out or is told +// the install is in progress. The Install lock is what makes that true, and this +// is the only test that drives both sides of it at once, under -race, through +// one package-level seam. +func TestDictationRecoveryRacesALivePromotion(t *testing.T) { + for _, tc := range []struct { + name string + gate func(t *testing.T, root, dest string) (reached <-chan struct{}, release func()) + }{ + { + // The previous install is on its way into the holder: for that + // instant the destination still holds it and the holder is empty. + name: "set-aside", + gate: func(t *testing.T, root, dest string) (<-chan struct{}, func()) { + return gateAt(t, func(from, to string) bool { + return from == dest && filepath.Base(to) == "install" + }) + }, + }, + { + // The destination is absent and the holder has the only copy of the + // previous install. A recovering pass that acted here would restore + // that copy over the publish this promotion is in the middle of. + name: "publish", + gate: func(t *testing.T, root, dest string) (<-chan struct{}, func()) { + return gateAt(t, func(from, to string) bool { + return to == dest && filepath.Base(from) == "stage" + }) + }, + }, + { + // The publish landed and the commit flag is written, so the copy in + // the holder is superseded and the promotion is deleting it. A + // recovering pass that reached the same holder would be deleting it + // too. + name: "reap", + gate: func(t *testing.T, root, dest string) (<-chan struct{}, func()) { + release := blockStep(t, "removeAll") + base := filepath.Base(dest) + return signalRemoveAll(t, func(path string) bool { + return strings.HasPrefix(filepath.Base(path), base+holderSuffix) + }), release + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + txn := lockFor(t, dest) + + var watch deleteWatch + watch.install(t) + reached, release := tc.gate(t, root, dest) + + var promoteErr error + promoted := make(chan struct{}) + go func() { + defer close(promoted) + promoteErr = promoteStagedDir(txn, stage, dest, "engine", nil) + // The recovering side is waiting on this lock, and the only + // thing that ends its wait is the promotion letting go of it. + txn.release() + }() + <-reached + + var recovered bool + var lockErr error + var reports []string + done := make(chan struct{}) + go func() { + defer close(done) + lockErr = withDestinationLock(context.Background(), root, filepath.Base(dest), testPublished, func(own *destTxn) error { + recovered = true + restoreInterruptedPromotion(own, dest, testPublished, reporterFor(&reports)) + return nil + }) + }() + release() + <-promoted + <-done + + if promoteErr != nil { + t.Errorf("the promotion that was already running must finish: %v", promoteErr) + } + // Waited, or was told the install is in progress. Either is a pass; + // acting on the destination while the promotion held it is not. + if !recovered && !errors.Is(lockErr, errInstallInProgress) { + t.Errorf("the recovering side should have waited or reported the install in progress, got %v", lockErr) + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "new" { + t.Errorf("dest engine = %q (err %v), want the promotion's own install %q", got, err, "new") + } + if _, denied := watch.counts(); len(denied) > 0 { + t.Errorf("neither side may delete an uncommitted copy, got %v", denied) + } + // The promotion cleaned up after itself, and recovery left nothing + // of its own behind. + base := filepath.Base(dest) + assertCopySet(t, root, []string{base + holderSuffix, base + keptSuffix}, nil, 1) + }) + } +} + +// ---- the rows no single crash of the writer can reach ---------------------- + +// The X rows of the Acceptance Examples. They need arrangements the six crash +// shapes cannot produce: a held lock, more than one candidate, a destination +// that exists and cannot serve, a restore that fails, or a legacy directory +// planted by hand. Everything else about them is the crash table's contract: +// the whole set of copies beside the destination is compared, not one path at a +// time, and every row runs twice because recovery keeps no memory. + +// xFile is the destination after a pass. An empty name means it must be absent, +// which is a real terminal state here and not a missing expectation. +type xFile struct { + name string + content string +} + +// xCopy is one copy and where it must be when the pass ends. final is written +// out rather than derived from a disposition, because these rows retain copies +// under three different names (its own, the Kept one, a freshly allocated one) +// and a derived name would encode the test's guess at the rule the row exists to +// check. An empty final is a delete. +type xCopy struct { + final string + file string + content string +} + +type xWant struct { + live xFile + copies []xCopy + // report is the fragments this pass's report must name. A copy recovery + // keeps and never names is one an operator cannot find. + report []string + // silent is the fragments it must NOT name. A sibling that merely collides + // with the prefix is not a copy of this install, and reporting on it would + // tell an operator their own directory is recovery's residue. + silent []string +} + +type dictationXRow struct { + id string + // arrange plants the state and returns what each pass must produce, plus, + // where the row needs one, the fault that makes the row's situation happen + // during a given pass. They come back together so both can close over the + // paths the arrangement planted. + arrange func(t *testing.T, root, dest string) (want func(pass int) xWant, impede func(t *testing.T, pass int) func()) + // recover overrides how the pass is driven, for the one row whose subject is + // the lock itself rather than what is on disk. + recover func(t *testing.T, root, dest string, pass int) []string +} + +// bothPasses is the common case: a row whose two passes look identical, because +// pass one reached a terminal state and pass two has nothing left to do. +func bothPasses(w xWant) func(int) xWant { + return func(int) xWant { return w } +} + +// faultDuring installs one fault for the length of a pass and returns the undo. +// The pass-scoped restore is the point: a row that fails a step on pass one only +// needs the seam back before pass two runs. +func faultDuring(t *testing.T, step string, match func(args ...string) bool) func() { + t.Helper() + saved := holderFS + injectFault(t, step, match, errors.New("injected recovery failure")) + return func() { holderFS = saved } +} + +// plantSibling creates a directory beside the destination that shares the holder +// prefix and is not a holder: someone else's directory, which recovery has to +// leave alone and not report on. +func plantSibling(t *testing.T, path, content string) string { + t.Helper() + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "notes"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func dictationXRows() []dictationXRow { + return []dictationXRow{ + { + // A destination that exists and cannot serve is not a reason to keep + // a usable copy out of it, and the husk is not this code's to delete + // either: it moves into a fresh sequenced holder of its own and is + // parked from there. + id: "X1", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + plantHolder(t, dest, 100, "v1", false) + plantUnusableDest(t, dest) + kept := keptNamed(dest, 101) + return func(pass int) xWant { + w := xWant{ + live: xFile{"engine", "v1"}, + copies: []xCopy{{final: kept, file: filepath.Join("install", "bin", "README"), content: "partial"}}, + } + if pass == 1 { + w.report = []string{kept} + } + return w + }, nil + }, + }, + { + // A Kept backup is permanent. A later publish is not evidence about + // it: the commit flag goes into the holder the publishing + // transaction set aside, and a Kept backup is never that holder. + id: "X2", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + stagedTree(t, dest, "live") + kept := plantKeptHolder(t, dest, 500, "unproven") + return bothPasses(xWant{ + live: xFile{"engine", "live"}, + copies: []xCopy{{final: kept, file: filepath.Join("install", "engine"), content: "unproven"}}, + }), nil + }, + }, + { + // The newest usable copy cannot be put back. Falling through to the + // older one would put an install at the destination that the next + // pass reads as having published over the newer copy, which is how a + // retained copy turns into a deleted one two passes later. + id: "X3", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + older := plantHolder(t, dest, 100, "v1", false) + newest := plantHolder(t, dest, 200, "v2", false) + want := bothPasses(xWant{ + copies: []xCopy{ + {final: older, file: filepath.Join("install", "engine"), content: "v1"}, + {final: newest, file: filepath.Join("install", "engine"), content: "v2"}, + }, + report: []string{newest}, + }) + return want, func(t *testing.T, pass int) func() { + return faultDuring(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(newest, "install") + }) + } + }, + }, + { + // Unusable is a selection decision, not a delete: the newest copy is + // skipped for one that can serve and is then kept like any other + // copy nothing proves was superseded. + id: "X4", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + plantHolder(t, dest, 100, "v1", false) + newest := plantHolder(t, dest, 200, "v2", false) + if err := os.Rename(filepath.Join(newest, "install", "engine"), filepath.Join(newest, "install", "partial")); err != nil { + t.Fatal(err) + } + kept := keptName(t, dest, newest) + return func(pass int) xWant { + w := xWant{ + live: xFile{"engine", "v1"}, + copies: []xCopy{{final: kept, file: filepath.Join("install", "partial"), content: "v2"}}, + } + if pass == 1 { + w.report = []string{kept} + } + return w + }, nil + }, + }, + { + // Unreadable is not unusable. A filesystem fault says nothing about + // the copy, so the whole destination stops rather than ruling on the + // copies that could be read while the newest one could not. + id: "X5", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + holder := plantHolder(t, dest, 100, "v1", false) + want := bothPasses(xWant{ + copies: []xCopy{{final: holder, file: filepath.Join("install", "engine"), content: "v1"}}, + report: []string{holder}, + }) + return want, func(t *testing.T, pass int) func() { + return faultDuring(t, "stat", func(args ...string) bool { + return args[0] == filepath.Join(holder, committedFile) + }) + } + }, + }, + { + // A destination another process holds is one recovery knows nothing + // about: on disk a live promotion mid-swap and a crashed one are the + // same thing, and the Install lock is the only thing that tells them + // apart. The lock is per open file description, so the handle this + // test holds during pass one is what a second process's hold looks + // like to the recovering side. + id: "X6", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + holder := plantHolder(t, dest, 100, "v1", false) + return func(pass int) xWant { + if pass == 1 { + return xWant{ + copies: []xCopy{{final: holder, file: filepath.Join("install", "engine"), content: "v1"}}, + } + } + return xWant{live: xFile{"engine", "v1"}} + }, nil + }, + recover: recoverAroundAHeldLock, + }, + { + // A name that shares the prefix and fails the grammar carries no + // order at all, so it is not a candidate for anything: not restored, + // not parked, not deleted. At this site it is not reported either, + // because the prefix is a suffix of someone's own directory name and + // telling an operator their notes are recovery's residue is worse + // than saying nothing. + id: "X8", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + sibling := plantSibling(t, dest+holderSuffix+"notes", "mine") + return bothPasses(xWant{ + copies: []xCopy{{final: sibling, file: "notes", content: "mine"}}, + silent: []string{sibling}, + }), nil + }, + }, + { + // The destination went away outside this code and the only copy left + // carries a commit flag. Committed is second in selection, never + // excluded from it: a copy that was superseded by an install that is + // no longer there is still a copy of something. + id: "X9", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + plantHolder(t, dest, 100, "v1", true) + return bothPasses(xWant{live: xFile{"engine", "v1"}}), nil + }, + }, + { + // Two copies, one destination: the newest goes back and the older is + // kept rather than deleted, because nothing proves anything + // published over it either. + id: "X10", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + older := plantHolder(t, dest, 100, "v1", false) + plantHolder(t, dest, 200, "v2", false) + kept := keptName(t, dest, older) + return func(pass int) xWant { + w := xWant{ + live: xFile{"engine", "v2"}, + copies: []xCopy{{final: kept, file: filepath.Join("install", "engine"), content: "v1"}}, + } + if pass == 1 { + w.report = []string{kept} + } + return w + }, nil + }, + }, + { + // v0.8.0 residue: a holder-shaped name with no sequence in it. It + // cannot be ordered against anything, so it is left exactly where it + // is, and like any other unstampable sibling at this site it is not + // reported. + id: "X11", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + residue := plantSibling(t, dest+holderSuffix+"1699999999", "resid") + return bothPasses(xWant{ + copies: []xCopy{{final: residue, file: "notes", content: "resid"}}, + silent: []string{residue}, + }), nil + }, + }, + { + // Same as X1 with the only candidate committed. The husk still moves + // aside first, and the copy that replaces it is the committed one, + // because there is no uncommitted copy to prefer. + id: "X12", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + plantHolder(t, dest, 100, "v1", true) + plantUnusableDest(t, dest) + kept := keptNamed(dest, 101) + return func(pass int) xWant { + w := xWant{ + live: xFile{"engine", "v1"}, + copies: []xCopy{{final: kept, file: filepath.Join("install", "bin", "README"), content: "partial"}}, + } + if pass == 1 { + w.report = []string{kept} + } + return w + }, nil + }, + }, + { + // Nothing beside it can replace the husk, so it is not taken apart: + // an operator with an unusable install directory has strictly more + // than one with no install directory at all. + id: "X13", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + holder := plantHolder(t, dest, 100, "v1", false) + if err := os.Rename(filepath.Join(holder, "install", "engine"), filepath.Join(holder, "install", "partial")); err != nil { + t.Fatal(err) + } + plantUnusableDest(t, dest) + kept := keptName(t, dest, holder) + return func(pass int) xWant { + w := xWant{ + live: xFile{filepath.Join("bin", "README"), "partial"}, + copies: []xCopy{{final: kept, file: filepath.Join("install", "partial"), content: "v1"}}, + } + if pass == 1 { + w.report = []string{"no usable install"} + } + return w + }, nil + }, + }, + { + // The husk was already aside when the restore failed, so recovery + // owes the destination its husk back: the row's whole claim is that + // the destination ends the pass exactly as it was found, with the + // candidate still where it was and the set-aside holder, now empty, + // gone. + id: "X14", + arrange: func(t *testing.T, root, dest string) (func(int) xWant, func(*testing.T, int) func()) { + holder := plantHolder(t, dest, 100, "v1", false) + plantUnusableDest(t, dest) + want := bothPasses(xWant{ + live: xFile{filepath.Join("bin", "README"), "partial"}, + copies: []xCopy{{final: holder, file: filepath.Join("install", "engine"), content: "v1"}}, + report: []string{holder}, + }) + return want, func(t *testing.T, pass int) func() { + return faultDuring(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(holder, "install") && args[1] == dest + }) + } + }, + }, + } +} + +// recoverAroundAHeldLock drives X6 through the production entry point rather +// than calling the reconcile directly: the lock is the row's subject, and +// withDestinationLock is the only place a caller finds out it could not have it. +// Pass one runs with the destination's lock held by a handle this test owns, +// which is what a second process holding it looks like from here; pass two runs +// with it free. +func recoverAroundAHeldLock(t *testing.T, root, dest string, pass int) []string { + t.Helper() + var reports []string + recovered := false + if pass == 1 { + held := lockFor(t, dest) + defer held.release() + // Long enough that a slow machine does not turn a wait into a skip by + // accident, short enough that the row does not sit on the default + // two-minute budget for a lock nothing is going to release. + saved := installLockWait + installLockWait = 200 * time.Millisecond + defer func() { installLockWait = saved }() + err := withDestinationLock(context.Background(), root, filepath.Base(dest), testPublished, func(own *destTxn) error { + recovered = true + return nil + }) + if recovered { + t.Fatal("the lock was held for the whole pass, so recovery must not have taken it") + } + if !errors.Is(err, errInstallInProgress) { + t.Fatalf("a held destination should report the install in progress, got %v", err) + } + return []string{err.Error()} + } + if err := withDestinationLock(context.Background(), root, filepath.Base(dest), testPublished, func(own *destTxn) error { + restoreInterruptedPromotion(own, dest, testPublished, reporterFor(&reports)) + return nil + }); err != nil { + t.Fatalf("the lock is free on this pass: %v", err) + } + return reports +} + +func TestDictationXMatrix(t *testing.T) { + for _, row := range dictationXRows() { + t.Run(row.id, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + want, impede := row.arrange(t, root, dest) + var txn *destTxn + if row.recover == nil { + // One handle for the whole row: the lock is per open file + // description, so a second acquire in this process would + // contend with the first rather than nest. + txn = lockFor(t, dest) + } + + var watch deleteWatch + for pass := 1; pass <= 2; pass++ { + var reports []string + func() { + saved := holderFS + defer func() { holderFS = saved }() + if impede != nil { + defer impede(t, pass)() + } + // After the impediment, so a delete the injected fault + // would have failed is still seen as a delete that was + // asked for. + watch.install(t) + if row.recover != nil { + reports = row.recover(t, root, dest, pass) + return + } + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reports)) + }() + assertDictationXState(t, dest, want(pass), reports, pass) + } + if _, denied := watch.counts(); len(denied) > 0 { + t.Errorf("no row may delete a copy nothing proves was superseded, got %v", denied) + } + }) + } +} + +// assertDictationXState asserts a row's whole terminal state: the destination, +// the content of every copy at the name it is supposed to be at, the full set of +// directories under both prefixes, and the report. Content rather than names +// alone, because a park that moved a name and lost the install passes every name +// assertion there is. +func assertDictationXState(t *testing.T, dest string, want xWant, reports []string, pass int) { + t.Helper() + if want.live.name == "" { + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Errorf("pass %d: %s should be absent: %v", pass, dest, err) + } + } else { + got, err := os.ReadFile(filepath.Join(dest, want.live.name)) + if err != nil || string(got) != want.live.content { + t.Errorf("pass %d: dest %s = %q (err %v), want %q", pass, want.live.name, got, err, want.live.content) + } + } + var paths []string + for _, c := range want.copies { + if c.final == "" { + continue + } + paths = append(paths, c.final) + got, err := os.ReadFile(filepath.Join(c.final, c.file)) + if err != nil || string(got) != c.content { + t.Errorf("pass %d: %s = %q (err %v), want %q", pass, filepath.Join(c.final, c.file), got, err, c.content) + } + } + base := filepath.Base(dest) + assertCopySet(t, filepath.Dir(dest), []string{base + holderSuffix, base + keptSuffix}, paths, pass) + for _, fragment := range want.report { + if !slices.ContainsFunc(reports, func(m string) bool { return strings.Contains(m, fragment) }) { + t.Errorf("pass %d: the report should name %q, got %v", pass, fragment, reports) + } + } + for _, fragment := range want.silent { + if slices.ContainsFunc(reports, func(m string) bool { return strings.Contains(m, fragment) }) { + t.Errorf("pass %d: the report should say nothing about %q, got %v", pass, fragment, reports) + } + } +} diff --git a/internal/dictation/download_test.go b/internal/dictation/download_test.go index da267e35a..087582d63 100644 --- a/internal/dictation/download_test.go +++ b/internal/dictation/download_test.go @@ -7,13 +7,22 @@ import ( "errors" "fmt" "io" + "math" "net/http" "net/http/httptest" "os" "path/filepath" + "reflect" "runtime" + "slices" + "strconv" "strings" + "sync" + "sync/atomic" "testing" + "time" + + "github.com/Gitlawb/zero/internal/lockutil" ) // Tiny tar.bz2 fixtures (generated in-repo): the engine has top/bin/sherpa-onnx- @@ -243,3 +252,2950 @@ func TestUnsupportedPlatformSetupError(t *testing.T) { t.Fatalf("want *SetupError for unsupported platform, got %v", err) } } + +func stagedTree(t *testing.T, dir, content string) string { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "engine"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return dir +} + +func TestPromoteStagedDirReplacesAPreviousInstall(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil); err != nil { + t.Fatalf("promote: %v", err) + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "new" { + t.Fatalf("engine = %q, err %v, want %q", got, err, "new") + } + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if strings.Contains(e.Name(), ".previous-") { + t.Errorf("the set-aside install should be cleaned up, found %s", e.Name()) + } + } +} + +func TestPromoteStagedDirWorksWithNoPreviousInstall(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil); err != nil { + t.Fatalf("promote: %v", err) + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "new" { + t.Fatalf("engine = %q, err %v, want %q", got, err, "new") + } +} + +// The old code deleted the previous install before it had anything to put in +// its place, so a failed promotion left the user with no engine at all. +func TestPromoteStagedDirKeepsThePreviousInstallWhenPromotionFails(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + // Fail only the promotion, so the restore that follows it still runs. The + // set-aside rename goes through the same seam now, and it has to stay real + // or there is no previous install to put back. + real := holderFS.rename + holderFS.rename = func(from, to string) error { + if from == stage { + return errors.New("injected promotion failure") + } + return real(from, to) + } + t.Cleanup(func() { holderFS.rename = real }) + + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil); err == nil { + t.Fatal("a failed promotion must be reported") + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil { + t.Fatalf("the previous install was not restored: %v", err) + } + if string(got) != "old" { + t.Fatalf("engine = %q, want the previous %q", got, "old") + } +} + +// If the restore fails too, the only remaining copy must survive the cleanup. +func TestPromoteStagedDirKeepsTheSetAsideCopyWhenRestoreAlsoFails(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-dir") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + // The promotion and the restore are the two renames into dest; the set-aside + // that fills the holder stays real, or there is no retained copy to assert on. + real := holderFS.rename + holderFS.rename = func(from, to string) error { + if to == dest { + return errors.New("injected rename failure") + } + return real(from, to) + } + t.Cleanup(func() { holderFS.rename = real }) + + err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil) + if err == nil { + t.Fatal("a failed promotion must be reported") + } + if !strings.Contains(err.Error(), "previous install left in") { + t.Errorf("the error should name the retained copy, got: %v", err) + } + entries, readErr := os.ReadDir(root) + if readErr != nil { + t.Fatal(readErr) + } + found := false + for _, e := range entries { + if !strings.Contains(e.Name(), ".previous-") { + continue + } + got, readErr := os.ReadFile(filepath.Join(root, e.Name(), "install", "engine")) + if readErr == nil && string(got) == "old" { + found = true + } + } + if !found { + t.Error("the previous install was deleted along with the holder dir") + } +} + +// A process stop between the two renames in promoteStagedDir leaves destDir +// absent and the only usable install in a .previous-* holder. Nothing else knows +// about that holder, so without a repair the engine is re-downloaded and a host +// that cannot reach the network stays without dictation despite having a copy. +func TestRestoreInterruptedPromotionPutsTheInstallBack(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + holder := plantHolder(t, dest, 1, "kept", false) + + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil { + t.Fatalf("the interrupted promotion was not restored: %v", err) + } + if string(got) != "kept" { + t.Fatalf("restored engine = %q, want %q", got, "kept") + } + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("the holder should be cleared after a restore, got %v", err) + } +} + +// A promotion that published its install but could not remove the holder leaves +// a complete copy of the OLD install beside a live one, and the commit flag +// inside it is the evidence that a publish actually landed over it. Only that +// flag licenses the delete: a copy with no flag may still be the last usable +// one, so it is parked rather than reaped, and a destination that merely EXISTS +// proves nothing about either. +func TestRestoreInterruptedPromotionDeletesOnlyCommittedHolders(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "new") + superseded := plantHolder(t, dest, 100, "old", true) + uncommitted := plantHolder(t, dest, 50, "older", false) + // One handle for both passes: the lock is per open file description, so a + // second acquire in this process contends with the first rather than nests. + txn := lockFor(t, dest) + + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + + // The live install is never touched. This is the assertion that matters most. + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "new" { + t.Fatalf("the live install must be left alone: got %q err %v", got, err) + } + if _, err := os.Stat(superseded); !os.IsNotExist(err) { + t.Errorf("a copy proven superseded by its commit flag should be reaped, got %v", err) + } + parked := keptName(t, dest, uncommitted) + if _, err := os.Stat(uncommitted); !os.IsNotExist(err) { + t.Errorf("the uncommitted copy should have left the scanned prefix, got %v", err) + } + kept, err := os.ReadFile(filepath.Join(parked, "install", "engine")) + if err != nil || string(kept) != "older" { + t.Fatalf("the uncommitted copy must survive the park intact: %q err %v", kept, err) + } + assertReports(t, reported, parked) + + // No memory: the parked copy is under a prefix the scan does not read, so a + // second pass has nothing left to decide. + reported = nil + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + if _, err := os.Stat(filepath.Join(parked, "install", "engine")); err != nil { + t.Errorf("the second pass must leave the parked copy alone: %v", err) + } + if len(reported) != 0 { + t.Errorf("a pass with nothing beside the destination should report nothing, got %v", reported) + } +} + +// A copy parked by recovery is permanent: a later install that publishes over +// the destination sets the CURRENT install aside, never a Kept backup, so it can +// never write the commit flag that would license removing one. Nothing but the +// operator takes a Kept backup off disk. +func TestRestoreInterruptedPromotionKeepsAParkedCopyAfterALaterInstall(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "new") + uncommitted := plantHolder(t, dest, 50, "older", false) + txn := lockFor(t, dest) + + restoreInterruptedPromotion(txn, dest, testPublished, nil) + parked := keptName(t, dest, uncommitted) + if _, err := os.Stat(filepath.Join(parked, "install", "engine")); err != nil { + t.Fatalf("seeding a parked copy: %v", err) + } + + // A real later install over the same destination. + stage := stagedTree(t, filepath.Join(root, "stage"), "newest") + if err := promoteStagedDir(txn, stage, dest, "engine", nil); err != nil { + t.Fatalf("the later install failed: %v", err) + } + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + + // Silence is the assertion that the Kept prefix is outside the scan: a pass + // that enumerated it would have to rule on the copy, and every ruling it + // could reach (park, delete, restore) says so in the report. + if len(reported) != 0 { + t.Errorf("recovery must not enumerate the Kept prefix, it reported %v", reported) + } + kept, err := os.ReadFile(filepath.Join(parked, "install", "engine")) + if err != nil || string(kept) != "older" { + t.Errorf("a Kept backup must survive a later successful install: %q err %v", kept, err) + } + live, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(live) != "newest" { + t.Errorf("the later install should be live: %q err %v", live, err) + } +} + +// A destDir that is merely NOT EMPTY is no evidence a promotion published there. +// An empty husk and a half-populated one are the same thing to recovery, and a +// copy beside one may be the last usable install there is. So the destination is +// not what wins: the usable copy is restored over the husk, and the husk itself +// is set aside rather than deleted, which is how an offline caller stops being +// stuck beside an install it cannot use. +func TestRestoreInterruptedPromotionReplacesADestThatIsNotAUsableInstall(t *testing.T) { + for _, tc := range []struct { + name string + seed func(t *testing.T, dest string) + usable func(string) bool + content string + }{ + { + name: "empty husk", + seed: func(t *testing.T, dest string) {}, + usable: func(dir string) bool { bin, _ := resolveEnginePaths(dir, false); return fileExists(bin) }, + content: "bin/sherpa-onnx-offline", + }, + { + name: "non-empty but no engine binary", + seed: func(t *testing.T, dest string) { + t.Helper() + plantUnusableDest(t, dest) + }, + usable: func(dir string) bool { bin, _ := resolveEnginePaths(dir, false); return fileExists(bin) }, + content: "bin/sherpa-onnx-offline", + }, + { + name: "model dir without tokens.txt", + seed: func(t *testing.T, dest string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dest, "something.onnx"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + }, + usable: dirHasModel, + content: "tokens.txt", + }, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + if err := os.MkdirAll(dest, 0o755); err != nil { + t.Fatal(err) + } + tc.seed(t, dest) + // The copy has to pass the case's own predicate, which is the whole + // point: recovery applies the CALLER's predicate to a candidate, not + // a structural guess about what an install looks like. + holder := plantHolder(t, dest, 100, "the only copy", false) + install := filepath.Join(holder, "install") + if err := os.MkdirAll(filepath.Dir(filepath.Join(install, tc.content)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(install, tc.content), []byte("x"), 0o755); err != nil { + t.Fatal(err) + } + + restoreInterruptedPromotion(lockFor(t, dest), dest, tc.usable, nil) + + if !tc.usable(dest) { + t.Errorf("the usable copy should be live at %s", dest) + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "the only copy" { + t.Errorf("the destination should hold the copy: got %q err %v", got, err) + } + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("the restored copy's holder should be cleared, got %v", err) + } + }) + } +} + +// A holder never replaces a destination that holds a USABLE install: that +// install is the published one and the copy beside it is the leftover. Against +// an unusable destination the ruling inverts, and the husk moves aside rather +// than being deleted. +func TestRestoreInterruptedPromotionKeepsAUsableDestAndReplacesAnUnusableOne(t *testing.T) { + for _, tc := range []struct { + name string + live string + committed bool + }{ + {name: "empty dest", live: ""}, + {name: "populated dest", live: "live", committed: true}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + if err := os.MkdirAll(dest, 0o755); err != nil { + t.Fatal(err) + } + if tc.live != "" { + if err := os.WriteFile(filepath.Join(dest, "engine"), []byte(tc.live), 0o644); err != nil { + t.Fatal(err) + } + } + holder := plantHolder(t, dest, 1, "stale", tc.committed) + + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if tc.live == "" { + // The husk was unusable, so the copy beside it wins and the husk + // is set aside under a Kept name rather than dropped. + if err != nil || string(got) != "stale" { + t.Fatalf("the usable copy should have replaced the husk: %q err %v", got, err) + } + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("the restored copy's holder should be cleared, got %v", err) + } + return + } + if err != nil || string(got) != tc.live { + t.Fatalf("engine = %q, err %v, want the live %q", got, err, tc.live) + } + // Its commit flag proves the live install published over it, which + // is the only thing that licenses the delete. + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("a live dest should reap the copy it provably superseded, got %v", err) + } + }) + } +} + +// An interrupted promotion of the MODEL leaves it in a holder exactly as it does +// for the engine, and the model side is what an offline user loses if nothing +// puts it back: with no network there is no download to fall back to. +func TestEnsureLocalEngineRestoresAnInterruptedModelPromotion(t *testing.T) { + srv := fakeReleaseServer(t, engineSHA, modelSHA) + dest := t.TempDir() + opts := DownloadOptions{ + DestRoot: dest, EngineVersion: "test", APIBase: srv.URL, platformKey: "linux-amd64", skipPinned: true, + } + comp, err := EnsureLocalEngine(context.Background(), opts) + if err != nil { + t.Fatalf("seeding the install: %v", err) + } + + // Stop the world where promoteStagedDir has moved the model aside but has + // not yet published the staged copy. + modelDir := filepath.Dir(comp.ModelPath) + holder := fmt.Sprintf("%s%s%020d%s", modelDir, holderSuffix, 1, holderSeqSuffix) + if err := os.MkdirAll(holder, 0o700); err != nil { + t.Fatal(err) + } + if err := writeHolderMarker(holder, txnMarker{Kind: holderMarkerKind, Dest: filepath.Base(modelDir), Seq: 1}); err != nil { + t.Fatal(err) + } + if err := os.Rename(modelDir, filepath.Join(holder, "install")); err != nil { + t.Fatal(err) + } + + offline := offlineAPIBase(t) + got, err := EnsureLocalEngine(context.Background(), DownloadOptions{ + DestRoot: dest, EngineVersion: "test", APIBase: offline, platformKey: "linux-amd64", skipPinned: true, + }) + if err != nil { + t.Fatalf("the model set aside by an interrupted promotion was not restored: %v", err) + } + if !fileExists(filepath.Join(got.ModelPath, "tokens.txt")) { + t.Errorf("model tokens.txt missing under %q", got.ModelPath) + } + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("the holder should be cleared after a restore, got %v", err) + } +} + +// offlineAPIBase returns an API base nothing is listening on, so any attempt to +// resolve a release asset fails instead of silently downloading. +func offlineAPIBase(t *testing.T) string { + t.Helper() + srv := httptest.NewServer(http.NewServeMux()) + url := srv.URL + srv.Close() + return url +} + +// lockFor takes the Install lock the promotion path now requires, for the +// destination the test is about to drive. Released in cleanup, and release is +// idempotent, so a test that goes on to call EnsureLocalEngine releases it +// early rather than waiting out its own lock. +func lockFor(t *testing.T, destDir string) *destTxn { + t.Helper() + txn, err := lockDestination(context.Background(), filepath.Dir(destDir), filepath.Base(destDir)) + if err != nil { + t.Fatalf("locking %s: %v", destDir, err) + } + t.Cleanup(txn.release) + return txn +} + +// testPublished is the "is this a real install" predicate these tests use: the +// fixtures write an "engine" file, so its presence is what publication means here. +func testPublished(dir string) bool { + _, err := os.Stat(filepath.Join(dir, "engine")) + return err == nil +} + +// plantHolder writes an install into a holder named and marked the way +// promoteStagedDir writes one, so recovery sees the same shape it does in +// production: the name alone is not ownership, and a fixture without the marker +// would test a directory recovery is supposed to leave alone. committed plants +// the flag the publish writes, which is the only evidence that licenses removing +// the copy, so a fixture has to say which of the two it is. +func plantHolder(t *testing.T, destDir string, seq int64, content string, committed bool) string { + t.Helper() + holder := fmt.Sprintf("%s%s%020d%s", destDir, holderSuffix, seq, holderSeqSuffix) + install := filepath.Join(holder, "install") + if err := os.MkdirAll(install, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(install, "engine"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + if err := writeHolderMarker(holder, txnMarker{Kind: holderMarkerKind, Dest: filepath.Base(destDir), Seq: seq}); err != nil { + t.Fatal(err) + } + if committed { + if err := writeCommitFlag(holder); err != nil { + t.Fatal(err) + } + } + return holder +} + +// keptName is the name a parked holder takes: the same sequence under the Kept +// prefix, which is what makes a retained copy findable by the operator and +// invisible to the next scan. +func keptName(t *testing.T, destDir, holder string) string { + t.Helper() + seq, ok := holderStamp(destDir, holder) + if !ok { + t.Fatalf("holder name %q carries no sequence", filepath.Base(holder)) + } + return fmt.Sprintf("%s%s%020d%s", destDir, keptSuffix, seq, holderSeqSuffix) +} + +// plantUnowned creates a directory carrying the exact holder name grammar and no +// marker: the shape recovery must retain in place rather than restore from or +// reap, since nothing on disk attributes it. +func plantUnowned(t *testing.T, destDir string, seq int64, content string) string { + t.Helper() + holder := fmt.Sprintf("%s%s%020d%s", destDir, holderSuffix, seq, holderSeqSuffix) + install := filepath.Join(holder, "install") + if err := os.MkdirAll(install, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(install, "engine"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return holder +} + +// plantUnusableDest leaves a directory at destDir that testPublished rejects: +// present, non-empty, and holding no install. It is the husk the review's P2 +// chain ends at, and the state recovery has to be able to leave. +func plantUnusableDest(t *testing.T, destDir string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(destDir, "bin"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(destDir, "bin", "README"), []byte("partial"), 0o644); err != nil { + t.Fatal(err) + } +} + +// reporterFor collects the recovery report so a retain assertion can check the +// copy was named. A copy recovery keeps and never names is one an operator +// cannot find. +func reporterFor(reported *[]string) func(string) { + return func(m string) { *reported = append(*reported, m) } +} + +// assertReports fails unless some report line mentions each wanted fragment. +func assertReports(t *testing.T, reported []string, want ...string) { + t.Helper() + for _, fragment := range want { + if !slices.ContainsFunc(reported, func(m string) bool { return strings.Contains(m, fragment) }) { + t.Errorf("the report should name %q, got %v", fragment, reported) + } + } +} + +// A cleanup that could not finish leaves an old holder behind; a later +// interrupted promotion adds a second one. Recovery has to put back the newer +// install, and Glob's lexical order is no evidence of which that is. +func TestRestoreInterruptedPromotionPrefersTheNewestHolder(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stale := plantHolder(t, dest, 100, "stale", false) + current := plantHolder(t, dest, 200, "current", false) + + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "current" { + t.Fatalf("restored engine = %q (err %v), want the newest holder's %q", got, err, "current") + } + if _, err := os.Stat(current); !os.IsNotExist(err) { + t.Errorf("the restored holder should be cleared, got %v", err) + } + // The loser is kept rather than deleted on a guess, and it moves under the + // Kept prefix so the next pass has nothing left to rule on. + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Errorf("the older holder should have left the scanned prefix, got %v", err) + } + if _, err := os.Stat(filepath.Join(keptName(t, dest, stale), "install", "engine")); err != nil { + t.Errorf("the older holder must be kept intact under the Kept prefix: %v", err) + } +} + +// interruptPromotion drives the REAL promotion into the state a process killed +// between its two renames leaves: destDir absent, the only install in a holder +// promoteStagedDir named. Both renames fail, so nothing is put back in process +// and the holder is retained rather than cleaned up. +func interruptPromotion(t *testing.T, txn *destTxn, destDir, label string) { + t.Helper() + stage := destDir + ".incoming" + if err := os.MkdirAll(stage, 0o755); err != nil { + t.Fatal(err) + } + real := holderFS.rename + holderFS.rename = func(from, to string) error { + if to == destDir { + return errors.New("injected rename failure") + } + return real(from, to) + } + err := promoteStagedDir(txn, stage, destDir, label, nil) + holderFS.rename = real + if err == nil { + t.Fatalf("a promotion whose publish and restore both fail must report an error") + } + if _, statErr := os.Lstat(destDir); !os.IsNotExist(statErr) { + t.Fatalf("the interrupted promotion should leave %s absent, got %v", destDir, statErr) + } +} + +// holdersFor lists the holders promoteStagedDir left beside destDir. +func holdersFor(t *testing.T, destDir string) []string { + t.Helper() + holders, err := filepath.Glob(destDir + holderSuffix + "*") + if err != nil { + t.Fatal(err) + } + return holders +} + +// End to end over the real transaction, for BOTH consumers of it. The engine and +// the model are installed for real, one of them is interrupted mid-promotion by +// the real promoteStagedDir, and the next start has to put it back with no +// network to fall back on. Nothing plants a holder by hand, so the name recovery +// orders by is the one promoteStagedDir writes -- the half a hand-built fixture +// cannot check. +func TestEnsureLocalEngineRecoversARealInterruptedPromotionOffline(t *testing.T) { + for _, tc := range []struct { + name string + label string + // dirFor picks the install this case interrupts, given a finished setup. + dirFor func(t *testing.T, dest string, comp EngineComponents) string + }{ + { + name: "engine", + label: "Engine", + dirFor: func(t *testing.T, dest string, comp EngineComponents) string { + t.Helper() + matches, err := filepath.Glob(filepath.Join(dest, "engine-*")) + if err != nil || len(matches) != 1 { + t.Fatalf("want exactly one engine dir under %s, got %v (err %v)", dest, matches, err) + } + return matches[0] + }, + }, + { + name: "model", + label: "Model", + dirFor: func(t *testing.T, dest string, comp EngineComponents) string { + t.Helper() + return filepath.Dir(comp.ModelPath) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + srv := fakeReleaseServer(t, engineSHA, modelSHA) + dest := t.TempDir() + comp, err := EnsureLocalEngine(context.Background(), DownloadOptions{ + DestRoot: dest, EngineVersion: "test", APIBase: srv.URL, platformKey: "linux-amd64", skipPinned: true, + }) + if err != nil { + t.Fatalf("seeding the install: %v", err) + } + + target := tc.dirFor(t, dest, comp) + targetTxn := lockFor(t, target) + interruptPromotion(t, targetTxn, target, tc.label) + // Released before EnsureLocalEngine below, which takes the same + // lock: the wait is cross-process but flock conflicts between two + // opens in one process too. + targetTxn.release() + holders := holdersFor(t, target) + if len(holders) != 1 { + t.Fatalf("want exactly one holder beside %s, got %v", target, holders) + } + // The name promoteStagedDir wrote must be one recovery can order by. + // Restoring a lone holder works either way, so without this the two + // halves could drift apart and only a second holder would show it. + if _, ok := holderStamp(target, holders[0]); !ok { + t.Fatalf("recovery cannot order the name promoteStagedDir wrote: %q", holders[0]) + } + + // No network: if the holder is not found there is nothing to fall + // back on, which is the failure the offline user actually sees. + got, err := EnsureLocalEngine(context.Background(), DownloadOptions{ + DestRoot: dest, EngineVersion: "test", APIBase: offlineAPIBase(t), platformKey: "linux-amd64", skipPinned: true, + }) + if err != nil { + t.Fatalf("the interrupted %s promotion was not recovered offline: %v", tc.name, err) + } + if !fileExists(got.BinaryPath) { + t.Errorf("engine binary missing after recovery: %q", got.BinaryPath) + } + if !fileExists(filepath.Join(got.ModelPath, "tokens.txt")) { + t.Errorf("model tokens.txt missing after recovery under %q", got.ModelPath) + } + if holders := holdersFor(t, target); len(holders) != 0 { + t.Errorf("the holder should be cleared after recovery, got %v", holders) + } + }) + } +} + +// A path is not a pattern. An install root containing a glob metacharacter -- a +// '[' is the one that silently matches nothing -- must not cost a user the +// install recovery exists to put back. +func TestRestoreInterruptedPromotionFindsHoldersUnderAnAwkwardPath(t *testing.T) { + dirNames := []string{"plain", "wei[rd", "br]ack"} + if runtime.GOOS != "windows" { + // Windows refuses these two in a filename outright, so there is no such + // path to defend there. The bracket cases above still cover the + // metacharacter that matches nothing instead of failing. + dirNames = append(dirNames, "sta*r", "que?ry") + } + for _, dirName := range dirNames { + t.Run(dirName, func(t *testing.T) { + root := filepath.Join(t.TempDir(), dirName) + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + install := filepath.Join(plantHolder(t, dest, 100, "kept", false), "install") + if _, err := os.Stat(install); err != nil { + t.Fatal(err) + } + + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "kept" { + t.Fatalf("the install was not restored under %q: got %q err %v", dirName, got, err) + } + }) + } +} + +// A backward clock leaves a STALE holder carrying a LARGER stamp. Recovery must +// still put back the install that was actually live last. Nothing is deleted +// either way at this site, so the failure is a stale restore, and the negative +// half of the assertion pins that the loser is kept. +func TestRestoreInterruptedPromotionPrefersTheRealNewerInstallOverAFutureStampedHolder(t *testing.T) { + const farFuture = int64(4_000_000_000_000_000_000) + + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + if err := os.MkdirAll(dest, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dest, "engine"), []byte("new"), 0o644); err != nil { + t.Fatal(err) + } + stale := plantHolder(t, dest, farFuture, "stale", false) + + // The real transaction sets "new" aside and never publishes. + txn := lockFor(t, dest) + interruptPromotion(t, txn, dest, "engine") + + restoreInterruptedPromotion(txn, dest, testPublished, nil) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "new" { + t.Errorf("restored %q (err %v), want the install that was live last, %q", got, err, "new") + } + // Whichever holder won, the one that lost is kept, under the Kept prefix. + if _, err := os.Stat(filepath.Join(keptName(t, dest, stale), "install", "engine")); err != nil { + t.Errorf("a holder that lost the ordering must be kept, not deleted: %v", err) + } +} + +// The allocator and the parser must agree, and the sequence must seed above a +// name a released binary wrote. +func TestHolderNamesAllocateInOrderAndParse(t *testing.T) { + t.Run("counts up and parses", func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + for want := int64(1); want <= 2; want++ { + seq, err := nextHolderSeq(dest) + if err != nil { + t.Fatal(err) + } + path, err := createSequencedHolder(dest, seq) + if err != nil { + t.Fatal(err) + } + stamp, ok := holderStamp(dest, path) + if !ok { + t.Fatalf("the allocator wrote a name recovery cannot order: %q", filepath.Base(path)) + } + if stamp != want { + t.Errorf("stamp = %d, want %d", stamp, want) + } + } + }) + + // No released version of this package ever wrote a holder: v0.8.0 staged + // under .stage-* and published with a plain rename. A nanosecond-shaped name + // beside an install is therefore something this code did not write, and + // counting it would let any such directory dictate the sequence, up to the + // maximum that refuses installs outright. + t.Run("ignores a nanosecond-shaped name it did not write", func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + const nano = int64(1_700_000_000_000_000_000) + var planted []string + for _, suffix := range []string{"x7Kq3", "12345"} { + path := fmt.Sprintf("%s%s%020d-%s", dest, holderSuffix, nano, suffix) + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } + planted = append(planted, path) + } + seq, err := nextHolderSeq(dest) + if err != nil { + t.Fatal(err) + } + if seq != 1 { + t.Errorf("next sequence = %d, want 1: neither planted name is one this code wrote", seq) + } + for _, path := range planted { + if _, err := os.Stat(path); err != nil { + t.Errorf("a name this code did not write must be left alone: %v", err) + } + } + }) +} + +func TestCreateSequencedHolderSkipsAnOccupiedNumber(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + taken := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 1, holderSeqSuffix) + if err := os.MkdirAll(taken, 0o700); err != nil { + t.Fatal(err) + } + + got, err := createSequencedHolder(dest, 1) + if err != nil { + t.Fatal(err) + } + want := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 2, holderSeqSuffix) + if got != want { + t.Errorf("claimed %q, want %q", got, want) + } +} + +func TestNextHolderSeqRefusesOverflow(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + if err := os.MkdirAll(fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, int64(math.MaxInt64), holderSeqSuffix), 0o700); err != nil { + t.Fatal(err) + } + if seq, err := nextHolderSeq(dest); err == nil { + t.Errorf("nextHolderSeq returned %d, want an error rather than a wrapped value", seq) + } +} + +func TestCreateSequencedHolderRefusesOutOfRange(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + if err := os.MkdirAll(root, 0o755); err != nil { + t.Fatal(err) + } + for _, n := range []int64{0, -1, math.MinInt64} { + if _, err := createSequencedHolder(dest, n); err == nil { + t.Errorf("createSequencedHolder(%d) succeeded, want an error", n) + } + } + if holders := holdersFor(t, dest); len(holders) != 0 { + t.Errorf("a refused allocation must create nothing, got %v", holders) + } +} + +// This site holds no lock, so exclusive creation is the only thing arbitrating. +func TestHolderSeqAllocatesDistinctValuesConcurrently(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + // 128, not a smaller number: against a check-then-create allocator this + // detects the duplicate in every run, where 16 caught it about half the time. + const writers = 128 + + var wg sync.WaitGroup + paths := make([]string, writers) + errs := make([]error, writers) + for i := 0; i < writers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + seq, err := nextHolderSeq(dest) + if err != nil { + errs[i] = err + return + } + paths[i], errs[i] = createSequencedHolder(dest, seq) + }(i) + } + wg.Wait() + + seen := map[string]bool{} + stamps := map[int64]bool{} + for i, path := range paths { + if errs[i] != nil { + t.Fatalf("writer %d: %v", i, errs[i]) + } + if seen[path] { + t.Errorf("two writers claimed the same name %q", path) + } + seen[path] = true + stamp, ok := holderStamp(dest, path) + if !ok { + t.Errorf("writer %d wrote an unorderable name %q", i, filepath.Base(path)) + continue + } + if stamps[stamp] { + t.Errorf("two writers claimed sequence %d", stamp) + } + stamps[stamp] = true + } +} + +// The holder wraps a complete previous install for as long as it exists, so it +// keeps the owner-only mode it has always had. os.Mkdir takes +// a mode where os.MkdirTemp did not, so the allocator has to name it. +func TestHolderKeepsOwnerOnlyPermissions(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Go permission bits do not map to Windows ACLs") + } + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + path, err := createSequencedHolder(dest, 1) + if err != nil { + t.Fatal(err) + } + requireUmaskAllowsWiderThan0700(t, root) + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o700 { + t.Errorf("holder mode = %o, want 0700", got) + } +} + +// requireUmaskAllowsWiderThan0700 skips when the process umask would strip the +// group and other bits anyway. Without it this assertion is vacuous under a +// umask of 077: a wrongly widened 0o755 comes back as 0700 and passes. +func requireUmaskAllowsWiderThan0700(t *testing.T, parent string) { + t.Helper() + control := filepath.Join(parent, ".umask-control") + if err := os.Mkdir(control, 0o755); err != nil { + t.Fatal(err) + } + defer func() { _ = os.RemoveAll(control) }() + info, err := os.Stat(control) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o755 { + t.Skipf("umask masks a 0755 request down to %o, so this assertion cannot tell 0700 from a widened mode", info.Mode().Perm()) + } +} + +// Recovery restores only from a holder it can prove it wrote for THIS install. +// Both decoys carry content that would land at the destination if the name alone +// were the attribution, and both sort FIRST lexically, so only the ownership +// rule can produce the wanted answer. +func TestRestoreInterruptedPromotionRestoresOnlyFromAnOwnedHolder(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + // A user's own directory that happens to start like a holder name. + collide := dest + holderSuffix + "aaa" + // A name this code would have written, with nothing behind it saying it did. + unmarked := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 900, holderSeqSuffix) + for _, decoy := range []string{collide, unmarked} { + if err := os.MkdirAll(filepath.Join(decoy, "install"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(decoy, "install", "engine"), []byte("not ours"), 0o644); err != nil { + t.Fatal(err) + } + } + owned := plantHolder(t, dest, 100, "ours", false) + + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "ours" { + t.Fatalf("recovery restored from a holder it cannot prove it wrote: got %q err %v", got, err) + } + if _, err := os.Stat(owned); !os.IsNotExist(err) { + t.Errorf("the restored holder should be cleared, got %v", err) + } + // Neither decoy is recovery's to move or delete. + for _, decoy := range []string{collide, unmarked} { + if _, err := os.Stat(filepath.Join(decoy, "install", "engine")); err != nil { + t.Errorf("%s must be left exactly as found: %v", filepath.Base(decoy), err) + } + } +} + +// A holder can be there without an install in it: the promotion creates the +// holder first, so a stop before the rename leaves an empty one. Recovery must +// step over it and keep looking rather than treating it as the newest word on +// what to restore. +func TestRestoreInterruptedPromotionSkipsAHolderWithNoInstall(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + usable := plantHolder(t, dest, 100, "kept", false) + // Newer AND owned, so ordering reaches it first and it is a real candidate, + // but it holds nothing: exactly what a stop between the allocation and the + // set-aside rename leaves. + empty := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 200, holderSeqSuffix) + if err := os.MkdirAll(empty, 0o700); err != nil { + t.Fatal(err) + } + if err := writeHolderMarker(empty, txnMarker{Kind: holderMarkerKind, Dest: filepath.Base(dest), Seq: 200}); err != nil { + t.Fatal(err) + } + + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "kept" { + t.Fatalf("the usable holder should be restored: got %q err %v", got, err) + } + if _, err := os.Stat(usable); !os.IsNotExist(err) { + t.Errorf("the restored holder should be cleared, got %v", err) + } + // A holder that is provably this code's and provably holds nothing costs + // nothing to remove, and leaving it is what accumulates scratch forever. + if _, err := os.Stat(empty); !os.IsNotExist(err) { + t.Errorf("an owned holder with no install should be removed, got %v", err) + } +} + +// ---- filesystem seam ------------------------------------------------------- + +// injectFault swaps one field of holderFS for a wrapper that fails the call +// whose arguments satisfy match and passes every other call through to the real +// primitive. The call ordinal is appended to the arguments handed to match, so +// nthCall can select by call order without a second counter, and it is counted +// with an atomic because the racing scenarios drive one seam from two +// goroutines. The whole struct is restored in t.Cleanup, so a test that injects +// twice unwinds in reverse. +func injectFault(t *testing.T, step string, match func(args ...string) bool, err error) { + t.Helper() + var calls atomic.Int64 + real := holderFS + t.Cleanup(func() { holderFS = real }) + fire := func(args ...string) bool { + n := calls.Add(1) + if match == nil { + return true + } + return match(append(args, strconv.FormatInt(n, 10))...) + } + switch step { + case "rename": + holderFS.rename = func(from, to string) error { + if fire(from, to) { + return err + } + return real.rename(from, to) + } + case "removeAll": + holderFS.removeAll = func(path string) error { + if fire(path) { + return err + } + return real.removeAll(path) + } + case "stat": + holderFS.stat = func(name string) (os.FileInfo, error) { + if fire(name) { + return nil, err + } + return real.stat(name) + } + case "lstat": + holderFS.lstat = func(name string) (os.FileInfo, error) { + if fire(name) { + return nil, err + } + return real.lstat(name) + } + case "readDir": + holderFS.readDir = func(name string) ([]os.DirEntry, error) { + if fire(name) { + return nil, err + } + return real.readDir(name) + } + case "readFile": + holderFS.readFile = func(name string) ([]byte, error) { + if fire(name) { + return nil, err + } + return real.readFile(name) + } + case "mkdir": + holderFS.mkdir = func(name string, perm os.FileMode) error { + if fire(name) { + return err + } + return real.mkdir(name, perm) + } + case "writeFile": + holderFS.writeFile = func(name string, data []byte, perm os.FileMode) error { + if fire(name) { + return err + } + return real.writeFile(name, data, perm) + } + case "create": + holderFS.create = func(name string, flag int, perm os.FileMode) (*os.File, error) { + if fire(name) { + return nil, err + } + return real.create(name, flag, perm) + } + case "createTemp": + holderFS.createTemp = func(dir, pattern string) (*os.File, error) { + if fire(dir, pattern) { + return nil, err + } + return real.createTemp(dir, pattern) + } + default: + t.Fatalf("injectFault: unknown step %q", step) + } +} + +// nthCall selects the nth call through an injected seam. It reads the ordinal +// injectFault appends to the arguments rather than counting for itself, so the +// two share one counter and a concurrent test has one place to be correct. +func nthCall(n int) func(args ...string) bool { + return func(args ...string) bool { + return len(args) > 0 && args[len(args)-1] == strconv.Itoa(n) + } +} + +// blockStep holds every call to step until the returned release runs, which is +// how a test parks one transaction mid-step and lets another reach the same +// destination. Releasing twice is safe, so a test can release on the happy path +// and still defer it. +func blockStep(t *testing.T, step string) (release func()) { + t.Helper() + gate := make(chan struct{}) + var once sync.Once + release = func() { once.Do(func() { close(gate) }) } + t.Cleanup(release) + injectFaultGate(t, step, gate) + return release +} + +// injectFaultGate is blockStep's half of the swap, split out so the wrapper it +// installs sits beside the fault wrappers above. +func injectFaultGate(t *testing.T, step string, gate <-chan struct{}) { + t.Helper() + real := holderFS + t.Cleanup(func() { holderFS = real }) + switch step { + case "rename": + holderFS.rename = func(from, to string) error { + <-gate + return real.rename(from, to) + } + case "removeAll": + holderFS.removeAll = func(path string) error { + <-gate + return real.removeAll(path) + } + case "stat": + holderFS.stat = func(name string) (os.FileInfo, error) { + <-gate + return real.stat(name) + } + default: + t.Fatalf("blockStep: unknown step %q", step) + } +} + +// The seam has to be the real filesystem until a test swaps a field. A nil field +// is a panic in whatever production path reaches it first, and a field that no +// longer behaves like its os function makes every test above it prove nothing. +func TestHolderFSDefaultsAreTheRealPrimitives(t *testing.T) { + for name, missing := range map[string]bool{ + "rename": holderFS.rename == nil, + "removeAll": holderFS.removeAll == nil, + "stat": holderFS.stat == nil, + "lstat": holderFS.lstat == nil, + "readDir": holderFS.readDir == nil, + "readFile": holderFS.readFile == nil, + "mkdir": holderFS.mkdir == nil, + "writeFile": holderFS.writeFile == nil, + "create": holderFS.create == nil, + "createTemp": holderFS.createTemp == nil, + } { + if missing { + t.Errorf("holderFS.%s is nil", name) + } + } + + root := t.TempDir() + dir := filepath.Join(root, "one") + if err := holderFS.mkdir(dir, 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + file := filepath.Join(dir, "a.txt") + if err := holderFS.writeFile(file, []byte("v0"), 0o600); err != nil { + t.Fatalf("writeFile: %v", err) + } + if _, err := holderFS.stat(file); err != nil { + t.Fatalf("stat: %v", err) + } + if _, err := holderFS.lstat(file); err != nil { + t.Fatalf("lstat: %v", err) + } + if got, err := holderFS.readFile(file); err != nil || string(got) != "v0" { + t.Fatalf("readFile = %q, %v, want %q", got, err, "v0") + } + entries, err := holderFS.readDir(dir) + if err != nil || len(entries) != 1 || entries[0].Name() != "a.txt" { + t.Fatalf("readDir = %v, %v, want one entry a.txt", entries, err) + } + tmp, err := holderFS.createTemp(dir, "seam-*") + if err != nil { + t.Fatalf("createTemp: %v", err) + } + tmpName := tmp.Name() + if err := tmp.Close(); err != nil { + t.Fatalf("close temp: %v", err) + } + opened, err := holderFS.create(tmpName, os.O_WRONLY, 0o600) + if err != nil { + t.Fatalf("create: %v", err) + } + if err := opened.Close(); err != nil { + t.Fatalf("close opened: %v", err) + } + moved := filepath.Join(root, "two") + if err := holderFS.rename(dir, moved); err != nil { + t.Fatalf("rename: %v", err) + } + if _, err := holderFS.stat(dir); !os.IsNotExist(err) { + t.Fatalf("stat of the old name = %v, want not-exist", err) + } + if err := holderFS.removeAll(moved); err != nil { + t.Fatalf("removeAll: %v", err) + } + if _, err := holderFS.stat(moved); !os.IsNotExist(err) { + t.Fatalf("stat after removeAll = %v, want not-exist", err) + } +} + +// The matrix needs both selections: "fail the rename whose source is seq 2's +// set-aside copy" is an argument match, and "fail the second remove" is an +// ordinal one. +// An ordinal alone cannot express the first, and an argument match cannot +// express a step that runs twice on the same path. +func TestInjectFaultMatchesByArgumentAndByOrdinal(t *testing.T) { + injected := errors.New("injected seam failure") + root := t.TempDir() + + t.Run("by argument", func(t *testing.T) { + injectFault(t, "rename", func(args ...string) bool { + return strings.HasSuffix(args[0], filepath.Join("seq", "backup")) + }, injected) + for _, name := range []string{"one", "seq", "three"} { + from := filepath.Join(root, name, "backup") + if err := os.MkdirAll(from, 0o700); err != nil { + t.Fatal(err) + } + err := holderFS.rename(from, filepath.Join(root, name, "moved")) + if name == "seq" { + if !errors.Is(err, injected) { + t.Fatalf("rename of %s = %v, want the injected failure", from, err) + } + continue + } + if err != nil { + t.Fatalf("rename of %s = %v, want it to pass through", from, err) + } + } + }) + + t.Run("by ordinal", func(t *testing.T) { + injectFault(t, "removeAll", nthCall(2), injected) + for i := 1; i <= 3; i++ { + dir := filepath.Join(root, fmt.Sprintf("remove-%d", i)) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + err := holderFS.removeAll(dir) + if i == 2 { + if !errors.Is(err, injected) { + t.Fatalf("removeAll call %d = %v, want the injected failure", i, err) + } + if _, statErr := os.Stat(dir); statErr != nil { + t.Fatalf("the failed call must not have removed %s: %v", dir, statErr) + } + continue + } + if err != nil { + t.Fatalf("removeAll call %d = %v, want it to pass through", i, err) + } + if _, statErr := os.Stat(dir); !os.IsNotExist(statErr) { + t.Fatalf("removeAll call %d left %s behind: %v", i, dir, statErr) + } + } + }) + + if got, want := reflect.ValueOf(holderFS.removeAll).Pointer(), reflect.ValueOf(os.RemoveAll).Pointer(); got != want { + t.Fatal("holderFS.removeAll was not restored after the subtest's cleanup") + } + if got, want := reflect.ValueOf(holderFS.rename).Pointer(), reflect.ValueOf(os.Rename).Pointer(); got != want { + t.Fatal("holderFS.rename was not restored after the subtest's cleanup") + } + + // One seam, two goroutines: the ordinal must come from a counter that is + // safe to increment concurrently, or the racing rows this seam exists for + // report a race instead of a result. + t.Run("concurrently", func(t *testing.T) { + injectFault(t, "stat", nthCall(1), injected) + start := make(chan struct{}) + errs := make([]error, 2) + var wg sync.WaitGroup + for i := range errs { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + _, errs[i] = holderFS.stat(root) + }(i) + } + close(start) + wg.Wait() + failed := 0 + for _, err := range errs { + if errors.Is(err, injected) { + failed++ + } else if err != nil { + t.Fatalf("the call that was not selected = %v, want it to pass through", err) + } + } + if failed != 1 { + t.Fatalf("%d of 2 concurrent calls failed, want exactly 1", failed) + } + }) +} + +// A blocked step has to stay blocked: the racing rows park one transaction +// inside a step and only then let the second one reach the same destination. +func TestBlockStepHoldsTheCallUntilRelease(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "blocked") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + release := blockStep(t, "removeAll") + done := make(chan error, 1) + go func() { done <- holderFS.removeAll(dir) }() + select { + case err := <-done: + t.Fatalf("removeAll returned %v before release", err) + case <-time.After(20 * time.Millisecond): + } + release() + release() + if err := <-done; err != nil { + t.Fatalf("removeAll after release: %v", err) + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Fatalf("the released call did not run: %v", err) + } +} + +// ---- the per-destination Install lock --------------------------------------- + +// shortenInstallLockWait cuts the wait budget so a test can observe an expiry +// without spending the production two minutes on it. +func shortenInstallLockWait(t *testing.T, d time.Duration) { + t.Helper() + real := installLockWait + installLockWait = d + t.Cleanup(func() { installLockWait = real }) +} + +// gateRename holds the one rename from -> to until the returned release runs and +// then fails it. blockStep gates every call to a step, which cannot tell a +// promotion's set-aside rename from the publish that follows it, and this test +// needs the transaction parked precisely between the two. +func gateRename(t *testing.T, from, to string, fail error) (reached <-chan struct{}, release func()) { + t.Helper() + gate := make(chan struct{}) + hit := make(chan struct{}) + var once, announced sync.Once + release = func() { once.Do(func() { close(gate) }) } + real := holderFS + t.Cleanup(release) + t.Cleanup(func() { holderFS = real }) + holderFS.rename = func(f, to2 string) error { + if f != from || to2 != to { + return real.rename(f, to2) + } + announced.Do(func() { close(hit) }) + <-gate + return fail + } + return hit, release +} + +// A promotion moves a whole install out of the way and puts it back, so running +// one without the destination's Install lock is what lets a second process +// delete the copy this one is about to roll back to. A handle for a DIFFERENT +// destination is the same defect wearing a lock, which is why destDir stays an +// explicit parameter. +func TestPromoteStagedDirRefusesWithoutADestinationLock(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "model-a") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + if err := promoteStagedDir(nil, stage, dest, "engine", nil); err == nil { + t.Error("a promotion with no Install lock must be refused") + } + if err := promoteStagedDir(lockFor(t, filepath.Join(root, "model-b")), stage, dest, "engine", nil); err == nil { + t.Error("a promotion holding another destination's Install lock must be refused") + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "old" { + t.Errorf("the destination was touched without its lock: %q err %v", got, err) + } + got, err = os.ReadFile(filepath.Join(stage, "engine")) + if err != nil || string(got) != "new" { + t.Errorf("the staged copy was touched without the lock: %q err %v", got, err) + } +} + +// Recovery restores and deletes whole installs, so it fails closed the same way, +// and it says so: it has no error to return and a silent skip reads as "there +// was nothing to recover". +func TestRestoreInterruptedPromotionRefusesWithoutADestinationLock(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + holder := plantHolder(t, dest, 100, "the only copy", false) + + var reported []string + restoreInterruptedPromotion(nil, dest, testPublished, func(s string) { reported = append(reported, s) }) + if len(reported) == 0 { + t.Error("a recovery pass that refuses to run must say so") + } + restoreInterruptedPromotion(lockFor(t, filepath.Join(root, "engine-other")), dest, testPublished, nil) + + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Errorf("recovery ran without the destination's lock: %v", err) + } + got, err := os.ReadFile(filepath.Join(holder, "install", "engine")) + if err != nil || string(got) != "the only copy" { + t.Errorf("the holder was touched without the lock: %q err %v", got, err) + } +} + +// The download half fails closed too, or a caller that forgot the lock still +// reaches promoteStagedDir with a full extraction behind it. +func TestDownloadVerifyExtractRefusesWithoutADestinationLock(t *testing.T) { + srv := fakeReleaseServer(t, engineSHA, modelSHA) + root := t.TempDir() + dest := filepath.Join(root, "engine-test-linux-amd64") + // A real asset the download would otherwise install, so the refusal is what + // keeps the destination empty rather than a download that was going to fail. + asset, err := resolveAsset(context.Background(), http.DefaultClient, srv.URL, "test", "sherpa-onnx-", engineAssetSuffix["linux-amd64"]) + if err != nil { + t.Fatal(err) + } + + var requests atomic.Int64 + client := &http.Client{Transport: countingTransport(&requests)} + if err := downloadVerifyExtract(context.Background(), client, asset, "", false, "Engine", dest, nil, func(string) {}); err == nil { + t.Fatal("a download with no Install lock must be refused") + } + // Refused before the transfer, not after it: the promotion at the end would + // refuse too, and then a caller that forgot the lock still pays for the + // whole download every start. + if got := requests.Load(); got != 0 { + t.Errorf("%d asset request(s) were made without the Install lock, want 0", got) + } + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Errorf("the destination was installed to without its lock: %v", err) + } +} + +// countingTransport counts the asset transfers a client performs, which is how a +// test tells "refused before the download" from "refused after it". +func countingTransport(n *atomic.Int64) http.RoundTripper { + return roundTripFunc(func(r *http.Request) (*http.Response, error) { + n.Add(1) + return http.DefaultTransport.RoundTrip(r) + }) +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +// The lock is the cross-process one, so a second open of the same lock file has +// to conflict, and a caller has to wait the holder out rather than fail on the +// first contended try. +func TestLockDestinationSerializesAcrossProcesses(t *testing.T) { + root := t.TempDir() + lockDir := filepath.Join(root, installLockDir) + if err := os.MkdirAll(lockDir, 0o700); err != nil { + t.Fatal(err) + } + held, err := lockutil.TryAcquireFileLockAt(root, filepath.Join(lockDir, "engine-a.lock")) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + start := time.Now() + if _, err := lockDestination(ctx, root, "engine-a"); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("lockDestination = %v, want the context error while another holder has the lock", err) + } + if waited := time.Since(start); waited < 150*time.Millisecond { + t.Errorf("lockDestination gave up after %v; it must wait the holder out", waited) + } + + if err := held.Release(); err != nil { + t.Fatal(err) + } + txn, err := lockDestination(context.Background(), root, "engine-a") + if err != nil { + t.Fatalf("the lock was not free after the holder released it: %v", err) + } + txn.release() +} + +// A destination name is one path component. Anything else would put the lock +// file somewhere other than beside its peers, so two callers for the same +// destination could hold different inodes and both proceed. +func TestLockDestinationRefusesADestinationThatIsNotABaseName(t *testing.T) { + root := t.TempDir() + for _, dest := range []string{"", ".", "..", "a/b", filepath.Join("..", "escape")} { + if _, err := lockDestination(context.Background(), root, dest); err == nil { + t.Errorf("lockDestination(%q) must be refused", dest) + } + } +} + +// The likeliest reason a caller finds the destination locked is that another +// process is installing the very thing it wants. Failing on the first contended +// try turns a normal race into a user-visible install failure. +func TestEnsureLocalEngineWaitsForAConcurrentInstall(t *testing.T) { + srv := fakeReleaseServer(t, engineSHA, modelSHA) + root := t.TempDir() + txn := lockFor(t, filepath.Join(root, "engine-test-linux-amd64")) + go func() { + time.Sleep(150 * time.Millisecond) + txn.release() + }() + + comp, err := EnsureLocalEngine(context.Background(), DownloadOptions{ + DestRoot: root, EngineVersion: "test", APIBase: srv.URL, platformKey: "linux-amd64", skipPinned: true, + }) + if err != nil { + t.Fatalf("an install that waited out a concurrent one must succeed: %v", err) + } + if !fileExists(comp.BinaryPath) { + t.Errorf("engine binary missing at %q", comp.BinaryPath) + } +} + +// The review's P2: a promotion parked between its two renames has the only copy +// of the install in a holder and destDir absent. A recovery pass that runs in +// that window restores the holder and removes it, and the promotion's rollback +// then has nothing to put back. The lock is what makes that window unreachable. +func TestRecoveryDoesNotDeleteALiveRollbackSource(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "live") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + reached, release := gateRename(t, stage, dest, errors.New("injected publish failure")) + txn := lockFor(t, dest) + promoted := make(chan error, 1) + go func() { promoted <- promoteStagedDir(txn, stage, dest, "engine", nil) }() + select { + case <-reached: + case <-time.After(10 * time.Second): + t.Fatal("the promotion never reached its publish rename") + } + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Fatalf("the parked promotion should leave %s absent, got %v", dest, err) + } + holders := holdersFor(t, dest) + if len(holders) != 1 { + t.Fatalf("want exactly one holder beside %s, got %v", dest, holders) + } + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + err := withDestinationLock(ctx, root, filepath.Base(dest), testPublished, func(rec *destTxn) error { + restoreInterruptedPromotion(rec, dest, testPublished, nil) + return nil + }) + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("recovery = %v, want the context error while a promotion holds the destination", err) + } + got, readErr := os.ReadFile(filepath.Join(holders[0], "install", "engine")) + if readErr != nil || string(got) != "live" { + t.Fatalf("recovery took the copy the promotion still needs: %q err %v", got, readErr) + } + + release() + if err := <-promoted; err == nil { + t.Fatal("the injected publish failure must be reported") + } + got, readErr = os.ReadFile(filepath.Join(dest, "engine")) + if readErr != nil || string(got) != "live" { + t.Fatalf("the rollback did not put the previous install back: %q err %v", got, readErr) + } + txn.release() +} + +// The two destinations are locked one after the other. Taking both up front +// would let a model install someone else is running block the engine step, which +// is shared and usually already done. +func TestEnsureLocalEngineLocksEngineAndModelSeparately(t *testing.T) { + shortenInstallLockWait(t, 200*time.Millisecond) + srv := fakeReleaseServer(t, engineSHA, modelSHA) + root := t.TempDir() + modelTxn := lockFor(t, filepath.Join(root, "model-moonshine-tiny-en-int8")) + + _, err := EnsureLocalEngine(context.Background(), DownloadOptions{ + DestRoot: root, EngineVersion: "test", APIBase: srv.URL, platformKey: "linux-amd64", skipPinned: true, + }) + if !errors.Is(err, errInstallInProgress) { + t.Fatalf("EnsureLocalEngine = %v, want the model step to report an install in progress", err) + } + bin, _ := resolveEnginePaths(filepath.Join(root, "engine-test-linux-amd64"), false) + if !fileExists(bin) { + t.Errorf("the engine step must not wait on the model's lock: %s missing", bin) + } + modelTxn.release() +} + +// The lock directory sits beside the installs and holds nothing anyone else +// needs to read. +func TestLockDirectoryIsOwnerOnly(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Go permission bits do not map to Windows ACLs") + } + root := t.TempDir() + requireUmaskAllowsWiderThan0700(t, root) + lockFor(t, filepath.Join(root, "engine-a")) + + info, err := os.Stat(filepath.Join(root, installLockDir)) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o700 { + t.Errorf("%s mode = %o, want 0700", installLockDir, got) + } +} + +// DestRoot is created lazily by the download that follows, and lockutil opens +// the root with O_DIRECTORY, so a first run has no root to lock in. +func TestLockDestinationCreatesTheDestRootFirst(t *testing.T) { + root := filepath.Join(t.TempDir(), "stt", "nested") + txn, err := lockDestination(context.Background(), root, "engine-a") + if err != nil { + t.Fatalf("lockDestination on a DestRoot that does not exist yet: %v", err) + } + defer txn.release() + if _, err := os.Stat(filepath.Join(root, installLockDir, "engine-a.lock")); err != nil { + t.Errorf("the lock file should sit under the created root: %v", err) + } +} + +// A wait that runs out is not an install failure. The other process was almost +// certainly installing the same thing, so the destination is re-checked before +// anyone is told anything went wrong, and what comes back when it really is +// missing is a named outcome the caller can report rather than a raw timeout. +func TestEnsureLocalEngineTreatsAnExpiredWaitAsBenign(t *testing.T) { + shortenInstallLockWait(t, 100*time.Millisecond) + + t.Run("a usable destination means the other install finished", func(t *testing.T) { + root := t.TempDir() + engineDir := filepath.Join(root, "engine-test-linux-amd64") + bin, server := enginePaths(engineDir, false) + if err := os.MkdirAll(filepath.Dir(bin), 0o755); err != nil { + t.Fatal(err) + } + for _, p := range []string{bin, server} { + if err := os.WriteFile(p, []byte("planted"), 0o755); err != nil { + t.Fatal(err) + } + } + modelDir := filepath.Join(root, "model-moonshine-tiny-en-int8") + if err := os.MkdirAll(modelDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(modelDir, "tokens.txt"), []byte("planted"), 0o644); err != nil { + t.Fatal(err) + } + txn := lockFor(t, engineDir) + defer txn.release() + + // Nothing is listening on the API base, so any download attempt fails + // outright rather than quietly re-installing what is already there. + comp, err := EnsureLocalEngine(context.Background(), DownloadOptions{ + DestRoot: root, EngineVersion: "test", APIBase: offlineAPIBase(t), platformKey: "linux-amd64", skipPinned: true, + }) + if err != nil { + t.Fatalf("an expired wait over a usable destination must read as installed: %v", err) + } + if comp.BinaryPath != bin { + t.Errorf("BinaryPath = %q, want the install already at the destination %q", comp.BinaryPath, bin) + } + }) + + t.Run("an unusable destination is a named in-progress outcome", func(t *testing.T) { + root := t.TempDir() + engineDir := filepath.Join(root, "engine-test-linux-amd64") + txn := lockFor(t, engineDir) + defer txn.release() + + _, err := EnsureLocalEngine(context.Background(), DownloadOptions{ + DestRoot: root, EngineVersion: "test", APIBase: offlineAPIBase(t), platformKey: "linux-amd64", skipPinned: true, + }) + if !errors.Is(err, errInstallInProgress) { + t.Fatalf("EnsureLocalEngine = %v, want an error satisfying errInstallInProgress", err) + } + if _, err := os.Lstat(engineDir); !os.IsNotExist(err) { + t.Errorf("nothing should have been created for a destination that was never locked: %v", err) + } + }) +} + +// The grammar is the first ownership filter: a name that does not read back as +// one createSequencedHolder wrote is not this code's to move or delete. Loose +// parsing is what let a sibling merely NAMED like a holder be attributed to the +// install, which is the review's P3 finding on this site. +func TestHolderStampRequiresTheExactGrammar(t *testing.T) { + dest := filepath.Join(t.TempDir(), "engine-1.2.3-linux-x64") + base := filepath.Base(dest) + for _, tc := range []struct { + name string + want int64 + ok bool + }{ + {base + ".previous-00000000000000000042-seq", 42, true}, + {base + ".kept-00000000000000000042-seq", 42, true}, + {base + ".previous-42-seq", 0, false}, + {base + ".previous-00000000000000000042-x7Kq3", 0, false}, + {base + ".previous-1234567890", 0, false}, + {base + ".previous-00000000000000000042-seq-extra", 0, false}, + {base + ".previous-0000000000000000004a-seq", 0, false}, + {base + ".previous-notes", 0, false}, + {base + ".previous-", 0, false}, + {base + ".kept-42-seq", 0, false}, + {"other-install.previous-00000000000000000042-seq", 0, false}, + } { + t.Run(tc.name, func(t *testing.T) { + got, ok := holderStamp(dest, filepath.Join(filepath.Dir(dest), tc.name)) + if ok != tc.ok || got != tc.want { + t.Errorf("holderStamp(%q) = (%d, %v), want (%d, %v)", tc.name, got, ok, tc.want, tc.ok) + } + }) + } +} + +// The allocator counts the Kept prefix as well as the scanned one. Recovery +// parks a holder by renaming it under .kept-, and a sequence that stopped +// counting there would hand the next promotion a number a parked copy already +// carries, so the park would collide or the ordering would repeat. +func TestNextHolderSeqCountsKeptNames(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + kept := fmt.Sprintf("%s%s%020d%s", dest, keptSuffix, 9, holderSeqSuffix) + if err := os.MkdirAll(kept, 0o700); err != nil { + t.Fatal(err) + } + + seq, err := nextHolderSeq(dest) + if err != nil { + t.Fatal(err) + } + if seq != 10 { + t.Errorf("nextHolderSeq = %d, want 10: a parked copy already holds sequence 9", seq) + } +} + +// The review's P3 reproduction. A directory whose NAME collides with the holder +// prefix is not this install's copy, and attributing one by name is what put a +// user's unrelated directory in reach of recovery's moves and deletes. +func TestOwnedHoldersBesideIgnoresAPrefixCollidingSibling(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "model-a") + other := filepath.Join(root, "model-b") + + mine := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 1, holderSeqSuffix) + makeDir(t, mine) + if err := writeHolderMarker(mine, txnMarker{Kind: holderMarkerKind, Dest: "model-a", Seq: 1}); err != nil { + t.Fatal(err) + } + // Named for model-a, marked for model-b: evidence it belongs to another + // destination's transaction, so model-a's pass has no claim on it at all. + foreign := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 2, holderSeqSuffix) + makeDir(t, foreign) + if err := writeHolderMarker(foreign, txnMarker{Kind: holderMarkerKind, Dest: filepath.Base(other), Seq: 2}); err != nil { + t.Fatal(err) + } + // A name the grammar rejects: a user's own directory that happens to start + // the same way. + collide := dest + holderSuffix + "mine" + makeDir(t, collide) + // The grammar passes and no marker backs it: retained and reported, never + // restored and never deleted. + unmarked := fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 3, holderSeqSuffix) + makeDir(t, unmarked) + + owned, unowned, _ := ownedHoldersBeside(dest) + + gotOwned := make([]string, 0, len(owned)) + for _, c := range owned { + gotOwned = append(gotOwned, c.path) + } + if want := []string{mine}; !slices.Equal(gotOwned, want) { + t.Errorf("owned = %v, want %v", gotOwned, want) + } + if want := []string{unmarked}; !slices.Equal(unowned, want) { + t.Errorf("unowned = %v, want %v", unowned, want) + } + for _, path := range []string{foreign, collide} { + if slices.Contains(gotOwned, path) || slices.Contains(unowned, path) { + t.Errorf("%s is not this destination's to classify", filepath.Base(path)) + } + } + if len(owned) == 1 && owned[0].seq != 1 { + t.Errorf("owned seq = %d, want 1", owned[0].seq) + } +} + +// makeDir creates one directory the fixtures need, with the mode the allocator +// gives a holder. +func makeDir(t *testing.T, path string) { + t.Helper() + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } +} + +// Parking is how a copy recovery will not restore and cannot prove superseded +// leaves the scan without leaving the disk: the same directory under a prefix +// the scan does not enumerate, keeping its sequence so the operator can still +// name it. +func TestParkKeptHolderMovesTheCopyUnderTheKeptPrefix(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + holder := plantHolder(t, dest, 7, "kept", false) + + if err := parkKeptHolder(holder); err != nil { + t.Fatalf("park: %v", err) + } + kept := fmt.Sprintf("%s%s%020d%s", dest, keptSuffix, 7, holderSeqSuffix) + got, err := os.ReadFile(filepath.Join(kept, "install", "engine")) + if err != nil || string(got) != "kept" { + t.Fatalf("parked copy = %q (err %v), want %q under %s", got, err, "kept", filepath.Base(kept)) + } + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("the holder should be gone from the scanned prefix, got %v", err) + } + owned, unowned, _ := ownedHoldersBeside(dest) + if len(owned) != 0 || len(unowned) != 0 { + t.Errorf("a parked copy must leave the scan: owned %v unowned %v", owned, unowned) + } +} + +// A park that would have to clear something first is a park onto a copy that is +// not ours to remove, so it fails and leaves both where they are. Only a +// clobbering implementation (remove the destination, then rename) can lose the +// occupant. +func TestParkKeptHolderDoesNotClobber(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + holder := plantHolder(t, dest, 7, "new", false) + occupied := fmt.Sprintf("%s%s%020d%s", dest, keptSuffix, 7, holderSeqSuffix) + makeDir(t, filepath.Join(occupied, "install")) + if err := os.WriteFile(filepath.Join(occupied, "install", "engine"), []byte("older"), 0o644); err != nil { + t.Fatal(err) + } + + if err := parkKeptHolder(holder); err == nil { + t.Error("parking onto an occupied Kept name must fail") + } + got, err := os.ReadFile(filepath.Join(occupied, "install", "engine")) + if err != nil || string(got) != "older" { + t.Errorf("the occupant was clobbered: %q err %v", got, err) + } + if _, err := os.Stat(filepath.Join(holder, "install", "engine")); err != nil { + t.Errorf("a failed park must leave the holder where it is: %v", err) + } +} + +// soleHolder returns the one holder beside destDir, failing when there is not +// exactly one. +func soleHolder(t *testing.T, destDir string) string { + t.Helper() + holders := holdersFor(t, destDir) + if len(holders) != 1 { + t.Fatalf("want exactly one holder beside %s, got %v", destDir, holders) + } + return holders[0] +} + +// hasFile reports whether path exists. +func hasFile(t *testing.T, path string) bool { + t.Helper() + _, err := os.Lstat(path) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + return err == nil +} + +// The marker goes in before the first destructive rename. A holder that takes +// the only copy of an install before it can be proven ours is one recovery has +// to leave in place forever, since nothing on disk says who wrote it. +func TestPromoteStagedDirWritesTheMarkerBeforeSettingAside(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + injectFault(t, "rename", func(args ...string) bool { + return filepath.Base(args[1]) == "install" + }, errors.New("injected set-aside failure")) + // The holder is cleaned up on this failure path, which is right and would + // also erase what this test is asserting on. + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil); err == nil { + t.Fatal("a failed set-aside must be reported") + } + holder := soleHolder(t, dest) + m, err := readHolderMarker(holder) + if err != nil { + t.Fatalf("the holder was filled before it carried a marker: %v", err) + } + seq, ok := holderStamp(dest, holder) + if !ok { + t.Fatalf("the allocator wrote a name the grammar rejects: %q", filepath.Base(holder)) + } + if m.Kind != holderMarkerKind || m.Dest != filepath.Base(dest) || m.Seq != seq { + t.Errorf("marker = %+v, want kind %q dest %q seq %d", m, holderMarkerKind, filepath.Base(dest), seq) + } + if hasFile(t, filepath.Join(holder, "install")) { + t.Error("the set-aside failed, so nothing should have moved into the holder") + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "old" { + t.Errorf("the previous install must be untouched: %q err %v", got, err) + } +} + +// The same ordering on recovery's own set-aside. Only promoteStagedDir's copy of +// this window was pinned, and the recovery path writes its own: a husk moved +// into a holder before the marker lands is the only copy of whatever was at the +// destination, sitting in a directory nothing on disk attributes, which recovery +// can then neither restore from nor ever reclaim. +func TestSetAsideUnusableDestWritesTheMarkerBeforeMovingTheHusk(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + // A destination that exists and holds no usable install, plus a usable copy + // beside it: the one state that reaches the set-aside, since the husk moves + // only after a candidate has been selected. + if err := os.MkdirAll(dest, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dest, "husk"), []byte("partial"), 0o644); err != nil { + t.Fatal(err) + } + candidate := plantHolder(t, dest, 1, "the copy", false) + + injectFault(t, "rename", func(args ...string) bool { + return args[0] == dest + }, errors.New("injected set-aside failure")) + // The holder is removed on this failure path, which is right and would also + // erase what this test is asserting on. + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) + + var husk string + for _, h := range holdersFor(t, dest) { + if h != candidate { + if husk != "" { + t.Fatalf("want one holder beside the candidate, got %v", holdersFor(t, dest)) + } + husk = h + } + } + if husk == "" { + t.Fatal("the set-aside allocated no holder, so this test proves nothing about the order") + } + m, err := readHolderMarker(husk) + if err != nil { + t.Fatalf("the holder was moved into before it carried a marker: %v", err) + } + seq, ok := holderStamp(dest, husk) + if !ok { + t.Fatalf("the allocator wrote a name the grammar rejects: %q", filepath.Base(husk)) + } + if m.Kind != holderMarkerKind || m.Dest != filepath.Base(dest) || m.Seq != seq { + t.Errorf("marker = %+v, want kind %q dest %q seq %d", m, holderMarkerKind, filepath.Base(dest), seq) + } + if hasFile(t, filepath.Join(husk, "install")) { + t.Error("the set-aside failed, so nothing should have moved into the holder") + } + if got, err := os.ReadFile(filepath.Join(dest, "husk")); err != nil || string(got) != "partial" { + t.Errorf("the destination must be left exactly as found: %q err %v", got, err) + } +} + +// A torn marker write must leave no marker at all: the atomic rename is what +// makes "missing" and "ours" the only two answers a crash can produce, so a +// partial file can never be read as ownership. +func TestPromoteStagedDirMarkerWriteIsAtomic(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + injectFault(t, "rename", func(args ...string) bool { + return filepath.Base(args[1]) == holderMarkerFile + }, errors.New("injected marker publish failure")) + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil); err == nil { + t.Fatal("a failed marker write must be reported") + } + holder := soleHolder(t, dest) + if hasFile(t, filepath.Join(holder, holderMarkerFile)) { + t.Error("a marker that was never published must not be readable") + } + if _, err := readHolderMarker(holder); !errors.Is(err, errMarkerMissing) { + t.Errorf("readHolderMarker = %v, want errMarkerMissing", err) + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "old" { + t.Errorf("the previous install must be untouched: %q err %v", got, err) + } +} + +// The commit flag is what proves to a later pass that the copy in the holder was +// superseded by a publish that actually landed. It goes in after the publish and +// before the cleanup, so a crash in that window costs a retained copy rather +// than the install. +func TestPromoteStagedDirCreatesTheCommitFlagAfterPublish(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + // Fail only the final cleanup, so the holder the flag was written into is + // still there to assert on. + injectFault(t, "removeAll", nil, errors.New("injected cleanup failure")) + + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil); err != nil { + t.Fatalf("promote: %v", err) + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "new" { + t.Fatalf("engine = %q err %v, want %q", got, err, "new") + } + holder := soleHolder(t, dest) + if !hasFile(t, filepath.Join(holder, committedFile)) { + t.Error("a published promotion must mark the copy it superseded") + } + if kept, err := os.ReadFile(filepath.Join(holder, "install", "engine")); err != nil || string(kept) != "old" { + t.Errorf("the superseded copy = %q err %v, want %q", kept, err, "old") + } +} + +// A publish that never happened must leave no evidence that it did. The flag is +// the only thing licensing a later pass to delete the copy in the holder, so a +// flag beside a copy that was never superseded authorizes deleting the only +// install the user has. Both renames into the destination fail, which is the +// state a crash mid-publish leaves: the holder retained, holding everything. +func TestPromoteStagedDirPublishFailureLeavesNoCommitFlag(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + injectFault(t, "rename", func(args ...string) bool { + return args[1] == dest + }, errors.New("injected publish failure")) + + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", nil); err == nil { + t.Fatal("a failed publish must be reported") + } + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Fatalf("neither rename into %s succeeded, so it should be absent, got %v", dest, err) + } + holder := soleHolder(t, dest) + if kept, err := os.ReadFile(filepath.Join(holder, "install", "engine")); err != nil || string(kept) != "old" { + t.Fatalf("the only copy = %q err %v, want it retained as %q", kept, err, "old") + } + if found := findNamed(t, root, committedFile); len(found) != 0 { + t.Errorf("a promotion that never published must leave no commit flag, found %v", found) + } +} + +// A commit flag that cannot be written leaves the superseded copy on disk. The +// alternative is deleting a whole install on the word of a step that just +// failed, and the report is what tells the user where the copy went. +func TestPromoteStagedDirFailedCommitFlagKeepsTheHolder(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "old") + stage := stagedTree(t, filepath.Join(root, "stage"), "new") + + injectFault(t, "create", func(args ...string) bool { + return filepath.Base(args[0]) == committedFile + }, errors.New("injected commit flag failure")) + var reported []string + report := func(line string) { reported = append(reported, line) } + + if err := promoteStagedDir(lockFor(t, dest), stage, dest, "engine", report); err != nil { + t.Fatalf("a published install must not be reported as a failure: %v", err) + } + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "new" { + t.Fatalf("engine = %q err %v, want the published %q", got, err, "new") + } + holder := soleHolder(t, dest) + if kept, err := os.ReadFile(filepath.Join(holder, "install", "engine")); err != nil || string(kept) != "old" { + t.Errorf("the superseded copy = %q err %v, want it retained as %q", kept, err, "old") + } + if hasFile(t, filepath.Join(holder, committedFile)) { + t.Error("the flag write failed, so no flag should exist") + } + if !slices.ContainsFunc(reported, func(line string) bool { return strings.Contains(line, holder) }) { + t.Errorf("the report must name the retained copy, got %v", reported) + } +} + +// findNamed lists every path under root whose base name is name. +func findNamed(t *testing.T, root, name string) []string { + t.Helper() + var found []string + if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.Name() == name { + found = append(found, path) + } + return nil + }); err != nil { + t.Fatal(err) + } + return found +} + +// ---- recovery reconciliation ---------------------------------------------- + +// Which copy is NEWEST and which copy is USABLE are different questions, and the +// review's P2 is what happens when the code answers only the first: a stop +// between the holder's creation and the set-aside leaves a newer holder whose +// install directory is there and empty, and restoring from it publishes nothing +// over a destination that has nothing either. The caller's predicate is what +// decides, and it has to reach every candidate, not just the winner. +func TestRestoreInterruptedPromotionAppliesThePredicateToEveryHolder(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + usable := plantHolder(t, dest, 1, "the only copy", false) + // Newer, owned, and holding an install directory with nothing in it: the + // exact shape a Stat-only check reads as the copy to restore. + newer := plantHolder(t, dest, 2, "", false) + if err := os.Remove(filepath.Join(newer, "install", "engine")); err != nil { + t.Fatal(err) + } + + var reported []string + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, reporterFor(&reported)) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "the only copy" { + t.Fatalf("recovery published a copy its own caller cannot use: %q err %v", got, err) + } + if _, err := os.Stat(usable); !os.IsNotExist(err) { + t.Errorf("the restored copy's holder should be cleared, got %v", err) + } + parked := keptName(t, dest, newer) + if _, err := os.Stat(parked); err != nil { + t.Errorf("the unusable copy should be kept under the Kept prefix: %v", err) + } + assertReports(t, reported, parked) +} + +// Falling back to an older copy after the newest one could not be restored is +// what manufactures the provenance loss: an older tree lands at the destination +// and the next pass reads it as evidence that the newer copy was superseded. So +// a failed restore stops the destination, keeps every copy, and says so. +func TestRestoreInterruptedPromotionStopsWhenTheNewestRestoreFails(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + older := plantHolder(t, dest, 1, "older", false) + newest := plantHolder(t, dest, 2, "newest", false) + txn := lockFor(t, dest) + + // The fault belongs to the two failing passes only, so the third pass runs + // against a real filesystem and shows the retry the report promises. + func() { + saved := holderFS + defer func() { holderFS = saved }() + injectFault(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(newest, "install") + }, errors.New("injected restore failure")) + + for pass := 1; pass <= 2; pass++ { + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Fatalf("pass %d: an older copy was installed after the newest failed: %v", pass, err) + } + for _, holder := range []string{older, newest} { + if _, err := os.Stat(filepath.Join(holder, "install", "engine")); err != nil { + t.Errorf("pass %d: every copy must be kept where it is: %v", pass, err) + } + } + assertReports(t, reported, newest) + } + }() + + restoreInterruptedPromotion(txn, dest, testPublished, nil) + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "newest" { + t.Fatalf("the pass after the fault cleared should restore the newest copy: %q err %v", got, err) + } + if _, err := os.Stat(filepath.Join(keptName(t, dest, older), "install", "engine")); err != nil { + t.Errorf("the older copy must be kept: %v", err) + } +} + +// The review's P2 chain end to end: recovery leaves a partial tree at the +// destination, the caller's predicate rejects it, and an offline start then has +// no engine while a usable copy sits beside it. The exit is to set the husk +// aside and restore the copy, which costs one Kept backup and leaves the user +// with a working install instead of a permanent stuck state. +func TestRestoreInterruptedPromotionExitsTheStuckState(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + plantUnusableDest(t, dest) + holder := plantHolder(t, dest, 1, "the only copy", false) + txn := lockFor(t, dest) + + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "the only copy" { + t.Fatalf("the usable copy should be live at the destination: %q err %v", got, err) + } + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("the restored copy's holder should be cleared, got %v", err) + } + // The husk moves into a FRESH sequenced holder, above the one the copy came + // out of, so holders never nest and the operator can name it. + husk := filepath.Join(keptName(t, dest, fmt.Sprintf("%s%s%020d%s", dest, holderSuffix, 2, holderSeqSuffix)), "install") + if _, err := os.Stat(filepath.Join(husk, "bin", "README")); err != nil { + t.Errorf("the husk must be kept, not deleted: %v", err) + } + assertReports(t, reported, filepath.Dir(husk)) + + // No memory: the destination is usable now and the husk is under the Kept + // prefix, so there is nothing left to rule on. + reported = nil + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + if got, err := os.ReadFile(filepath.Join(dest, "engine")); err != nil || string(got) != "the only copy" { + t.Errorf("the second pass changed the live install: %q err %v", got, err) + } + if len(reported) != 0 { + t.Errorf("the second pass should have nothing to report, got %v", reported) + } + + // A later real install sets the now-usable destination aside as an ordinary + // holder and the Kept husk is still where recovery put it. + stage := stagedTree(t, filepath.Join(root, "stage"), "newest") + if err := promoteStagedDir(txn, stage, dest, "engine", nil); err != nil { + t.Fatalf("the later install failed: %v", err) + } + if _, err := os.Stat(filepath.Join(husk, "bin", "README")); err != nil { + t.Errorf("a later install must not disturb a Kept backup: %v", err) + } +} + +// An unusable destination with nothing usable beside it is not a state recovery +// can improve, and moving the husk out of the way anyway would leave the caller +// with no destination at all. So the destination is left exactly as found and +// only the report changes. +func TestRestoreInterruptedPromotionLeavesAnUnusableDestinationWithNoCandidateAlone(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + plantUnusableDest(t, dest) + // Owned, uncommitted, and holding an install nothing can use. + unusable := plantHolder(t, dest, 1, "", false) + if err := os.Remove(filepath.Join(unusable, "install", "engine")); err != nil { + t.Fatal(err) + } + unowned := plantUnowned(t, dest, 2, "not ours") + txn := lockFor(t, dest) + + for pass := 1; pass <= 2; pass++ { + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + + if _, err := os.Stat(filepath.Join(dest, "bin", "README")); err != nil { + t.Fatalf("pass %d: the destination must be left exactly as found: %v", pass, err) + } + if testPublished(dest) { + t.Fatalf("pass %d: nothing was restored, so the destination cannot have become usable", pass) + } + parked := keptName(t, dest, unusable) + if _, err := os.Stat(filepath.Join(parked, "install")); err != nil { + t.Errorf("pass %d: the unusable copy should be kept under the Kept prefix: %v", pass, err) + } + if _, err := os.Stat(filepath.Join(unowned, "install", "engine")); err != nil { + t.Errorf("pass %d: an unowned directory is never moved: %v", pass, err) + } + if pass == 1 { + assertReports(t, reported, parked, unowned, dest) + } else { + assertReports(t, reported, unowned, dest) + } + } +} + +// The husk moves only after a candidate is chosen, and it moves BACK when the +// restore fails. Parking it first would strand the destination's own contents +// under a prefix recovery never enumerates, which is the one outcome worse than +// the stuck state this branch exists to exit. +func TestRestoreInterruptedPromotionPutsTheHuskBackWhenTheRestoreFails(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + plantUnusableDest(t, dest) + holder := plantHolder(t, dest, 1, "the only copy", false) + txn := lockFor(t, dest) + + func() { + saved := holderFS + defer func() { holderFS = saved }() + injectFault(t, "rename", func(args ...string) bool { + return args[0] == filepath.Join(holder, "install") + }, errors.New("injected restore failure")) + + for pass := 1; pass <= 2; pass++ { + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + + if _, err := os.Stat(filepath.Join(dest, "bin", "README")); err != nil { + t.Fatalf("pass %d: the destination must be left exactly as found: %v", pass, err) + } + if _, err := os.Stat(filepath.Join(holder, "install", "engine")); err != nil { + t.Errorf("pass %d: the copy must stay at its own name: %v", pass, err) + } + kept, err := filepath.Glob(dest + keptSuffix + "*") + if err != nil { + t.Fatal(err) + } + if len(kept) != 0 { + t.Errorf("pass %d: a failed restore must leave no Kept backup, got %v", pass, kept) + } + assertReports(t, reported, holder) + } + }() + + restoreInterruptedPromotion(txn, dest, testPublished, nil) + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "the only copy" { + t.Fatalf("the pass after the fault cleared should restore the copy: %q err %v", got, err) + } + kept, err := filepath.Glob(dest + keptSuffix + "*") + if err != nil { + t.Fatal(err) + } + if len(kept) != 1 { + t.Fatalf("the husk should be kept exactly once, got %v", kept) + } + if _, err := os.Stat(filepath.Join(kept[0], "install", "bin", "README")); err != nil { + t.Errorf("the husk must be kept intact: %v", err) + } +} + +// A committed copy is only provably superseded by a destination that is present +// AND usable. With no such destination it is the last usable copy there is, so +// it is the fallback selection rather than something to reclaim. +func TestRestoreInterruptedPromotionRestoresACommittedHolderOverAnUnusableDestination(t *testing.T) { + for _, tc := range []struct { + name string + seedDest func(t *testing.T, dest string) + wantHusk bool + }{ + { + name: "unusable destination", + seedDest: plantUnusableDest, + wantHusk: true, + }, + { + name: "absent destination", + seedDest: func(t *testing.T, dest string) {}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + tc.seedDest(t, dest) + holder := plantHolder(t, dest, 1, "the only copy", true) + + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, nil) + + got, err := os.ReadFile(filepath.Join(dest, "engine")) + if err != nil || string(got) != "the only copy" { + t.Fatalf("the last usable copy should be live: %q err %v", got, err) + } + if _, err := os.Stat(holder); !os.IsNotExist(err) { + t.Errorf("the restored copy's holder should be cleared, got %v", err) + } + kept, err := filepath.Glob(dest + keptSuffix + "*") + if err != nil { + t.Fatal(err) + } + if tc.wantHusk { + if len(kept) != 1 { + t.Fatalf("the husk should be kept exactly once, got %v", kept) + } + if _, err := os.Stat(filepath.Join(kept[0], "install", "bin", "README")); err != nil { + t.Errorf("the husk must be kept intact: %v", err) + } + return + } + if len(kept) != 0 { + t.Errorf("nothing was set aside, so there is no Kept backup to write: %v", kept) + } + }) + } +} + +// A directory that merely starts like a holder name was never this install's, +// so it is neither acted on nor reported on this install's account. Reporting it +// would be as wrong as moving it: it tells the operator a copy of their install +// is somewhere it is not. +func TestRestoreInterruptedPromotionIgnoresAPrefixCollidingSibling(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "live") + sibling := dest + holderSuffix + "notes" + if err := os.MkdirAll(sibling, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(sibling, "note"), []byte("mine"), 0o644); err != nil { + t.Fatal(err) + } + committed := plantHolder(t, dest, 1, "old", true) + + var reported []string + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, reporterFor(&reported)) + + if got, err := os.ReadFile(filepath.Join(sibling, "note")); err != nil || string(got) != "mine" { + t.Errorf("a prefix-colliding sibling must be left exactly as found: %q err %v", got, err) + } + if _, err := os.Stat(committed); !os.IsNotExist(err) { + t.Errorf("the superseded copy should be reaped, got %v", err) + } + for _, m := range reported { + if strings.Contains(m, sibling) { + t.Errorf("a sibling that is not this install's must not be reported on its account: %q", m) + } + } +} + +// Unusable and unreadable are different answers. Unusable is durable and the +// candidate is skipped and kept; unreadable is a filesystem fault, and deciding +// anything on it would turn a transient error into a permanent ruling. So a +// candidate that cannot be read stops the destination with everything intact. +func TestRestoreInterruptedPromotionStopsOnAnUnreadableHolder(t *testing.T) { + for _, tc := range []struct { + name string + // seed makes the holder's install unreadable without touching the holder + // itself: chmod the holder and the MARKER stops being readable too, the + // directory is classified unowned, and the branch under test is never + // reached. + seed func(t *testing.T, holder string) + // portable rows run everywhere. The chmod rows do not, and this + // distinction is the axis the no-fallback rule rests on, so at least one + // row has to hold on the Windows leg and under root. + portable bool + }{ + { + name: "the install cannot be listed at the seam", + portable: true, + seed: func(t *testing.T, holder string) { + injectFault(t, "readDir", func(args ...string) bool { + return args[0] == filepath.Join(holder, "install") + }, os.ErrPermission) + }, + }, + { + name: "install is a symlink through an unreadable directory", + seed: func(t *testing.T, holder string) { + t.Helper() + locked := filepath.Join(holder, "locked") + if err := os.Rename(filepath.Join(holder, "install"), locked); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(locked, "inner"), filepath.Join(holder, "install")); err != nil { + t.Fatal(err) + } + if err := os.Chmod(locked, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(locked, 0o700) }) + }, + }, + { + name: "install itself cannot be read", + seed: func(t *testing.T, holder string) { + t.Helper() + install := filepath.Join(holder, "install") + if err := os.Chmod(install, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(install, 0o700) }) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if !tc.portable { + if os.Geteuid() == 0 { + t.Skip("root ignores the directory permissions this row relies on") + } + if runtime.GOOS == "windows" { + t.Skip("POSIX directory permissions") + } + } + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + // A second usable owned holder sits below the unreadable one, and it + // is what makes "stopped" distinguishable from "ran out of + // candidates": with the unreadable holder as the only candidate, + // turning the stop into a continue leaves every assertion below + // true, because there is nothing left for the pass to publish or + // park either way. + older := plantHolder(t, dest, 1, "the older copy", false) + holder := plantHolder(t, dest, 2, "the only copy", false) + tc.seed(t, holder) + + var reported []string + // The delete is recorded rather than inferred from the disk: a + // holder whose own contents cannot be read cannot be removed + // either, so a pass that classified it as empty and ASKED for the + // delete leaves exactly the same directory behind as one that + // never touched it. + removed := recordRemoveAll(t) + restoreInterruptedPromotion(lockFor(t, dest), dest, testPublished, reporterFor(&reported)) + + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Errorf("nothing readable was found, so nothing may be published: %v", err) + } + if _, err := os.Lstat(holder); err != nil { + t.Errorf("a copy that could not be read must be left where it is: %v", err) + } + if slices.Contains(*removed, holder) { + t.Errorf("a copy that could not be read must never be handed to a delete: %v", *removed) + } + // The older copy is the claim the fixture exists for: a pass that + // carried on past the unreadable holder would publish this one, or + // park it on the way, and either is a ruling made on a fault. + if got, err := os.ReadFile(filepath.Join(older, "install", "engine")); err != nil || string(got) != "the older copy" { + t.Errorf("the older copy must be left exactly as found: %q err %v", got, err) + } + if slices.Contains(*removed, older) { + t.Errorf("the older copy must never be handed to a delete: %v", *removed) + } + assertNoOtherEntries(t, root, filepath.Base(older), filepath.Base(holder), installLockDir) + assertReports(t, reported, holder) + }) + } +} + +// A directory carrying the exact generated name with no marker behind it is one +// this code cannot claim. Restoring from it would publish a stranger's tree as +// the user's install, and deleting it would destroy something that was never +// ours, so it is kept where it is and named. +func TestRestoreInterruptedPromotionRetainsUnownedHolders(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + unowned := plantUnowned(t, dest, 1, "not ours") + txn := lockFor(t, dest) + + for pass := 1; pass <= 2; pass++ { + var reported []string + restoreInterruptedPromotion(txn, dest, testPublished, reporterFor(&reported)) + + if _, err := os.Lstat(dest); !os.IsNotExist(err) { + t.Errorf("pass %d: an unowned directory must never be restored: %v", pass, err) + } + if got, err := os.ReadFile(filepath.Join(unowned, "install", "engine")); err != nil || string(got) != "not ours" { + t.Errorf("pass %d: an unowned directory must be left exactly as found: %q err %v", pass, got, err) + } + assertReports(t, reported, unowned) + } +} + +// The predicate is what tells a published install from a directory that merely +// exists. Without one there is no such evidence, and every ruling recovery could +// make rests on it, so a caller that supplies none gets no mutation at all. +func TestRestoreInterruptedPromotionNilPredicateNeverDeletesOrRestores(t *testing.T) { + root := t.TempDir() + dest := filepath.Join(root, "engine-1.2.3-linux-x64") + stagedTree(t, dest, "live") + committed := plantHolder(t, dest, 1, "old", true) + + restoreInterruptedPromotion(lockFor(t, dest), dest, nil, nil) + + if got, err := os.ReadFile(filepath.Join(dest, "engine")); err != nil || string(got) != "live" { + t.Errorf("the destination must be left alone: %q err %v", got, err) + } + if got, err := os.ReadFile(filepath.Join(committed, "install", "engine")); err != nil || string(got) != "old" { + t.Errorf("with no predicate nothing is proven superseded: %q err %v", got, err) + } + assertNoOtherEntries(t, root, filepath.Base(dest), filepath.Base(committed), installLockDir) +} + +// recordRemoveAll collects every path recovery asks the seam to delete, and +// passes each call through. It is how a test tells "the delete was refused by +// the filesystem" from "the delete was never asked for", which the directory +// left on disk cannot show. +func recordRemoveAll(t *testing.T) *[]string { + t.Helper() + var paths []string + real := holderFS + t.Cleanup(func() { holderFS = real }) + holderFS.removeAll = func(path string) error { + paths = append(paths, path) + return real.removeAll(path) + } + return &paths +} + +// ---- the reclaim surface --------------------------------------------------- + +// plantKeptHolder writes what recovery leaves under the Kept prefix: a holder, +// named and marked the way promoteStagedDir writes one, renamed to the Kept name +// recovery parks it under. Going through the real helpers is what keeps the +// fixture from testing a shape production never writes. +func plantKeptHolder(t *testing.T, destDir string, seq int64, content string) string { + t.Helper() + holder := plantHolder(t, destDir, seq, content, false) + kept := keptName(t, destDir, holder) + if err := os.Rename(holder, kept); err != nil { + t.Fatal(err) + } + return kept +} + +// keptHolderBytes is what the listing should report for a backup +// plantKeptHolder wrote: the marker and the one file in the copy. Summed from +// the two names the fixture created rather than by walking, so the assertion is +// not the implementation restated. +func keptHolderBytes(t *testing.T, kept, content string) int64 { + t.Helper() + info, err := os.Stat(filepath.Join(kept, holderMarkerFile)) + if err != nil { + t.Fatal(err) + } + return info.Size() + int64(len(content)) +} + +func findKept(t *testing.T, list []KeptBackup, path string) KeptBackup { + t.Helper() + for _, b := range list { + if b.Path == path { + return b + } + } + t.Fatalf("%s is missing from the listing %v", path, list) + return KeptBackup{} +} + +// A Kept backup an operator cannot find is one that can never be reclaimed, and +// this listing is the only thing that finds them: recovery deliberately never +// enumerates the prefix. Here the copy is the only offline copy of an engine or +// a model, so a directory nothing attributes is reported unowned rather than +// dropped: the operator has to be able to see recovery's residue. +func TestListKeptBackupsReportsDestSeqAndSize(t *testing.T) { + root := t.TempDir() + engine := filepath.Join(root, "engine-a") + model := filepath.Join(root, "model-b") + first := plantKeptHolder(t, engine, 1, "engine-bytes") + second := plantKeptHolder(t, model, 2, "model") + + // Kept grammar, nothing attributing it: the shape a crash between the mkdir + // and the marker write leaves, parked by a later pass. + bare := fmt.Sprintf("%s%s%020d%s", engine, keptSuffix, 3, holderSeqSuffix) + if err := os.Mkdir(bare, 0o755); err != nil { + t.Fatal(err) + } + // A marker naming another destination is evidence this copy belongs to a + // transaction that is not the one the name claims. + skewed := plantKeptHolder(t, engine, 4, "skew") + if err := writeHolderMarker(skewed, txnMarker{Kind: holderMarkerKind, Dest: "engine-z", Seq: 4}); err != nil { + t.Fatal(err) + } + // A sibling that fails the grammar was never a holder and is not listed. + if err := os.Mkdir(engine+keptSuffix+"notanumber"+holderSeqSuffix, 0o755); err != nil { + t.Fatal(err) + } + + list, err := ListKeptBackups(root) + if err != nil { + t.Fatalf("ListKeptBackups: %v", err) + } + if len(list) != 4 { + t.Fatalf("listed %d kept backups, want 4: %v", len(list), list) + } + + got := findKept(t, list, first) + if !got.Owned || got.Dest != "engine-a" || got.Seq != 1 || got.Bytes != keptHolderBytes(t, first, "engine-bytes") { + t.Errorf("first = %+v, want owned engine-a seq 1 bytes %d", got, keptHolderBytes(t, first, "engine-bytes")) + } + got = findKept(t, list, second) + if !got.Owned || got.Dest != "model-b" || got.Seq != 2 || got.Bytes != keptHolderBytes(t, second, "model") { + t.Errorf("second = %+v, want owned model-b seq 2 bytes %d", got, keptHolderBytes(t, second, "model")) + } + for _, path := range []string{bare, skewed} { + got := findKept(t, list, path) + if got.Owned { + t.Errorf("%s is attributed by nothing on disk and must be listed unowned, got %+v", path, got) + } + // The destination has to be empty too: naming one off the directory's + // own name tells the operator this copy belongs to an install the + // marker does not support. + if got.Dest != "" { + t.Errorf("%s: unowned entries carry no destination, got %q", path, got.Dest) + } + } +} + +// Removal carries the same ownership proof recovery's own deletes carry. This is +// the one thing that deletes a Kept backup here, and the copy it deletes may be +// the only offline copy of an engine, so an entry nothing attributes survives it +// and the operator is told to remove that one by hand. +func TestRemoveKeptBackupRefusesAnUnownedEntry(t *testing.T) { + t.Run("no marker", func(t *testing.T) { + root := t.TempDir() + kept := plantKeptHolder(t, filepath.Join(root, "engine-a"), 1, "engine") + if err := os.Remove(filepath.Join(kept, holderMarkerFile)); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(root, filepath.Base(kept)); err == nil { + t.Fatal("a directory with no marker must not be removed") + } + if _, err := os.Stat(filepath.Join(kept, "install", "engine")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) + t.Run("marker disagrees with the sequence", func(t *testing.T) { + root := t.TempDir() + kept := plantKeptHolder(t, filepath.Join(root, "engine-a"), 1, "engine") + if err := writeHolderMarker(kept, txnMarker{Kind: holderMarkerKind, Dest: "engine-a", Seq: 42}); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(root, filepath.Base(kept)); err == nil { + t.Fatal("a marker for another sequence must not license a removal") + } + if _, err := os.Stat(filepath.Join(kept, "install", "engine")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) + t.Run("marker disagrees with the destination", func(t *testing.T) { + root := t.TempDir() + kept := plantKeptHolder(t, filepath.Join(root, "engine-a"), 1, "engine") + if err := writeHolderMarker(kept, txnMarker{Kind: holderMarkerKind, Dest: "engine-z", Seq: 1}); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(root, filepath.Base(kept)); err == nil { + t.Fatal("a marker for another destination must not license a removal") + } + if _, err := os.Stat(filepath.Join(kept, "install", "engine")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) +} + +// The lock is what separates a Kept backup from a copy a live install is about +// to roll back to: on disk the two are the same directory, and removing one +// mid-promotion takes the only copy of the install. +func TestRemoveKeptBackupRefusesWhileTheDestinationIsLocked(t *testing.T) { + root := t.TempDir() + kept := plantKeptHolder(t, filepath.Join(root, "engine-a"), 1, "engine") + lockDir := filepath.Join(root, installLockDir) + if err := os.MkdirAll(lockDir, 0o700); err != nil { + t.Fatal(err) + } + held, err := lockutil.TryAcquireFileLockAt(root, filepath.Join(lockDir, "engine-a.lock")) + if err != nil { + t.Fatalf("take the lock as the other install would: %v", err) + } + if err := RemoveKeptBackup(root, filepath.Base(kept)); err == nil { + _ = held.Release() + t.Fatal("a removal must not run while another process holds the destination") + } + if _, err := os.Stat(filepath.Join(kept, "install", "engine")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + if err := held.Release(); err != nil { + t.Fatal(err) + } + if err := RemoveKeptBackup(root, filepath.Base(kept)); err != nil { + t.Fatalf("the same removal must go through once the lock is free: %v", err) + } + if _, err := os.Stat(kept); !os.IsNotExist(err) { + t.Errorf("the copy should be gone, got %v", err) + } +} + +// The attribution runs twice: once before the lock and once under it. The first +// check happens with nothing excluding a live promotion, so a directory that +// stopped being this install's in between must not be deleted on the strength of +// that reading. Both halves of the second check are driven here, because +// deleting the whole block leaves the package green. +func TestRemoveKeptBackupRefusesWhenAttributionChangesUnderTheLock(t *testing.T) { + t.Run("the marker goes unreadable", func(t *testing.T) { + root := t.TempDir() + kept := plantKeptHolder(t, filepath.Join(root, "engine-a"), 1, "engine") + marker := filepath.Join(kept, holderMarkerFile) + + var read atomic.Bool + injectFault(t, "readFile", func(args ...string) bool { + // The pre-lock check reads the marker first; the re-read under the + // lock is the one this fault is for. + return args[0] == marker && !read.CompareAndSwap(false, true) + }, errors.New("injected marker read failure")) + + err := RemoveKeptBackup(root, filepath.Base(kept)) + if err == nil || !strings.Contains(err.Error(), "no longer attributable") { + t.Fatalf("RemoveKeptBackup = %v, want a refusal naming the attribution that changed", err) + } + if _, err := os.Stat(filepath.Join(kept, "install", "engine")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) + t.Run("the marker names another install", func(t *testing.T) { + root := t.TempDir() + kept := plantKeptHolder(t, filepath.Join(root, "engine-a"), 1, "engine") + marker := filepath.Join(kept, holderMarkerFile) + // A readable marker for a different install is the other half: the + // re-read succeeds and answers with a destination the first one did not, + // which is what a directory reused by another promotion looks like from + // here. + other, err := json.Marshal(txnMarker{Kind: holderMarkerKind, Dest: "engine-z", Seq: 1}) + if err != nil { + t.Fatal(err) + } + + real := holderFS + t.Cleanup(func() { holderFS = real }) + var read atomic.Bool + holderFS.readFile = func(name string) ([]byte, error) { + if name != marker || read.CompareAndSwap(false, true) { + return real.readFile(name) + } + return other, nil + } + + if err := RemoveKeptBackup(root, filepath.Base(kept)); err == nil || !strings.Contains(err.Error(), "no longer attributable") { + t.Fatalf("RemoveKeptBackup = %v, want a refusal naming the attribution that changed", err) + } + if _, err := os.Stat(filepath.Join(kept, "install", "engine")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } + }) +} + +// The command takes a name, never a path. The Kept grammar here is a +// destination's own name followed by the suffix, so a name carrying a separator +// still passes it: "../x.kept-...-seq" parses as destination "../x". Only the +// base-name check keeps that from joining to a directory outside the root, and +// it has to run before any filesystem call. +func TestRemoveKeptBackupRejectsANameThatIsNotABaseName(t *testing.T) { + root := t.TempDir() + suffix := fmt.Sprintf("%s%020d%s", keptSuffix, 1, holderSeqSuffix) + for _, step := range []string{"removeAll", "stat", "readFile", "mkdir"} { + injectFault(t, step, func(args ...string) bool { + t.Errorf("a rejected name must not reach the filesystem, got %s(%v)", step, args) + return false + }, nil) + } + names := []string{ + "../x" + suffix, + "a/b" + suffix, + filepath.Join(root, "engine-a") + suffix, + "." + suffix, + ".." + suffix, + "", ".", "..", + } + for _, name := range names { + if err := RemoveKeptBackup(root, name); err == nil { + t.Errorf("RemoveKeptBackup(%q) = nil, want a refusal", name) + } + } +} + +// The scanned prefix is recovery's, not the operator's: a holder under it may +// belong to a promotion running right now, and recovery has a disposition for it +// either way. Only the Kept prefix is this command's to touch. +func TestRemoveKeptBackupNeverTouchesTheScannedPrefix(t *testing.T) { + root := t.TempDir() + holder := plantHolder(t, filepath.Join(root, "engine-a"), 1, "engine", false) + if err := RemoveKeptBackup(root, filepath.Base(holder)); err == nil { + t.Fatal("a holder under the scanned prefix must not be removed by name") + } + if _, err := os.Stat(filepath.Join(holder, "install", "engine")); err != nil { + t.Errorf("the copy must be left intact: %v", err) + } +} + +// A marker this pass could not read is not a marker proving the holder belongs +// to someone else. Treating the two the same lets recovery walk past the copy +// whose state it failed to establish and install an older one, which is the +// provenance loss the whole no-fallback rule exists to prevent. +func TestRestoreInterruptedPromotionStopsWhenAMarkerCannotBeRead(t *testing.T) { + root := t.TempDir() + destDir := filepath.Join(root, "engine-a") + newest := plantHolder(t, destDir, 2, "v2", false) + older := plantHolder(t, destDir, 1, "v1", false) + txn := lockFor(t, destDir) + + injectFault(t, "readFile", func(args ...string) bool { + return strings.HasPrefix(args[0], newest) + }, errors.New("injected marker read failure")) + + var reported []string + restoreInterruptedPromotion(txn, destDir, testPublished, reporterFor(&reported)) + + if _, err := os.Lstat(destDir); !os.IsNotExist(err) { + t.Fatalf("recovery installed a copy while one holder's marker was unreadable: %v", err) + } + for _, holder := range []string{newest, older} { + if _, err := os.Stat(filepath.Join(holder, "install", "engine")); err != nil { + t.Errorf("every copy must be retained until the unreadable one can be read: %v", err) + } + } + if len(reported) == 0 { + t.Error("an unreadable marker must be reported") + } +} + +// A handle kept past its release must be refused. Reporting the lock as still +// held after it was given up is the fail-open direction: the guard exists so a +// call site that forgets the lifecycle is stopped, not trusted. +func TestDestTxnDoesNotClaimALockItReleased(t *testing.T) { + root := t.TempDir() + destDir := filepath.Join(root, "engine-a") + txn, err := lockDestination(context.Background(), root, "engine-a") + if err != nil { + t.Fatal(err) + } + if !txn.holds(destDir) { + t.Fatal("a fresh handle must hold its own destination") + } + txn.release() + if txn.holds(destDir) { + t.Error("a released handle must not report that it still holds the destination") + } +}