diff --git a/acfs/acfs.go b/acfs/acfs.go index 0b900d7..90a9af8 100644 --- a/acfs/acfs.go +++ b/acfs/acfs.go @@ -45,26 +45,31 @@ const ( ) // ListEach visits the children of a root-confined directory in name order. -func (r Root) ListEach(ctx context.Context, logicalPath string, visit func(acfstypes.Entry) error) error { +// With options.MaxEntries, it visits only the first MaxEntries names and holds +// no more than twice that many in memory. +func (r Root) ListEach(ctx context.Context, logicalPath string, options acfstypes.ListOptions, visit func(acfstypes.Entry) error) (acfstypes.ListResult, error) { if visit == nil { - return errors.New("visit callback is required") + return acfstypes.ListResult{}, errors.New("visit callback is required") + } + if options.MaxEntries < 0 { + return acfstypes.ListResult{}, errors.New("list limit must be non-negative") } if err := ctx.Err(); err != nil { - return err + return acfstypes.ListResult{}, err } relativePath, err := kitfs.NormalizeLogicalPath(logicalPath) if err != nil { - return err + return acfstypes.ListResult{}, err } err = rejectReservedPath(relativePath) if err != nil { - return err + return acfstypes.ListResult{}, err } root, err := os.OpenRoot(r.Path) if err != nil { - return fmt.Errorf("open workspace root: %w", err) + return acfstypes.ListResult{}, fmt.Errorf("open workspace root: %w", err) } defer func() { _ = root.Close() @@ -72,28 +77,29 @@ func (r Root) ListEach(ctx context.Context, logicalPath string, visit func(acfst resolvedPath, _, err := resolvePath(root, relativePath, resolveOptions{FollowFinal: true}) if err != nil { - return err + return acfstypes.ListResult{}, err } - names, err := readDirectory(ctx, root, resolvedPath, logicalPath) + names, truncated, err := readDirectory(ctx, root, resolvedPath, logicalPath, options.MaxEntries) if err != nil { - return err + return acfstypes.ListResult{}, err } directory, err := root.OpenRoot(resolvedPath) if err != nil { - return fmt.Errorf("open directory %q: %w", logicalPath, err) + return acfstypes.ListResult{}, fmt.Errorf("open directory %q: %w", logicalPath, err) } defer func() { _ = directory.Close() }() - return visitChildren(ctx, directory, names, kit.Ternary(resolvedPath == ".", "/", "/"+resolvedPath+"/"), false, visit) + count, err := visitChildren(ctx, directory, names, kit.Ternary(resolvedPath == ".", "/", "/"+resolvedPath+"/"), false, visit) + return acfstypes.ListResult{Count: count, Truncated: truncated}, err } // List returns the direct children of a root-confined directory. func (r Root) List(ctx context.Context, logicalPath string) ([]acfstypes.Entry, error) { entries := make([]acfstypes.Entry, 0) - err := r.ListEach(ctx, logicalPath, func(entry acfstypes.Entry) error { + _, err := r.ListEach(ctx, logicalPath, acfstypes.ListOptions{}, func(entry acfstypes.Entry) error { entries = append(entries, entry) return nil }) @@ -103,32 +109,56 @@ func (r Root) List(ctx context.Context, logicalPath string) ([]acfstypes.Entry, return entries, nil } -// readDirectory returns the sorted child names of resolvedPath. -func readDirectory(ctx context.Context, root *os.Root, resolvedPath, displayPath string) ([]string, error) { +// readDirectory returns the sorted child names of resolvedPath, without +// reserved temporary names. With maxEntries, it keeps only the first maxEntries +// names and reports whether it dropped any. +func readDirectory(ctx context.Context, root *os.Root, resolvedPath, displayPath string, maxEntries int) ([]string, bool, error) { directory, err := root.OpenFile(resolvedPath, os.O_RDONLY|syscall.O_NONBLOCK, 0) if err != nil { - return nil, fmt.Errorf("open directory %q: %w", displayPath, err) + return nil, false, fmt.Errorf("open directory %q: %w", displayPath, err) } defer func() { _ = directory.Close() }() var names []string + truncated := false for { if ctxErr := ctx.Err(); ctxErr != nil { - return nil, ctxErr + return nil, false, ctxErr } batch, readErr := directory.Readdirnames(directoryReadBatchSize) - names = append(names, batch...) + for _, name := range batch { + if !strings.HasPrefix(name, temporaryWritePrefix) { + names = append(names, name) + } + } + // Trimming at twice the cap bounds memory with amortized O(n log cap) sorting. + if maxEntries > 0 && len(names) >= 2*maxEntries { + names = sortAndTrim(names, maxEntries) + truncated = true + } if errors.Is(readErr, io.EOF) { break } if readErr != nil { - return nil, fmt.Errorf("read directory %q: %w", displayPath, readErr) + return nil, false, fmt.Errorf("read directory %q: %w", displayPath, readErr) } } + if maxEntries > 0 && len(names) > maxEntries { + truncated = true + } + return sortAndTrim(names, maxEntries), truncated, nil +} + +// sortAndTrim sorts names and keeps at most maxEntries of them; zero keeps all. +func sortAndTrim(names []string, maxEntries int) []string { slices.Sort(names) - return names, nil + if maxEntries == 0 || len(names) <= maxEntries { + return names + } + clear(names[maxEntries:]) + return names[:maxEntries] } // Stat returns metadata for a root-confined path, following the final @@ -218,7 +248,7 @@ func (r Root) Walk(ctx context.Context, logicalPath string, options acfstypes.Wa } directories = directories[:last] - err := r.ListEach(ctx, w.level.directory.path, visitEntry) + _, err := r.ListEach(ctx, w.level.directory.path, acfstypes.ListOptions{}, visitEntry) if errors.Is(err, fs.SkipAll) { return w.result, nil } @@ -257,13 +287,17 @@ func (w *walker) entry(entry acfstypes.Entry) error { depth := w.level.directory.depth + 1 if w.options.MaxDepth > 0 && depth >= w.options.MaxDepth { - err := w.root.ListEach(w.ctx, entry.Path, func(acfstypes.Entry) error { + probe, err := w.root.ListEach(w.ctx, entry.Path, acfstypes.ListOptions{MaxEntries: 1}, func(acfstypes.Entry) error { w.result.Truncated = true return fs.SkipAll }) if err != nil && !errors.Is(err, fs.SkipAll) { return err } + // A truncated probe saw a second child even if the first vanished. + if probe.Truncated { + w.result.Truncated = true + } return nil } w.level.children = append(w.level.children, walkDirectory{path: entry.Path, depth: depth}) diff --git a/acfs/acfs_test.go b/acfs/acfs_test.go index 336983e..e3dbfba 100644 --- a/acfs/acfs_test.go +++ b/acfs/acfs_test.go @@ -470,7 +470,7 @@ func TestListEachSeesChangesMadeByVisit(t *testing.T) { } var visited []acfstypes.Entry - err := Root{Path: root}.ListEach(t.Context(), "/parent/child", func(entry acfstypes.Entry) error { + _, err := Root{Path: root}.ListEach(t.Context(), "/parent/child", acfstypes.ListOptions{}, func(entry acfstypes.Entry) error { if entry.Name == "a" { if err := os.Remove(filepath.Join(directory, "b")); err != nil { return err @@ -500,6 +500,58 @@ func TestListEachSeesChangesMadeByVisit(t *testing.T) { } } +func TestListEachMaxEntries(t *testing.T) { + t.Parallel() + + root := t.TempDir() + names := []string{"g", "c", "j", "a", "e", "h", "b", "i", "d", "f"} + for _, name := range names { + writeFixtureFile(t, filepath.Join(root, name), "x", 0o600) + } + // A reserved name sorts first; it must not take a slot. + writeFixtureFile(t, filepath.Join(root, temporaryWritePrefix+"fixture"), "x", 0o600) + + tests := []struct { + name string + want []string + maxEntries int + wantTruncated bool + }{ + {name: "unlimited", maxEntries: 0, want: []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}}, + {name: "trimmed while reading", maxEntries: 2, want: []string{"a", "b"}, wantTruncated: true}, + {name: "trimmed after reading", maxEntries: 6, want: []string{"a", "b", "c", "d", "e", "f"}, wantTruncated: true}, + {name: "exactly full", maxEntries: 10, want: []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + var got []string + result, err := Root{Path: root}.ListEach(t.Context(), "/", acfstypes.ListOptions{MaxEntries: test.maxEntries}, func(entry acfstypes.Entry) error { + got = append(got, entry.Name) + return nil + }) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(got, test.want) { + t.Fatalf("ListEach names = %q, want %q", got, test.want) + } + want := acfstypes.ListResult{Count: len(test.want), Truncated: test.wantTruncated} + if result != want { + t.Fatalf("ListEach result = %+v, want %+v", result, want) + } + }) + } + + _, err := Root{Path: root}.ListEach(t.Context(), "/", acfstypes.ListOptions{MaxEntries: -1}, func(acfstypes.Entry) error { + return nil + }) + if err == nil { + t.Fatal("ListEach accepted a negative limit") + } +} + func TestReservedWriteNamesAreHiddenAndRejected(t *testing.T) { t.Parallel() @@ -551,7 +603,7 @@ func BenchmarkListEach(b *testing.B) { writeBenchmarkEntries(b, root, 1_000) b.ReportAllocs() for b.Loop() { - err := Root{Path: root}.ListEach(b.Context(), "/", func(acfstypes.Entry) error { + _, err := Root{Path: root}.ListEach(b.Context(), "/", acfstypes.ListOptions{}, func(acfstypes.Entry) error { return nil }) if err != nil { diff --git a/acfs/entry.go b/acfs/entry.go index 9b1cf86..3c9e489 100644 --- a/acfs/entry.go +++ b/acfs/entry.go @@ -62,17 +62,15 @@ func entryLinkTarget(directory *os.Root, name, relativePath string, info os.File } } -// visitChildren visits the Entry of each child of directory, hiding reserved -// temporary names. Each child is stat'ed afresh, so changes made by visit are seen. -func visitChildren(ctx context.Context, directory *os.Root, names []string, pathPrefix string, host bool, visit func(acfstypes.Entry) error) error { +// visitChildren visits the Entry of each child of directory and returns how many +// it visited. Each child is stat'ed afresh, so changes made by visit are seen. +func visitChildren(ctx context.Context, directory *os.Root, names []string, pathPrefix string, host bool, visit func(acfstypes.Entry) error) (int, error) { var modeText string var lastMode os.FileMode + count := 0 for _, name := range names { - if strings.HasPrefix(name, temporaryWritePrefix) { - continue - } if err := ctx.Err(); err != nil { - return err + return count, err } logicalPath := pathPrefix + name @@ -82,14 +80,14 @@ func visitChildren(ctx context.Context, directory *os.Root, names []string, path continue } if err != nil { - return fmt.Errorf("stat %q: %w", logicalPath, rootPathError("statat", relativePath, err)) + return count, fmt.Errorf("stat %q: %w", logicalPath, rootPathError("statat", relativePath, err)) } linkTarget, err := entryLinkTarget(directory, name, relativePath, info, host) if errors.Is(err, fs.ErrNotExist) { continue } if err != nil { - return err + return count, err } if modeText == "" || info.Mode() != lastMode { lastMode, modeText = info.Mode(), kitfs.FormatMode(info.Mode()) @@ -97,10 +95,11 @@ func visitChildren(ctx context.Context, directory *os.Root, names []string, path err = visit(newEntry(name, logicalPath, modeText, linkTarget, info)) if err != nil { - return fmt.Errorf("visit %q: %w", logicalPath, err) + return count, fmt.Errorf("visit %q: %w", logicalPath, err) } + count++ } - return nil + return count, nil } // rootPathError rewrites an error from a call relative to an opened directory diff --git a/acfs/fs.go b/acfs/fs.go index b6ba1db..6ba3ffa 100644 --- a/acfs/fs.go +++ b/acfs/fs.go @@ -14,7 +14,7 @@ type ( // Root-only; Host has CopyDir and MirrorDir. FS interface { List(ctx context.Context, path string) ([]acfstypes.Entry, error) - ListEach(ctx context.Context, path string, visit func(acfstypes.Entry) error) error + ListEach(ctx context.Context, path string, options acfstypes.ListOptions, visit func(acfstypes.Entry) error) (acfstypes.ListResult, error) Stat(ctx context.Context, path string, options acfstypes.StatOptions) (acfstypes.Entry, error) Exists(ctx context.Context, path string) (bool, error) diff --git a/acfs/hostdir.go b/acfs/hostdir.go index 8f5eabe..8c5aeaf 100644 --- a/acfs/hostdir.go +++ b/acfs/hostdir.go @@ -14,39 +14,44 @@ import ( acfstypes "kit.getarcane.app/acfs/types" ) -// ListEach visits the children of a host directory in name order. -func (Host) ListEach(ctx context.Context, hostPath string, visit func(acfstypes.Entry) error) error { +// ListEach visits the children of a host directory in name order, bounded by +// options like Root.ListEach. +func (Host) ListEach(ctx context.Context, hostPath string, options acfstypes.ListOptions, visit func(acfstypes.Entry) error) (acfstypes.ListResult, error) { if visit == nil { - return errors.New("visit callback is required") + return acfstypes.ListResult{}, errors.New("visit callback is required") + } + if options.MaxEntries < 0 { + return acfstypes.ListResult{}, errors.New("list limit must be non-negative") } if err := ctx.Err(); err != nil { - return err + return acfstypes.ListResult{}, err } directoryPath, err := cleanHostPath(hostPath) if err != nil { - return err + return acfstypes.ListResult{}, err } // os.Root only stats children relative to the descriptor; nothing is confined. directory, err := os.OpenRoot(directoryPath) if err != nil { - return fmt.Errorf("open directory %q: %w", directoryPath, err) + return acfstypes.ListResult{}, fmt.Errorf("open directory %q: %w", directoryPath, err) } defer func() { _ = directory.Close() }() - names, err := readDirectory(ctx, directory, ".", directoryPath) + names, truncated, err := readDirectory(ctx, directory, ".", directoryPath, options.MaxEntries) if err != nil { - return err + return acfstypes.ListResult{}, err } - return visitChildren(ctx, directory, names, kit.Ternary(filepath.Dir(directoryPath) == directoryPath, directoryPath, directoryPath+string(filepath.Separator)), true, visit) + count, err := visitChildren(ctx, directory, names, kit.Ternary(filepath.Dir(directoryPath) == directoryPath, directoryPath, directoryPath+string(filepath.Separator)), true, visit) + return acfstypes.ListResult{Count: count, Truncated: truncated}, err } // List returns the direct children of a host directory. func (h Host) List(ctx context.Context, hostPath string) ([]acfstypes.Entry, error) { entries := make([]acfstypes.Entry, 0) - err := h.ListEach(ctx, hostPath, func(entry acfstypes.Entry) error { + _, err := h.ListEach(ctx, hostPath, acfstypes.ListOptions{}, func(entry acfstypes.Entry) error { entries = append(entries, entry) return nil }) diff --git a/acfs/internal/cli/run.go b/acfs/internal/cli/run.go index 3c55792..41a5c33 100644 --- a/acfs/internal/cli/run.go +++ b/acfs/internal/cli/run.go @@ -149,27 +149,15 @@ func runList(ctx context.Context, args []string, stdout io.Writer) error { bufferedOutput := bufio.NewWriterSize(temporaryOutput, 64*1024) started := false - count := 0 - errLimitReached := errors.New("list limit reached") var encodedEntry bytes.Buffer entryEncoder := json.NewEncoder(&encodedEntry) entryEncoder.SetEscapeHTML(false) - err = acfs.Root{Path: values.root}.ListEach(ctx, values.path, func(entry acfstypes.Entry) error { - if *limit > 0 && count == *limit { - return errLimitReached - } - if !started { - if _, writeErr := io.WriteString(bufferedOutput, `{"entries":[`); writeErr != nil { - return fmt.Errorf("write list prefix: %w", writeErr) - } - started = true - } - if count > 0 { - if _, writeErr := io.WriteString(bufferedOutput, ","); writeErr != nil { - return fmt.Errorf("write list delimiter: %w", writeErr) - } + result, err := acfs.Root{Path: values.root}.ListEach(ctx, values.path, acfstypes.ListOptions{MaxEntries: *limit}, func(entry acfstypes.Entry) error { + separator := kit.Ternary(started, ",", `{"entries":[`) + if _, writeErr := io.WriteString(bufferedOutput, separator); writeErr != nil { + return fmt.Errorf("write list entry: %w", writeErr) } - count++ + started = true encodedEntry.Reset() if encodeErr := entryEncoder.Encode(entry); encodeErr != nil { @@ -181,8 +169,7 @@ func runList(ctx context.Context, args []string, stdout io.Writer) error { } return nil }) - truncated := errors.Is(err, errLimitReached) - if err != nil && !truncated { + if err != nil { return err } if !started { @@ -191,7 +178,7 @@ func runList(ctx context.Context, args []string, stdout io.Writer) error { return fmt.Errorf("write list prefix: %w", err) } } - _, err = fmt.Fprintf(bufferedOutput, `]%s,"version":%d}`+"\n", kit.Ternary(truncated, `,"truncated":true`, ""), acfstypes.ProtocolVersion) + _, err = fmt.Fprintf(bufferedOutput, `]%s,"version":%d}`+"\n", kit.Ternary(result.Truncated, `,"truncated":true`, ""), acfstypes.ProtocolVersion) if err != nil { return fmt.Errorf("write list suffix: %w", err) } diff --git a/acfs/types/entry.go b/acfs/types/entry.go index 38e6ea1..741f974 100644 --- a/acfs/types/entry.go +++ b/acfs/types/entry.go @@ -61,6 +61,18 @@ type ( FollowFinal bool } + // ListOptions bounds a directory listing. Zero values are unlimited. + ListOptions struct { + MaxEntries int + } + + // ListResult summarizes a bounded directory listing. Truncated reports that + // the directory held more than MaxEntries visible names. + ListResult struct { + Count int + Truncated bool + } + // WalkOptions bounds a recursive traversal. Zero values are unlimited. WalkOptions struct { MaxDepth int