Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 55 additions & 21 deletions acfs/acfs.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,55 +45,61 @@ const (
)

// ListEach visits the children of a root-confined directory in name order.
func (r Root) ListEach(ctx context.Context, logicalPath string, visit func(acfstypes.Entry) error) error {
// With options.MaxEntries, it visits only the first MaxEntries names and holds
// no more than twice that many in memory.
func (r Root) ListEach(ctx context.Context, logicalPath string, options acfstypes.ListOptions, visit func(acfstypes.Entry) error) (acfstypes.ListResult, error) {
if visit == nil {
return errors.New("visit callback is required")
return acfstypes.ListResult{}, errors.New("visit callback is required")
}
if options.MaxEntries < 0 {
return acfstypes.ListResult{}, errors.New("list limit must be non-negative")
}
if err := ctx.Err(); err != nil {
return err
return acfstypes.ListResult{}, err
}

relativePath, err := kitfs.NormalizeLogicalPath(logicalPath)
if err != nil {
return err
return acfstypes.ListResult{}, err
}
err = rejectReservedPath(relativePath)
if err != nil {
return err
return acfstypes.ListResult{}, err
}

root, err := os.OpenRoot(r.Path)
if err != nil {
return fmt.Errorf("open workspace root: %w", err)
return acfstypes.ListResult{}, fmt.Errorf("open workspace root: %w", err)
}
defer func() {
_ = root.Close()
}()

resolvedPath, _, err := resolvePath(root, relativePath, resolveOptions{FollowFinal: true})
if err != nil {
return err
return acfstypes.ListResult{}, err
}

names, err := readDirectory(ctx, root, resolvedPath, logicalPath)
names, truncated, err := readDirectory(ctx, root, resolvedPath, logicalPath, options.MaxEntries)
if err != nil {
return err
return acfstypes.ListResult{}, err
}

directory, err := root.OpenRoot(resolvedPath)
if err != nil {
return fmt.Errorf("open directory %q: %w", logicalPath, err)
return acfstypes.ListResult{}, fmt.Errorf("open directory %q: %w", logicalPath, err)
}
defer func() {
_ = directory.Close()
}()
return visitChildren(ctx, directory, names, kit.Ternary(resolvedPath == ".", "/", "/"+resolvedPath+"/"), false, visit)
count, err := visitChildren(ctx, directory, names, kit.Ternary(resolvedPath == ".", "/", "/"+resolvedPath+"/"), false, visit)
return acfstypes.ListResult{Count: count, Truncated: truncated}, err
}

// List returns the direct children of a root-confined directory.
func (r Root) List(ctx context.Context, logicalPath string) ([]acfstypes.Entry, error) {
entries := make([]acfstypes.Entry, 0)
err := r.ListEach(ctx, logicalPath, func(entry acfstypes.Entry) error {
_, err := r.ListEach(ctx, logicalPath, acfstypes.ListOptions{}, func(entry acfstypes.Entry) error {
entries = append(entries, entry)
return nil
})
Expand All @@ -103,32 +109,56 @@ func (r Root) List(ctx context.Context, logicalPath string) ([]acfstypes.Entry,
return entries, nil
}

// readDirectory returns the sorted child names of resolvedPath.
func readDirectory(ctx context.Context, root *os.Root, resolvedPath, displayPath string) ([]string, error) {
// readDirectory returns the sorted child names of resolvedPath, without
// reserved temporary names. With maxEntries, it keeps only the first maxEntries
// names and reports whether it dropped any.
func readDirectory(ctx context.Context, root *os.Root, resolvedPath, displayPath string, maxEntries int) ([]string, bool, error) {
directory, err := root.OpenFile(resolvedPath, os.O_RDONLY|syscall.O_NONBLOCK, 0)
if err != nil {
return nil, fmt.Errorf("open directory %q: %w", displayPath, err)
return nil, false, fmt.Errorf("open directory %q: %w", displayPath, err)
}
defer func() {
_ = directory.Close()
}()

var names []string
truncated := false
for {
if ctxErr := ctx.Err(); ctxErr != nil {
return nil, ctxErr
return nil, false, ctxErr
}
batch, readErr := directory.Readdirnames(directoryReadBatchSize)
names = append(names, batch...)
for _, name := range batch {
if !strings.HasPrefix(name, temporaryWritePrefix) {
names = append(names, name)
}
}
// Trimming at twice the cap bounds memory with amortized O(n log cap) sorting.
if maxEntries > 0 && len(names) >= 2*maxEntries {
names = sortAndTrim(names, maxEntries)
truncated = true
}
if errors.Is(readErr, io.EOF) {
break
}
if readErr != nil {
return nil, fmt.Errorf("read directory %q: %w", displayPath, readErr)
return nil, false, fmt.Errorf("read directory %q: %w", displayPath, readErr)
}
}
if maxEntries > 0 && len(names) > maxEntries {
truncated = true
}
return sortAndTrim(names, maxEntries), truncated, nil
}

// sortAndTrim sorts names and keeps at most maxEntries of them; zero keeps all.
func sortAndTrim(names []string, maxEntries int) []string {
slices.Sort(names)
return names, nil
if maxEntries == 0 || len(names) <= maxEntries {
return names
}
clear(names[maxEntries:])
return names[:maxEntries]
}

// Stat returns metadata for a root-confined path, following the final
Expand Down Expand Up @@ -218,7 +248,7 @@ func (r Root) Walk(ctx context.Context, logicalPath string, options acfstypes.Wa
}
directories = directories[:last]

err := r.ListEach(ctx, w.level.directory.path, visitEntry)
_, err := r.ListEach(ctx, w.level.directory.path, acfstypes.ListOptions{}, visitEntry)
if errors.Is(err, fs.SkipAll) {
return w.result, nil
}
Expand Down Expand Up @@ -257,13 +287,17 @@ func (w *walker) entry(entry acfstypes.Entry) error {

depth := w.level.directory.depth + 1
if w.options.MaxDepth > 0 && depth >= w.options.MaxDepth {
err := w.root.ListEach(w.ctx, entry.Path, func(acfstypes.Entry) error {
probe, err := w.root.ListEach(w.ctx, entry.Path, acfstypes.ListOptions{MaxEntries: 1}, func(acfstypes.Entry) error {
w.result.Truncated = true
return fs.SkipAll
})
if err != nil && !errors.Is(err, fs.SkipAll) {
return err
}
// A truncated probe saw a second child even if the first vanished.
if probe.Truncated {
w.result.Truncated = true
}
return nil
}
w.level.children = append(w.level.children, walkDirectory{path: entry.Path, depth: depth})
Expand Down
56 changes: 54 additions & 2 deletions acfs/acfs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,7 @@ func TestListEachSeesChangesMadeByVisit(t *testing.T) {
}

var visited []acfstypes.Entry
err := Root{Path: root}.ListEach(t.Context(), "/parent/child", func(entry acfstypes.Entry) error {
_, err := Root{Path: root}.ListEach(t.Context(), "/parent/child", acfstypes.ListOptions{}, func(entry acfstypes.Entry) error {
if entry.Name == "a" {
if err := os.Remove(filepath.Join(directory, "b")); err != nil {
return err
Expand Down Expand Up @@ -500,6 +500,58 @@ func TestListEachSeesChangesMadeByVisit(t *testing.T) {
}
}

func TestListEachMaxEntries(t *testing.T) {
t.Parallel()

root := t.TempDir()
names := []string{"g", "c", "j", "a", "e", "h", "b", "i", "d", "f"}
for _, name := range names {
writeFixtureFile(t, filepath.Join(root, name), "x", 0o600)
}
// A reserved name sorts first; it must not take a slot.
writeFixtureFile(t, filepath.Join(root, temporaryWritePrefix+"fixture"), "x", 0o600)

tests := []struct {
name string
want []string
maxEntries int
wantTruncated bool
}{
{name: "unlimited", maxEntries: 0, want: []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}},
{name: "trimmed while reading", maxEntries: 2, want: []string{"a", "b"}, wantTruncated: true},
{name: "trimmed after reading", maxEntries: 6, want: []string{"a", "b", "c", "d", "e", "f"}, wantTruncated: true},
{name: "exactly full", maxEntries: 10, want: []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()

var got []string
result, err := Root{Path: root}.ListEach(t.Context(), "/", acfstypes.ListOptions{MaxEntries: test.maxEntries}, func(entry acfstypes.Entry) error {
got = append(got, entry.Name)
return nil
})
if err != nil {
t.Fatal(err)
}
if !slices.Equal(got, test.want) {
t.Fatalf("ListEach names = %q, want %q", got, test.want)
}
want := acfstypes.ListResult{Count: len(test.want), Truncated: test.wantTruncated}
if result != want {
t.Fatalf("ListEach result = %+v, want %+v", result, want)
}
})
}

_, err := Root{Path: root}.ListEach(t.Context(), "/", acfstypes.ListOptions{MaxEntries: -1}, func(acfstypes.Entry) error {
return nil
})
if err == nil {
t.Fatal("ListEach accepted a negative limit")
}
}

func TestReservedWriteNamesAreHiddenAndRejected(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -551,7 +603,7 @@ func BenchmarkListEach(b *testing.B) {
writeBenchmarkEntries(b, root, 1_000)
b.ReportAllocs()
for b.Loop() {
err := Root{Path: root}.ListEach(b.Context(), "/", func(acfstypes.Entry) error {
_, err := Root{Path: root}.ListEach(b.Context(), "/", acfstypes.ListOptions{}, func(acfstypes.Entry) error {
return nil
})
if err != nil {
Expand Down
21 changes: 10 additions & 11 deletions acfs/entry.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,17 +62,15 @@ func entryLinkTarget(directory *os.Root, name, relativePath string, info os.File
}
}

// visitChildren visits the Entry of each child of directory, hiding reserved
// temporary names. Each child is stat'ed afresh, so changes made by visit are seen.
func visitChildren(ctx context.Context, directory *os.Root, names []string, pathPrefix string, host bool, visit func(acfstypes.Entry) error) error {
// visitChildren visits the Entry of each child of directory and returns how many
// it visited. Each child is stat'ed afresh, so changes made by visit are seen.
func visitChildren(ctx context.Context, directory *os.Root, names []string, pathPrefix string, host bool, visit func(acfstypes.Entry) error) (int, error) {
var modeText string
var lastMode os.FileMode
count := 0
for _, name := range names {
if strings.HasPrefix(name, temporaryWritePrefix) {
continue
}
if err := ctx.Err(); err != nil {
return err
return count, err
}

logicalPath := pathPrefix + name
Expand All @@ -82,25 +80,26 @@ func visitChildren(ctx context.Context, directory *os.Root, names []string, path
continue
}
if err != nil {
return fmt.Errorf("stat %q: %w", logicalPath, rootPathError("statat", relativePath, err))
return count, fmt.Errorf("stat %q: %w", logicalPath, rootPathError("statat", relativePath, err))
}
linkTarget, err := entryLinkTarget(directory, name, relativePath, info, host)
if errors.Is(err, fs.ErrNotExist) {
continue
}
if err != nil {
return err
return count, err
}
if modeText == "" || info.Mode() != lastMode {
lastMode, modeText = info.Mode(), kitfs.FormatMode(info.Mode())
}

err = visit(newEntry(name, logicalPath, modeText, linkTarget, info))
if err != nil {
return fmt.Errorf("visit %q: %w", logicalPath, err)
return count, fmt.Errorf("visit %q: %w", logicalPath, err)
}
count++
}
return nil
return count, nil
}

// rootPathError rewrites an error from a call relative to an opened directory
Expand Down
2 changes: 1 addition & 1 deletion acfs/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ type (
// Root-only; Host has CopyDir and MirrorDir.
FS interface {
List(ctx context.Context, path string) ([]acfstypes.Entry, error)
ListEach(ctx context.Context, path string, visit func(acfstypes.Entry) error) error
ListEach(ctx context.Context, path string, options acfstypes.ListOptions, visit func(acfstypes.Entry) error) (acfstypes.ListResult, error)
Stat(ctx context.Context, path string, options acfstypes.StatOptions) (acfstypes.Entry, error)
Exists(ctx context.Context, path string) (bool, error)

Expand Down
25 changes: 15 additions & 10 deletions acfs/hostdir.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,39 +14,44 @@ import (
acfstypes "kit.getarcane.app/acfs/types"
)

// ListEach visits the children of a host directory in name order.
func (Host) ListEach(ctx context.Context, hostPath string, visit func(acfstypes.Entry) error) error {
// ListEach visits the children of a host directory in name order, bounded by
// options like Root.ListEach.
func (Host) ListEach(ctx context.Context, hostPath string, options acfstypes.ListOptions, visit func(acfstypes.Entry) error) (acfstypes.ListResult, error) {
if visit == nil {
return errors.New("visit callback is required")
return acfstypes.ListResult{}, errors.New("visit callback is required")
}
if options.MaxEntries < 0 {
return acfstypes.ListResult{}, errors.New("list limit must be non-negative")
}
if err := ctx.Err(); err != nil {
return err
return acfstypes.ListResult{}, err
}
directoryPath, err := cleanHostPath(hostPath)
if err != nil {
return err
return acfstypes.ListResult{}, err
}

// os.Root only stats children relative to the descriptor; nothing is confined.
directory, err := os.OpenRoot(directoryPath)
if err != nil {
return fmt.Errorf("open directory %q: %w", directoryPath, err)
return acfstypes.ListResult{}, fmt.Errorf("open directory %q: %w", directoryPath, err)
}
defer func() {
_ = directory.Close()
}()
names, err := readDirectory(ctx, directory, ".", directoryPath)
names, truncated, err := readDirectory(ctx, directory, ".", directoryPath, options.MaxEntries)
if err != nil {
return err
return acfstypes.ListResult{}, err
}

return visitChildren(ctx, directory, names, kit.Ternary(filepath.Dir(directoryPath) == directoryPath, directoryPath, directoryPath+string(filepath.Separator)), true, visit)
count, err := visitChildren(ctx, directory, names, kit.Ternary(filepath.Dir(directoryPath) == directoryPath, directoryPath, directoryPath+string(filepath.Separator)), true, visit)
return acfstypes.ListResult{Count: count, Truncated: truncated}, err
}

// List returns the direct children of a host directory.
func (h Host) List(ctx context.Context, hostPath string) ([]acfstypes.Entry, error) {
entries := make([]acfstypes.Entry, 0)
err := h.ListEach(ctx, hostPath, func(entry acfstypes.Entry) error {
_, err := h.ListEach(ctx, hostPath, acfstypes.ListOptions{}, func(entry acfstypes.Entry) error {
entries = append(entries, entry)
return nil
})
Expand Down
Loading
Loading