diff --git a/drivers/alias/cache.go b/drivers/alias/cache.go new file mode 100644 index 0000000000..88a90c4d06 --- /dev/null +++ b/drivers/alias/cache.go @@ -0,0 +1,252 @@ +package alias + +import ( + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type aliasListCacheEntry struct { + objs []model.Obj + expires time.Time +} + +type aliasResolveCacheEntry struct { + resolved aliasResolved + expires time.Time +} + +type aliasResolved struct { + paths []string + skipped bool + obj model.Object + hasObj bool +} + +type aliasCache struct { + mu sync.RWMutex + lists map[string]aliasListCacheEntry + resolved map[string]aliasResolveCacheEntry +} + +func (c *aliasCache) init() { + c.mu.Lock() + defer c.mu.Unlock() + c.lists = make(map[string]aliasListCacheEntry) + c.resolved = make(map[string]aliasResolveCacheEntry) +} + +func (c *aliasCache) clear() { + c.mu.Lock() + defer c.mu.Unlock() + c.lists = make(map[string]aliasListCacheEntry) + c.resolved = make(map[string]aliasResolveCacheEntry) +} + +func (d *Alias) cacheTTL() time.Duration { + if !d.AliasCacheEnabled { + return 0 + } + expiration := d.AliasCacheExpiration + if expiration <= 0 { + expiration = 30 + } + return time.Minute * time.Duration(expiration) +} + +func (d *Alias) cacheMaxEntries() int { + if !d.AliasCacheEnabled || d.AliasCacheMaxEntries <= 0 { + return 0 + } + return d.AliasCacheMaxEntries +} + +func (c *aliasCache) listKey(dirPaths []string, withDetails bool) string { + parts := make([]string, 0, len(dirPaths)) + for _, dirPath := range dirPaths { + if dirPath == "" { + continue + } + parts = append(parts, utils.FixAndCleanPath(dirPath)) + } + key := "backend:" + strings.Join(parts, "\x00") + if withDetails { + return key + "\x00details" + } + return key +} + +func (c *aliasCache) getList(key string, ttl time.Duration) ([]model.Obj, bool) { + if ttl <= 0 { + return nil, false + } + c.mu.RLock() + entry, ok := c.lists[key] + c.mu.RUnlock() + if !ok { + return nil, false + } + if time.Now().After(entry.expires) { + c.deleteList(key) + return nil, false + } + return cloneObjs(entry.objs), true +} + +func (c *aliasCache) setList(key string, objs []model.Obj, ttl time.Duration, maxEntries int) { + if ttl <= 0 { + return + } + now := time.Now() + c.mu.Lock() + defer c.mu.Unlock() + c.pruneExpiredLocked(now) + c.lists[key] = aliasListCacheEntry{ + objs: cloneObjs(objs), + expires: now.Add(ttl), + } + c.enforceMaxEntriesLocked(maxEntries) +} + +func (c *aliasCache) deleteList(key string) { + c.mu.Lock() + defer c.mu.Unlock() + delete(c.lists, key) +} + +func (c *aliasCache) getResolved(path string, ttl time.Duration) (aliasResolved, bool) { + if ttl <= 0 { + return aliasResolved{}, false + } + path = utils.FixAndCleanPath(path) + c.mu.RLock() + entry, ok := c.resolved[path] + c.mu.RUnlock() + if !ok { + return aliasResolved{}, false + } + if time.Now().After(entry.expires) { + c.deleteResolved(path) + return aliasResolved{}, false + } + return cloneResolved(entry.resolved), true +} + +func (c *aliasCache) setResolved(path string, resolved aliasResolved, ttl time.Duration, maxEntries int) { + if ttl <= 0 || len(resolved.paths) == 0 { + return + } + now := time.Now() + c.mu.Lock() + defer c.mu.Unlock() + c.pruneExpiredLocked(now) + c.resolved[utils.FixAndCleanPath(path)] = aliasResolveCacheEntry{ + resolved: cloneResolved(resolved), + expires: now.Add(ttl), + } + c.enforceMaxEntriesLocked(maxEntries) +} + +func (c *aliasCache) deleteResolved(path string) { + c.mu.Lock() + defer c.mu.Unlock() + delete(c.resolved, utils.FixAndCleanPath(path)) +} + +func (c *aliasCache) deleteResolvedPrefix(prefix string) { + prefix = utils.FixAndCleanPath(prefix) + if prefix == "/" { + c.clear() + return + } + c.mu.Lock() + defer c.mu.Unlock() + for key := range c.resolved { + if key == prefix || strings.HasPrefix(key, prefix+"/") { + delete(c.resolved, key) + } + } +} + +func (c *aliasCache) pruneExpiredLocked(now time.Time) { + for key, entry := range c.lists { + if now.After(entry.expires) { + delete(c.lists, key) + } + } + for key, entry := range c.resolved { + if now.After(entry.expires) { + delete(c.resolved, key) + } + } +} + +func (c *aliasCache) enforceMaxEntriesLocked(maxEntries int) { + if maxEntries <= 0 { + return + } + for len(c.lists)+len(c.resolved) > maxEntries { + if c.deleteOldestLocked() { + continue + } + return + } +} + +func (c *aliasCache) deleteOldestLocked() bool { + var oldestKey string + var oldestList bool + var oldestExpires time.Time + found := false + for key, entry := range c.lists { + if !found || entry.expires.Before(oldestExpires) { + oldestKey = key + oldestList = true + oldestExpires = entry.expires + found = true + } + } + for key, entry := range c.resolved { + if !found || entry.expires.Before(oldestExpires) { + oldestKey = key + oldestList = false + oldestExpires = entry.expires + found = true + } + } + if !found { + return false + } + if oldestList { + delete(c.lists, oldestKey) + } else { + delete(c.resolved, oldestKey) + } + return true +} + +func cloneObjs(objs []model.Obj) []model.Obj { + if len(objs) == 0 { + return nil + } + cloned := make([]model.Obj, len(objs)) + copy(cloned, objs) + return cloned +} + +func cloneStrings(items []string) []string { + if len(items) == 0 { + return nil + } + cloned := make([]string, len(items)) + copy(cloned, items) + return cloned +} + +func cloneResolved(resolved aliasResolved) aliasResolved { + resolved.paths = cloneStrings(resolved.paths) + return resolved +} diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index e1ba41eb61..c6ae826781 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -9,6 +9,8 @@ import ( "net/url" stdpath "path" "strings" + "sync" + "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -28,6 +30,7 @@ type Alias struct { rootOrder []string pathMap map[string][]string root model.Obj + cache aliasCache } func (d *Alias) Config() driver.Config { @@ -39,6 +42,7 @@ func (d *Alias) GetAddition() driver.Additional { } func (d *Alias) Init(ctx context.Context) error { + d.cache.init() paths := strings.Split(d.Paths, "\n") d.rootOrder = make([]string, 0, len(paths)) d.pathMap = make(map[string][]string) @@ -100,6 +104,7 @@ func (d *Alias) Drop(ctx context.Context) error { d.rootOrder = nil d.pathMap = nil d.root = nil + d.cache.clear() return nil } @@ -116,55 +121,50 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { if len(roots) == 0 { return nil, errs.ObjectNotFound } + ttl := d.cacheTTL() + if cached, ok := d.cache.getResolved(path, ttl); ok { + if cached.hasObj { + d.prefetchParentList(ctx, path, sub, roots, &cached.obj) + return d.buildBalancedObj(ctx, sub, cached.paths[0], &cached.obj, cached.paths[1:], cached.skipped), nil + } + obj, err := d.getFromResolvedPaths(ctx, sub, cached) + if err == nil { + d.prefetchParentList(ctx, path, sub, roots, obj) + return obj, nil + } + d.cache.deleteResolved(path) + } + if ttl > 0 { + if resolved, ok := d.getResolvedFromSearchIndex(ctx, roots, sub); ok { + rawPath := resolved.paths[0] + if resolved.hasObj { + d.cache.setResolved(path, resolved, ttl, d.cacheMaxEntries()) + d.prefetchParentList(ctx, path, sub, roots, &resolved.obj) + return d.buildBalancedObj(ctx, sub, rawPath, &resolved.obj, resolved.paths[1:], resolved.skipped), nil + } + obj, err := fs.Get(ctx, rawPath, &fs.GetArgs{NoLog: true}) + if err == nil { + resolved = newAliasResolved(resolved.paths, resolved.skipped, obj) + d.cache.setResolved(path, resolved, ttl, d.cacheMaxEntries()) + d.prefetchParentList(ctx, path, sub, roots, obj) + return d.buildBalancedObj(ctx, sub, rawPath, obj, resolved.paths[1:], resolved.skipped), nil + } + } + } for idx, root := range roots { rawPath := stdpath.Join(root, sub) obj, err := fs.Get(ctx, rawPath, &fs.GetArgs{NoLog: true}) if err != nil { continue } - mask := model.GetObjMask(obj) &^ model.Temp - if sub == "" { - // 根目录 - mask |= model.Locked | model.Virtual - } - ret := model.Object{ - Path: rawPath, - Name: obj.GetName(), - Size: obj.GetSize(), - Modified: obj.ModTime(), - IsFolder: obj.IsDir(), - HashInfo: obj.GetHash(), - Mask: mask, - } - obj = &ret - if d.ProviderPassThrough && !obj.IsDir() { - if storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{}); err == nil { - obj = &model.ObjectProvider{ - Object: ret, - Provider: model.Provider{ - Provider: storage.Config().Name, - }, - } - } + resolvedPaths := make([]string, 0, len(roots)-idx) + for _, root := range roots[idx:] { + resolvedPaths = append(resolvedPaths, stdpath.Join(root, sub)) } - - roots = roots[idx+1:] - var objs BalancedObjs - if idx > 0 { - objs = make(BalancedObjs, 0, len(roots)+2) - } else { - objs = make(BalancedObjs, 0, len(roots)+1) - } - objs = append(objs, obj) - if idx > 0 { - objs = append(objs, nil) - } - for _, d := range roots { - objs = append(objs, &tempObj{model.Object{ - Path: stdpath.Join(d, sub), - }}) - } - return objs, nil + resolved := newAliasResolved(resolvedPaths, idx > 0, obj) + d.cache.setResolved(path, resolved, ttl, d.cacheMaxEntries()) + d.prefetchParentList(ctx, path, sub, roots, obj) + return d.buildBalancedObj(ctx, sub, rawPath, obj, resolved.paths[1:], resolved.skipped), nil } return nil, errs.ObjectNotFound } @@ -174,25 +174,54 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ if !ok { return d.listRoot(ctx, args.WithStorageDetails && d.DetailsPassThrough, args.Refresh), nil } - - // 因为alias是NoCache且Get方法不会返回NotSupport或NotImplement错误 - // 所以这里对象不会传回到alias,也就不需要返回BalancedObjs了 - objMap := make(map[string]model.Obj) + withDetails := args.WithStorageDetails && d.DetailsPassThrough + dirPaths := make([]string, 0, len(dirs)) for _, dir := range dirs { if dir == nil { continue } - dirPath := dir.GetPath() - tmp, err := fs.List(ctx, dirPath, &fs.ListArgs{ - NoLog: true, - Refresh: args.Refresh, - WithStorageDetails: args.WithStorageDetails && d.DetailsPassThrough, - }) - if err != nil { + dirPaths = append(dirPaths, dir.GetPath()) + } + ttl := d.cacheTTL() + cacheKey := d.cache.listKey(dirPaths, withDetails) + if !args.Refresh { + if objs, ok := d.cache.getList(cacheKey, ttl); ok { + return objs, nil + } + } else { + d.cache.deleteList(cacheKey) + if prefix := d.aliasInternalPath(args.ReqPath); prefix != "" { + d.cache.deleteResolvedPrefix(prefix) + } + } + + objs, cacheable := d.listAliasDirs(ctx, args.ReqPath, dirs, dirPaths, args) + if cacheable { + d.cache.setList(cacheKey, objs, ttl, d.cacheMaxEntries()) + } + return objs, nil +} + +func (d *Alias) listAliasDirs(ctx context.Context, reqPath string, dirs []model.Obj, dirPaths []string, args model.ListArgs) ([]model.Obj, bool) { + // 因为alias是NoCache且Get方法不会返回NotSupport或NotImplement错误 + // 所以这里对象不会传回到alias,也就不需要返回BalancedObjs了 + objMap := make(map[string]model.Obj) + resolveMap := make(map[string]aliasResolved) + listResults, cacheable := d.listBackendDirs(ctx, dirPaths, args) + for dirIndex, tmp := range listResults { + if tmp == nil { continue } + dirPath := dirPaths[dirIndex] for _, obj := range tmp { name := obj.GetName() + childPath := stdpath.Join(dirPath, name) + if resolved, exists := resolveMap[name]; exists { + resolved.paths = append(resolved.paths, childPath) + resolveMap[name] = resolved + } else { + resolveMap[name] = newAliasResolved([]string{childPath}, dirIndex > 0, obj) + } if _, exists := objMap[name]; exists { continue } @@ -203,6 +232,7 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ Size: obj.GetSize(), Modified: obj.ModTime(), IsFolder: obj.IsDir(), + HashInfo: obj.GetHash(), Mask: mask, } var objRet model.Obj @@ -238,7 +268,136 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ model.ExtractFolder(objs, sort.ExtractFolder) } } - return objs, nil + if cacheable { + d.cacheResolvedChildren(reqPath, resolveMap) + } + return objs, cacheable +} + +func (d *Alias) prefetchParentList(ctx context.Context, reqPath, sub string, roots []string, obj model.Obj) { + ttl := d.cacheTTL() + if ttl <= 0 || obj == nil || obj.IsDir() || len(roots) == 0 || sub == "" { + return + } + parentReqPath := stdpath.Dir(utils.FixAndCleanPath(reqPath)) + if parentReqPath == "." || parentReqPath == "/" { + return + } + parentSub := stdpath.Dir(sub) + if parentSub == "." { + parentSub = "" + } + parentResolved, ok := d.cache.getResolved(parentReqPath, ttl) + if !ok { + if parentSub == "" { + parentResolved = aliasResolved{paths: cloneStrings(roots)} + ok = true + } else if resolved, hit := d.getResolvedFromSearchIndex(ctx, roots, parentSub); hit { + parentResolved = resolved + ok = true + } + } + if !ok { + paths := make([]string, 0, len(roots)) + for _, root := range roots { + paths = append(paths, stdpath.Join(root, parentSub)) + } + parentResolved = aliasResolved{paths: paths} + } + if len(parentResolved.paths) == 0 { + return + } + d.cache.setResolved(parentReqPath, parentResolved, ttl, d.cacheMaxEntries()) + dirPaths := cloneStrings(parentResolved.paths) + cacheKey := d.cache.listKey(dirPaths, false) + if _, ok := d.cache.getList(cacheKey, ttl); ok { + return + } + go func() { + bgCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + args := model.ListArgs{ + ReqPath: parentReqPath, + } + dirs := aliasDirsFromPaths(dirPaths) + objs, cacheable := d.listAliasDirs(bgCtx, parentReqPath, dirs, dirPaths, args) + if cacheable { + d.cache.setList(cacheKey, objs, ttl, d.cacheMaxEntries()) + } + }() +} + +func aliasDirsFromPaths(paths []string) []model.Obj { + dirs := make([]model.Obj, 0, len(paths)) + for _, path := range paths { + dirs = append(dirs, &model.Object{ + Name: stdpath.Base(path), + Path: path, + IsFolder: true, + }) + } + return dirs +} + +func (d *Alias) listBackendDirs(ctx context.Context, dirPaths []string, args model.ListArgs) ([][]model.Obj, bool) { + results := make([][]model.Obj, len(dirPaths)) + cacheable := true + concurrency := d.listConcurrency(len(dirPaths)) + if concurrency <= 1 { + for i, dirPath := range dirPaths { + tmp, err := fs.List(ctx, dirPath, &fs.ListArgs{ + NoLog: true, + Refresh: args.Refresh, + WithStorageDetails: args.WithStorageDetails && d.DetailsPassThrough, + }) + if err == nil { + results[i] = tmp + } else if !errs.IsNotFoundError(err) { + cacheable = false + } + } + return results, cacheable + } + + var wg sync.WaitGroup + var mu sync.Mutex + sem := make(chan struct{}, concurrency) + for i, dirPath := range dirPaths { + wg.Add(1) + go func(i int, dirPath string) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + tmp, err := fs.List(ctx, dirPath, &fs.ListArgs{ + NoLog: true, + Refresh: args.Refresh, + WithStorageDetails: args.WithStorageDetails && d.DetailsPassThrough, + }) + if err == nil { + results[i] = tmp + } else if !errs.IsNotFoundError(err) { + mu.Lock() + cacheable = false + mu.Unlock() + } + }(i, dirPath) + } + wg.Wait() + return results, cacheable +} + +func (d *Alias) listConcurrency(total int) int { + if total <= 1 { + return 1 + } + concurrency := d.AliasListConcurrency + if concurrency <= 1 { + return 1 + } + if concurrency > total { + return total + } + return concurrency } func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { @@ -348,6 +507,7 @@ func (d *Alias) Other(ctx context.Context, args model.OtherArgs) (interface{}, e } func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + defer d.cache.clear() objs, err := d.getWriteObjs(ctx, parentDir) if err == nil { for _, obj := range objs { @@ -358,6 +518,7 @@ func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string } func (d *Alias) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + defer d.cache.clear() srcs, dsts, err := d.getMoveObjs(ctx, srcObj, dstDir) if err == nil { for i, dst := range dsts { @@ -375,6 +536,7 @@ func (d *Alias) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + defer d.cache.clear() objs, err := d.getWriteObjs(ctx, srcObj) if err == nil { for _, obj := range objs { @@ -385,6 +547,7 @@ func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) er } func (d *Alias) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + defer d.cache.clear() srcs, dsts, err := d.getCopyObjs(ctx, srcObj, dstDir) if err == nil { for i, src := range srcs { @@ -397,6 +560,7 @@ func (d *Alias) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { + defer d.cache.clear() objs, err := d.getWriteObjs(ctx, obj) if err == nil { for _, obj := range objs { @@ -407,6 +571,7 @@ func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { } func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + defer d.cache.clear() objs, err := d.getPutObjs(ctx, dstDir) if err == nil { if len(objs) == 1 { @@ -445,6 +610,7 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, } func (d *Alias) PutURL(ctx context.Context, dstDir model.Obj, name, url string) error { + defer d.cache.clear() objs, err := d.getPutObjs(ctx, dstDir) if err == nil { for _, obj := range objs { @@ -507,6 +673,7 @@ func (d *Alias) Extract(ctx context.Context, obj model.Obj, args model.ArchiveIn } func (d *Alias) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error { + defer d.cache.clear() srcs, dsts, err := d.getCopyObjs(ctx, srcObj, dstDir) if err == nil { for i, src := range srcs { diff --git a/drivers/alias/index.go b/drivers/alias/index.go new file mode 100644 index 0000000000..e9bfd18d7c --- /dev/null +++ b/drivers/alias/index.go @@ -0,0 +1,106 @@ +package alias + +import ( + "context" + stdpath "path" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/search" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type aliasSearchIndexState struct { + node model.SearchNode + exists bool + reliable bool +} + +func (d *Alias) getResolvedFromSearchIndex(ctx context.Context, roots []string, sub string) (aliasResolved, bool) { + if sub == "" || !aliasSearchIndexReady() { + return aliasResolved{}, false + } + known := make(map[string]aliasSearchIndexState) + paths := make([]string, 0, len(roots)) + firstIndex := -1 + for idx, root := range roots { + rawPath := stdpath.Join(root, sub) + _, exists, reliable := aliasSearchIndexPath(ctx, rawPath, known) + if exists { + if firstIndex < 0 { + firstIndex = idx + } + paths = append(paths, rawPath) + } + if !reliable { + return aliasResolved{}, false + } + } + if len(paths) == 0 { + return aliasResolved{}, false + } + return aliasResolved{ + paths: paths, + skipped: firstIndex > 0, + }, true +} + +func aliasSearchIndexReady() bool { + if setting.GetStr(conf.SearchIndex) == "none" { + return false + } + if search.Running() { + return false + } + progress, err := search.Progress() + return err == nil && progress.IsDone && progress.Error == "" +} + +func aliasSearchIndexPath(ctx context.Context, rawPath string, known map[string]aliasSearchIndexState) (node model.SearchNode, exists, reliable bool) { + rawPath = utils.FixAndCleanPath(rawPath) + if rawPath == "/" { + return model.SearchNode{ + Parent: "/", + Name: "", + IsDir: true, + }, true, true + } + if state, ok := known[rawPath]; ok { + return state.node, state.exists, state.reliable + } + parent := stdpath.Dir(rawPath) + name := stdpath.Base(rawPath) + nodes, err := search.Get(ctx, parent) + if err != nil { + known[rawPath] = aliasSearchIndexState{} + return model.SearchNode{}, false, false + } + for _, node := range nodes { + if node.Name == name { + known[rawPath] = aliasSearchIndexState{ + node: node, + exists: true, + reliable: true, + } + return node, true, true + } + } + if len(nodes) > 0 { + known[rawPath] = aliasSearchIndexState{ + exists: false, + reliable: true, + } + return model.SearchNode{}, false, true + } + _, _, parentReliable := aliasSearchIndexPath(ctx, parent, known) + if parentReliable { + known[rawPath] = aliasSearchIndexState{ + exists: false, + reliable: true, + } + return model.SearchNode{}, false, true + } + known[rawPath] = aliasSearchIndexState{} + return model.SearchNode{}, false, false +} diff --git a/drivers/alias/meta.go b/drivers/alias/meta.go index 72eb3c877e..5257b7812b 100644 --- a/drivers/alias/meta.go +++ b/drivers/alias/meta.go @@ -15,6 +15,10 @@ type Addition struct { DownloadPartSize int `json:"download_part_size" default:"0" type:"number" required:"false" help:"Need to enable proxy. Unit: KB"` ProviderPassThrough bool `json:"provider_pass_through" type:"bool" default:"false"` DetailsPassThrough bool `json:"details_pass_through" type:"bool" default:"false"` + AliasCacheEnabled bool `json:"alias_cache_enabled" type:"bool" default:"false" help:"Cache aggregated Alias directory listings and resolved backend paths"` + AliasCacheExpiration int `json:"alias_cache_expiration" default:"30" required:"false" type:"number" help:"Alias cache expiration in minutes"` + AliasCacheMaxEntries int `json:"alias_cache_max_entries" default:"0" required:"false" type:"number" help:"Maximum Alias cache entries. 0 means unlimited"` + AliasListConcurrency int `json:"alias_list_concurrency" default:"1" required:"false" type:"number" help:"Maximum number of backend folders to list concurrently. 1 or less means serial listing"` } var config = driver.Config{ diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 8e5eb8a843..1a545a46f1 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -12,6 +12,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -96,6 +97,113 @@ func (d *Alias) getRootsAndPath(path string) (roots []string, sub string) { return d.pathMap[before], after } +func (d *Alias) aliasInternalPath(reqPath string) string { + if reqPath == "" { + return "" + } + reqPath = utils.FixAndCleanPath(reqPath) + mountPath := utils.GetActualMountPath(d.MountPath) + if reqPath == mountPath { + return "/" + } + if strings.HasPrefix(reqPath, mountPath+"/") { + return strings.TrimPrefix(reqPath, mountPath) + } + return reqPath +} + +func (d *Alias) cacheResolvedChildren(reqPath string, children map[string]aliasResolved) { + ttl := d.cacheTTL() + if ttl <= 0 || reqPath == "" || len(children) == 0 { + return + } + parent := d.aliasInternalPath(reqPath) + if parent == "" { + return + } + maxEntries := d.cacheMaxEntries() + for name, resolved := range children { + d.cache.setResolved(stdpath.Join(parent, name), resolved, ttl, maxEntries) + } +} + +func (d *Alias) getFromResolvedPaths(ctx context.Context, sub string, resolved aliasResolved) (model.Obj, error) { + for idx, rawPath := range resolved.paths { + obj, err := fs.Get(ctx, rawPath, &fs.GetArgs{NoLog: true}) + if err != nil { + continue + } + return d.buildBalancedObj(ctx, sub, rawPath, obj, resolved.paths[idx+1:], resolved.skipped || idx > 0), nil + } + return nil, errs.ObjectNotFound +} + +func newAliasResolved(paths []string, skipped bool, obj model.Obj) aliasResolved { + resolved := aliasResolved{ + paths: paths, + skipped: skipped, + } + if obj != nil { + resolved.obj = model.Object{ + ID: obj.GetID(), + Path: obj.GetPath(), + Name: obj.GetName(), + Size: obj.GetSize(), + Modified: obj.ModTime(), + Ctime: obj.CreateTime(), + IsFolder: obj.IsDir(), + HashInfo: obj.GetHash(), + Mask: model.GetObjMask(obj), + } + resolved.hasObj = true + } + return resolved +} + +func (d *Alias) buildBalancedObj(ctx context.Context, sub, rawPath string, obj model.Obj, remainingPaths []string, skipped bool) BalancedObjs { + mask := model.GetObjMask(obj) &^ model.Temp + if sub == "" { + // root directory + mask |= model.Locked | model.Virtual + } + ret := model.Object{ + Path: rawPath, + Name: obj.GetName(), + Size: obj.GetSize(), + Modified: obj.ModTime(), + IsFolder: obj.IsDir(), + HashInfo: obj.GetHash(), + Mask: mask, + } + var first model.Obj = &ret + if d.ProviderPassThrough && !obj.IsDir() { + if storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{}); err == nil { + first = &model.ObjectProvider{ + Object: ret, + Provider: model.Provider{ + Provider: storage.Config().Name, + }, + } + } + } + + capacity := len(remainingPaths) + 1 + if skipped { + capacity++ + } + objs := make(BalancedObjs, 0, capacity) + objs = append(objs, first) + if skipped { + objs = append(objs, nil) + } + for _, path := range remainingPaths { + objs = append(objs, &tempObj{model.Object{ + Path: path, + }}) + } + return objs +} + func (d *Alias) link(ctx context.Context, reqPath string, args model.LinkArgs) (*model.Link, model.Obj, error) { storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil { diff --git a/internal/search/search.go b/internal/search/search.go index 85be849513..f8d871f058 100644 --- a/internal/search/search.go +++ b/internal/search/search.go @@ -48,9 +48,19 @@ func Init(mode string) error { } func Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { + if instance == nil { + return nil, 0, errs.SearchNotAvailable + } return instance.Search(ctx, req) } +func Get(ctx context.Context, parent string) ([]model.SearchNode, error) { + if instance == nil { + return nil, errs.SearchNotAvailable + } + return instance.Get(ctx, parent) +} + func Index(ctx context.Context, parent string, obj model.Obj) error { if instance == nil { return errs.SearchNotAvailable