Skip to content

fix(op): keep written objects visible until a List returns them #3208

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zeroornull wants to merge 3 commits into
base: main
Choose a base branch
from
Open

fix(op): keep written objects visible until a List returns them #3208

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1d12b99
fix(op): keep written objects visible until a List returns them
zeroornull Oct 18, 2026
b524ea0
fix(op): reclaim expired pending entries across directories
cursoragent Oct 26, 2026
4e3f676
fix(op): preserve pending listing consistency
Suyunmeng Oct 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion internal/op/cache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,9 @@ func (cm *CacheManager) DeleteDirectoryTree(storage driver.Driver, dirPath strin
if storage.Config().NoCache {
return
}
cm.deleteDirectoryTree(Key(storage, dirPath))
key := Key(storage, dirPath)
cm.deleteDirectoryTree(key)
clearPendingTree(key)
}
func (cm *CacheManager) deleteDirectoryTree(key string) {
if dirCache, exists := cm.dirCache.Pop(key); exists {
Expand Down Expand Up @@ -83,6 +85,17 @@ func (cm *CacheManager) removeDirectoryObject(storage driver.Driver, dirPath str
}
}

func (dc *directoryCache) HasObject(name string) bool {
dc.mu.RLock()
defer dc.mu.RUnlock()
for _, obj := range dc.objs {
if obj.GetName() == name {
return true
}
}
return false
}

// cache user data
func (cm *CacheManager) SetUser(username string, user *model.User) {
cm.userCache.Set(username, user)
Expand Down Expand Up @@ -151,6 +164,7 @@ func (cm *CacheManager) ClearAll() {
cm.userCache.Clear()
cm.settingCache.Clear()
cm.detailCache.Clear()
clearAllPending()
}

type directoryCache struct {
Expand Down
154 changes: 90 additions & 64 deletions internal/op/fs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,39 +77,47 @@ func list(ctx context.Context, storage driver.Driver, path string, args model.Li
}

if !storage.Config().NoCache {
if len(files) > 0 {
log.Debugf("set cache: %s => %+v", key, files)

ttl := storage.GetStorage().CacheExpiration

customCachePolicies := storage.GetStorage().CustomCachePolicies
if len(customCachePolicies) > 0 {
for configPolicy := range strings.SplitSeq(customCachePolicies, "\n") {
pattern, ttlstr, ok := strings.Cut(strings.TrimSpace(configPolicy), ":")
if !ok {
log.Warnf("Malformed custom cache policy entry: %s in storage %s for path %s. Expected format: pattern:ttl", configPolicy, storage.GetStorage().MountPath, path)
continue
}
if match, err1 := doublestar.Match(pattern, path); err1 != nil {
log.Warnf("Invalid glob pattern in custom cache policy: %s, error: %v", pattern, err1)
continue
} else if !match {
continue
}

if configTtl, err1 := strconv.ParseInt(ttlstr, 10, 64); err1 == nil {
ttl = int(configTtl)
break
}
// keep objects written through OpenList that a lagging List does not return yet
var pendingDir *dirPending
var pendingVersion uint64
files, pendingDir, pendingVersion = mergePendingState(key, files)
if storage.Config().LocalSort {
model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
}
model.ExtractFolder(files, storage.GetStorage().ExtractFolder)
ttl := storage.GetStorage().CacheExpiration
customCachePolicies := storage.GetStorage().CustomCachePolicies
if len(customCachePolicies) > 0 {
for configPolicy := range strings.SplitSeq(customCachePolicies, "\n") {
pattern, ttlstr, ok := strings.Cut(strings.TrimSpace(configPolicy), ":")
if !ok {
log.Warnf("Malformed custom cache policy entry: %s in storage %s for path %s. Expected format: pattern:ttl", configPolicy, storage.GetStorage().MountPath, path)
continue
}
if match, err1 := doublestar.Match(pattern, path); err1 != nil {
log.Warnf("Invalid glob pattern in custom cache policy: %s, error: %v", pattern, err1)
continue
} else if !match {
continue
}
}

duration := time.Minute * time.Duration(ttl)
Cache.dirCache.SetWithTTL(key, newDirectoryCache(files), duration)
} else {
log.Debugf("del cache: %s", key)
Cache.deleteDirectoryTree(key)
if configTtl, err1 := strconv.ParseInt(ttlstr, 10, 64); err1 == nil {
ttl = int(configTtl)
break
}
}
}
duration := time.Minute * time.Duration(ttl)

cachePendingResult(key, pendingDir, pendingVersion, func() {
if len(files) > 0 {
log.Debugf("set cache: %s => %+v", key, files)
Cache.dirCache.SetWithTTL(key, newDirectoryCache(files), duration)
} else {
log.Debugf("del cache: %s", key)
Cache.deleteDirectoryTree(key)
}
})
}
return files, nil
})
Expand Down Expand Up @@ -164,7 +172,8 @@ func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj

// try get from cache first
dir, name := stdpath.Split(path)
dirCache, dirCacheExists := Cache.dirCache.Get(Key(storage, dir))
dirKey := Key(storage, dir)
dirCache, dirCacheExists := Cache.dirCache.Get(dirKey)
refreshList := false
excludeTemp := utils.IsBool(excludeTempObj...)
if dirCacheExists {
Expand All @@ -191,6 +200,13 @@ func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj
}
}

// an object written through OpenList that the cached listing does not contain (yet)
if !refreshList {
if obj, ok := getPending(dirKey, name); ok && !(excludeTemp && model.ObjHasMask(obj, model.Temp)) {
return obj, nil
}
}

if !dirCacheExists || refreshList {
var obj model.Obj
list(ctx, storage, dir, model.ListArgs{Refresh: refreshList}, func(objs []model.Obj) error {
Expand Down Expand Up @@ -423,20 +439,22 @@ func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
Cache.linkCache.DeleteKey(stdpath.Join(dstKey, srcRawObj.GetName()))
}
if !storage.Config().NoCache {
if srcRawObj.IsDir() {
clearPendingTree(stdpath.Join(srcKey, srcRawObj.GetName()))
}
delPending(srcKey, srcRawObj.GetName())
if cache, exist := Cache.dirCache.Get(srcKey); exist {
if srcRawObj.IsDir() {
Cache.deleteDirectoryTree(stdpath.Join(srcKey, srcRawObj.GetName()))
}
cache.RemoveObject(srcRawObj.GetName())
}
if cache, exist := Cache.dirCache.Get(dstKey); exist {
if newObj == nil {
newObj = &model.ObjWrapMask{Obj: srcRawObj, Mask: model.Temp}
} else {
newObj = wrapObjName(storage, newObj)
}
cache.UpdateObject(srcRawObj.GetName(), newObj)
if newObj == nil {
newObj = &model.ObjWrapMask{Obj: srcRawObj, Mask: model.Temp}
} else {
newObj = wrapObjName(storage, newObj)
}
putPending(dstKey, newObj)
}

if ctx.Value(conf.SkipHookKey) != nil || !needHandleObjsUpdateHook() {
Expand Down Expand Up @@ -487,14 +505,19 @@ func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string)
Cache.linkCache.DeleteKey(stdpath.Join(dirKey, dstName))
}
if !storage.Config().NoCache {
if srcRawObj.IsDir() {
clearPendingTree(stdpath.Join(dirKey, oldName))
}
delPending(dirKey, oldName)
if newObj == nil {
newObj = &model.ObjWrapMask{Obj: &model.ObjWrapName{Name: dstName, Obj: srcObj}, Mask: model.Temp}
}
newObj = wrapObjName(storage, newObj)
addPending(dirKey, newObj)
if cache, exist := Cache.dirCache.Get(dirKey); exist {
if srcRawObj.IsDir() {
Cache.deleteDirectoryTree(stdpath.Join(dirKey, oldName))
}
if newObj == nil {
newObj = &model.ObjWrapMask{Obj: &model.ObjWrapName{Name: dstName, Obj: srcObj}, Mask: model.Temp}
}
newObj = wrapObjName(storage, newObj)
cache.UpdateObject(oldName, newObj)
}
}
Expand Down Expand Up @@ -603,6 +626,13 @@ func Remove(ctx context.Context, storage driver.Driver, path string) error {
err = s.Remove(ctx, model.UnwrapObjName(rawObj))
if err == nil {
Cache.removeDirectoryObject(storage, dirPath, rawObj)
if !storage.Config().NoCache {
dirKey := Key(storage, dirPath)
if rawObj.IsDir() {
clearPendingTree(stdpath.Join(dirKey, rawObj.GetName()))
}
delPending(dirKey, rawObj.GetName())
}
}
default:
return errs.NotImplement
Expand Down Expand Up @@ -682,19 +712,17 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod
if err == nil {
Cache.linkCache.DeleteKey(Key(storage, dstPath))
if !storage.Config().NoCache {
if cache, exist := Cache.dirCache.Get(Key(storage, dstDirPath)); exist {
if newObj == nil {
newObj = &model.Object{
Name: file.GetName(),
Size: file.GetSize(),
Modified: file.ModTime(),
Ctime: file.CreateTime(),
Mask: model.Temp,
}
if newObj == nil {
newObj = &model.Object{
Name: file.GetName(),
Size: file.GetSize(),
Modified: file.ModTime(),
Ctime: file.CreateTime(),
Mask: model.Temp,
}
newObj = wrapObjName(storage, newObj)
cache.UpdateObject(newObj.GetName(), newObj)
}
newObj = wrapObjName(storage, newObj)
putPending(Key(storage, dstDirPath), newObj)
}

if ctx.Value(conf.SkipHookKey) == nil && needHandleObjsUpdateHook() {
Expand Down Expand Up @@ -750,19 +778,17 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url
if err == nil {
Cache.linkCache.DeleteKey(Key(storage, dstPath))
if !storage.Config().NoCache {
if cache, exist := Cache.dirCache.Get(Key(storage, dstDirPath)); exist {
if newObj == nil {
t := time.Now()
newObj = &model.Object{
Name: dstName,
Modified: t,
Ctime: t,
Mask: model.Temp,
}
if newObj == nil {
t := time.Now()
newObj = &model.Object{
Name: dstName,
Modified: t,
Ctime: t,
Mask: model.Temp,
}
newObj = wrapObjName(storage, newObj)
cache.UpdateObject(newObj.GetName(), newObj)
}
newObj = wrapObjName(storage, newObj)
putPending(Key(storage, dstDirPath), newObj)

if ctx.Value(conf.SkipHookKey) == nil && needHandleObjsUpdateHook() {
go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false)
Expand Down
Loading
Loading