From f98f115ac2d70245f1089188a9c972783001439a Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Wed, 15 Sep 2021 09:24:41 -0700 Subject: [PATCH] fs: Fix non-progressing scanner (#13218) Scanner would keep doing the same cycle in FS mode leading to missed updates. Add a few sanity checks and handle errors better. --- cmd/data-scanner.go | 17 +++++++++++------ cmd/fs-v1.go | 9 ++++++++- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index 820383e2b..8b785da27 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -405,11 +405,11 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error { // Parse entName = pathClean(path.Join(folder.name, entName)) - if entName == "" { + if entName == "" || entName == folder.name { if f.dataUsageScannerDebug { - console.Debugf(scannerLogPrefix+" no bucket (%s,%s)\n", f.root, entName) + console.Debugf(scannerLogPrefix+" no entity (%s,%s)\n", f.root, entName) } - return errDoneForNow + return nil } bucket, prefix := path2BucketObjectWithBasePath(f.root, entName) if bucket == "" { @@ -435,7 +435,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int if typ&os.ModeDir != 0 { h := hashPath(entName) _, exists := f.oldCache.Cache[h.Key()] - + if h == thisHash { + return nil + } this := cachedFolder{name: entName, parent: &thisHash, objectHealProbDiv: folder.objectHealProbDiv} delete(abandonedChildren, h.Key()) // h.Key() already accounted for. if exists { @@ -468,11 +470,14 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int item.heal = item.heal && f.healObjectSelect > 0 sz, err := f.getSize(item) - if err == errSkipFile { + if err != nil { wait() // wait to proceed to next entry. - + if err != errSkipFile && f.dataUsageScannerDebug { + console.Debugf(scannerLogPrefix+" getSize \"%v/%v\" returned err: %v\n", bucket, item.objectPath(), err) + } return nil } + // successfully read means we have a valid object. foundObjects = true // Remove filename i.e is the meta file to construct object name diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 58bf69d12..1cda0e1e0 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -258,6 +258,12 @@ func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates cha updates <- totalCache.dui(dataUsageRoot, buckets) return nil } + for i, b := range buckets { + if isReservedOrInvalidBucket(b.Name, false) { + // Delete bucket... + buckets = append(buckets[:i], buckets[i+1:]...) + } + } totalCache.Info.BloomFilter = bf.bytes() @@ -282,6 +288,7 @@ func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates cha bCache.Info.Name = b.Name } bCache.Info.BloomFilter = totalCache.Info.BloomFilter + bCache.Info.NextCycle = wantCycle upds := make(chan dataUsageEntry, 1) var wg sync.WaitGroup wg.Add(1) @@ -290,7 +297,7 @@ func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates cha for update := range upds { totalCache.replace(b.Name, dataUsageRoot, update) if intDataUpdateTracker.debug { - logger.Info(color.Green("NSScanner:")+" Got update:", len(totalCache.Cache)) + logger.Info(color.Green("NSScanner:")+" Got update: %v", len(totalCache.Cache)) } cloned := totalCache.clone() updates <- cloned.dui(dataUsageRoot, buckets)