From 951b1e6a7a371ff07d826a12b8525bea7382362e Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 8 Sep 2021 08:15:42 -0700 Subject: [PATCH] fix: Optimize listing calls for NFS mounts (#13159) --no-compat should allow for some optimized behavior for NFS mounts by removing Stat() operations. --- cmd/fs-v1.go | 9 +++++++++ cmd/object-api-common.go | 7 ++++++- cmd/object-api-listobjects_test.go | 10 ---------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 7a0c6d2df..58bf69d12 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -854,6 +854,15 @@ func (fs *FSObjects) getObjectInfoNoFSLock(ctx context.Context, bucket, object s return fsMeta.ToObjectInfo(bucket, object, fi), nil } + if !globalCLIContext.StrictS3Compat { + // Stat the file to get file size. + fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object)) + if err != nil { + return oi, err + } + return fsMeta.ToObjectInfo(bucket, object, fi), nil + } + fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile) // Read `fs.json` to perhaps contend with // parallel Put() operations. diff --git a/cmd/object-api-common.go b/cmd/object-api-common.go index bed1c3014..0731e8681 100644 --- a/cmd/object-api-common.go +++ b/cmd/object-api-common.go @@ -276,8 +276,13 @@ func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, d var eof bool var nextMarker string + maxConcurrent := maxKeys / 10 + if maxConcurrent == 0 { + maxConcurrent = maxKeys + } + // List until maxKeys requested. - g := errgroup.WithNErrs(maxKeys).WithConcurrency(10) + g := errgroup.WithNErrs(maxKeys).WithConcurrency(maxConcurrent) ctx, cancel := g.WithCancelOnError(ctx) defer cancel() diff --git a/cmd/object-api-listobjects_test.go b/cmd/object-api-listobjects_test.go index 474c1b862..cc63b7e74 100644 --- a/cmd/object-api-listobjects_test.go +++ b/cmd/object-api-listobjects_test.go @@ -679,11 +679,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) { if testCase.result.Objects[j].Name != result.Objects[j].Name { t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name) } - // FIXME: we should always check for ETag - if result.Objects[j].ETag == "" && !strings.HasSuffix(result.Objects[j].Name, SlashSeparator) { - t.Errorf("Test %d: %s: Expected ETag to be not empty, but found empty instead (%v)", i+1, instanceType, result.Objects[j].Name) - } - } if len(testCase.result.Prefixes) != len(result.Prefixes) { @@ -1350,11 +1345,6 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand if testCase.result.Objects[j].Name != result.Objects[j].Name { t.Errorf("%s: Expected object name to be \"%s\", but found \"%s\" instead", instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name) } - // FIXME: we should always check for ETag - if result.Objects[j].ETag == "" && !strings.HasSuffix(result.Objects[j].Name, SlashSeparator) { - t.Errorf("%s: Expected ETag to be not empty, but found empty instead (%v)", instanceType, result.Objects[j].Name) - } - } if len(testCase.result.Prefixes) != len(result.Prefixes) {