diff --git a/cmd/erasure-healing_test.go b/cmd/erasure-healing_test.go index 62c6d873b..488d150ad 100644 --- a/cmd/erasure-healing_test.go +++ b/cmd/erasure-healing_test.go @@ -24,6 +24,7 @@ import ( "os" "path" "reflect" + "runtime" "testing" "time" @@ -145,6 +146,150 @@ func TestHealing(t *testing.T) { } } +func TestHealingDanglingObject(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + resetGlobalHealState() + defer resetGlobalHealState() + + nDisks := 16 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + + //defer removeRoots(fsDirs) + + // Everything is fine, should return nil + objLayer, disks, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...)) + if err != nil { + t.Fatal(err) + } + + bucket := getRandomBucketName() + object := getRandomObjectName() + data := bytes.Repeat([]byte("a"), 128*1024) + + err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{}) + if err != nil { + t.Fatalf("Failed to make a bucket - %v", err) + } + + // Enable versioning. + globalBucketMetadataSys.Update(bucket, bucketVersioningConfig, []byte(`Enabled`)) + + _, err = objLayer.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{ + Versioned: true, + }) + if err != nil { + t.Fatal(err) + } + + for _, fsDir := range fsDirs[:4] { + if err = os.Chmod(fsDir, 0400); err != nil { + t.Fatal(err) + } + } + + // Create delete marker under quorum. + objInfo, err := objLayer.DeleteObject(ctx, bucket, object, ObjectOptions{Versioned: true}) + if err != nil { + t.Fatal(err) + } + + for _, fsDir := range fsDirs[:4] { + if err = os.Chmod(fsDir, 0755); err != nil { + t.Fatal(err) + } + } + + fileInfoPreHeal, err := disks[0].ReadVersion(context.Background(), bucket, object, "", false) + if err != nil { + t.Fatal(err) + } + + if fileInfoPreHeal.NumVersions != 1 { + t.Fatalf("Expected versions 1, got %d", fileInfoPreHeal.NumVersions) + } + + if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true}, + func(bucket, object, vid string) error { + _, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{Remove: true}) + return err + }); err != nil { + t.Fatal(err) + } + + fileInfoPostHeal, err := disks[0].ReadVersion(context.Background(), bucket, object, "", false) + if err != nil { + t.Fatal(err) + } + + if fileInfoPostHeal.NumVersions != 2 { + t.Fatalf("Expected versions 2, got %d", fileInfoPreHeal.NumVersions) + } + + if objInfo.DeleteMarker { + if _, err = objLayer.DeleteObject(ctx, bucket, object, ObjectOptions{ + Versioned: true, + VersionID: objInfo.VersionID, + }); err != nil { + t.Fatal(err) + } + } + + for _, fsDir := range fsDirs[:4] { + if err = os.Chmod(fsDir, 0400); err != nil { + t.Fatal(err) + } + } + + rd := mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", "") + _, err = objLayer.PutObject(ctx, bucket, object, rd, ObjectOptions{ + Versioned: true, + }) + if err != nil { + t.Fatal(err) + } + + for _, fsDir := range fsDirs[:4] { + if err = os.Chmod(fsDir, 0755); err != nil { + t.Fatal(err) + } + } + + fileInfoPreHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false) + if err != nil { + t.Fatal(err) + } + + if fileInfoPreHeal.NumVersions != 1 { + t.Fatalf("Expected versions 1, got %d", fileInfoPreHeal.NumVersions) + } + + if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true}, + func(bucket, object, vid string) error { + _, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{Remove: true}) + return err + }); err != nil { + t.Fatal(err) + } + + fileInfoPostHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false) + if err != nil { + t.Fatal(err) + } + + if fileInfoPostHeal.NumVersions != 2 { + t.Fatalf("Expected versions 2, got %d", fileInfoPreHeal.NumVersions) + } +} + func TestHealObjectCorrupted(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -166,8 +311,8 @@ func TestHealObjectCorrupted(t *testing.T) { t.Fatal(err) } - bucket := "bucket" - object := "object" + bucket := getRandomBucketName() + object := getRandomObjectName() data := bytes.Repeat([]byte("a"), 5*1024*1024) var opts ObjectOptions diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 2053360d6..237b147d9 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1668,8 +1668,11 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str } fivs, err := entry.fileInfoVersions(bucket) if err != nil { - errCh <- err - cancel() + if err := healObject(bucket, entry.name, ""); err != nil { + errCh <- err + cancel() + return + } return } @@ -1705,9 +1708,13 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str agreed: healEntry, partial: func(entries metaCacheEntries, nAgreed int, errs []error) { entry, ok := entries.resolve(&resolver) - if ok { - healEntry(*entry) + if !ok { + // check if we can get one entry atleast + // proceed to heal nonetheless. + entry, _ = entries.firstFound() } + + healEntry(*entry) }, finished: nil, } diff --git a/cmd/metacache-entries.go b/cmd/metacache-entries.go index ae1db6eb1..802e03d43 100644 --- a/cmd/metacache-entries.go +++ b/cmd/metacache-entries.go @@ -24,6 +24,7 @@ import ( "sort" "strings" + "github.com/minio/minio/internal/logger" "github.com/minio/pkg/console" ) @@ -104,18 +105,21 @@ func resolveEntries(a, b *metaCacheEntry, bucket string) *metaCacheEntry { return a } - if !aFi.ModTime.Equal(bFi.ModTime) { + if aFi.NumVersions == bFi.NumVersions { + if aFi.ModTime.Equal(bFi.ModTime) { + return a + } if aFi.ModTime.After(bFi.ModTime) { return a } return b } - if aFi.NumVersions > bFi.NumVersions { - return a + if bFi.NumVersions > aFi.NumVersions { + return b } - return b + return a } // isInDir returns whether the entry is in the dir when considering the separator. @@ -269,6 +273,7 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa // Get new entry metadata if _, err := entry.fileInfo(r.bucket); err != nil { + logger.LogIf(context.Background(), err) continue } @@ -318,7 +323,11 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa return nil, false } - if r.candidates[0].n > r.candidates[1].n { + // if r.objQuorum == 1 then it is guaranteed that + // this resolver is for HealObjects(), so use resolveEntries() + // instead to resolve candidates, this check is only useful + // for regular cases of ListObjects() + if r.candidates[0].n > r.candidates[1].n && r.objQuorum > 1 { ok := r.candidates[0].e != nil && r.candidates[0].e.name != "" return r.candidates[0].e, ok } diff --git a/internal/config/api/api.go b/internal/config/api/api.go index 611dc4d93..0715a5465 100644 --- a/internal/config/api/api.go +++ b/internal/config/api/api.go @@ -84,7 +84,7 @@ var ( }, config.KV{ Key: apiListQuorum, - Value: "optimal", + Value: "strict", }, config.KV{ Key: apiReplicationWorkers,