diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index fd0064e43..56cbd283a 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -1088,37 +1088,64 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec writeQuorums[i] = getWriteQuorum(len(storageDisks)) } - versions := make([]FileInfo, len(objects)) + versionsMap := make(map[string]FileInfoVersions, len(objects)) for i := range objects { - if objects[i].VersionID == "" { - modTime := opts.MTime - if opts.MTime.IsZero() { - modTime = UTCNow() - } - uuid := opts.VersionID - if uuid == "" { - uuid = mustGetUUID() - } - if opts.Versioned || opts.VersionSuspended { - versions[i] = FileInfo{ - Name: objects[i].ObjectName, - ModTime: modTime, - Deleted: true, // delete marker - ReplicationState: objects[i].ReplicationState(), - } - versions[i].SetTierFreeVersionID(mustGetUUID()) - if opts.Versioned { - versions[i].VersionID = uuid - } - continue - } - } - versions[i] = FileInfo{ + // Construct the FileInfo data that needs to be preserved on the disk. + vr := FileInfo{ Name: objects[i].ObjectName, VersionID: objects[i].VersionID, ReplicationState: objects[i].ReplicationState(), + // save the index to set correct error at this index. + Idx: i, } - versions[i].SetTierFreeVersionID(mustGetUUID()) + vr.SetTierFreeVersionID(mustGetUUID()) + // VersionID is not set means delete is not specific about + // any version, look for if the bucket is versioned or not. + if objects[i].VersionID == "" { + if opts.Versioned || opts.VersionSuspended { + // Bucket is versioned and no version was explicitly + // mentioned for deletes, create a delete marker instead. + vr.ModTime = UTCNow() + vr.Deleted = true + // Versioning suspended means that we add a `null` version + // delete marker, if not add a new version for this delete + // marker. + if opts.Versioned { + vr.VersionID = mustGetUUID() + } + } + } + // De-dup same object name to collect multiple versions for same object. + v, ok := versionsMap[objects[i].ObjectName] + if ok { + v.Versions = append(v.Versions, vr) + } else { + v = FileInfoVersions{ + Name: vr.Name, + Versions: []FileInfo{vr}, + } + } + if vr.Deleted { + dobjects[i] = DeletedObject{ + DeleteMarker: vr.Deleted, + DeleteMarkerVersionID: vr.VersionID, + DeleteMarkerMTime: DeleteMarkerMTime{vr.ModTime}, + ObjectName: vr.Name, + ReplicationState: vr.ReplicationState, + } + } else { + dobjects[i] = DeletedObject{ + ObjectName: vr.Name, + VersionID: vr.VersionID, + ReplicationState: vr.ReplicationState, + } + } + versionsMap[objects[i].ObjectName] = v + } + + dedupVersions := make([]FileInfoVersions, 0, len(versionsMap)) + for _, version := range versionsMap { + dedupVersions = append(dedupVersions, version) } // Initialize list of errors. @@ -1130,17 +1157,24 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec wg.Add(1) go func(index int, disk StorageAPI) { defer wg.Done() + delObjErrs[index] = make([]error, len(objects)) if disk == nil { - delObjErrs[index] = make([]error, len(versions)) - for i := range versions { + for i := range objects { delObjErrs[index][i] = errDiskNotFound } return } - delObjErrs[index] = disk.DeleteVersions(ctx, bucket, versions) + errs := disk.DeleteVersions(ctx, bucket, dedupVersions) + for i, err := range errs { + if err == nil { + continue + } + for _, v := range dedupVersions[i].Versions { + delObjErrs[index][v.Idx] = err + } + } }(index, disk) } - wg.Wait() // Reduce errors for each object @@ -1162,28 +1196,17 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec } if errs[objIndex] == nil { - NSUpdated(bucket, objects[objIndex].ObjectName) - } - - if versions[objIndex].Deleted { - dobjects[objIndex] = DeletedObject{ - DeleteMarker: versions[objIndex].Deleted, - DeleteMarkerVersionID: versions[objIndex].VersionID, - DeleteMarkerMTime: DeleteMarkerMTime{versions[objIndex].ModTime}, - ObjectName: versions[objIndex].Name, - ReplicationState: versions[objIndex].ReplicationState, - } - } else { - dobjects[objIndex] = DeletedObject{ - ObjectName: versions[objIndex].Name, - VersionID: versions[objIndex].VersionID, - ReplicationState: versions[objIndex].ReplicationState, - } + defer NSUpdated(bucket, objects[objIndex].ObjectName) } } // Check failed deletes across multiple objects - for _, version := range versions { + for i, dobj := range dobjects { + // This object errored, no need to attempt a heal. + if errs[i] != nil { + continue + } + // Check if there is any offline disk and add it to the MRF list for _, disk := range storageDisks { if disk != nil && disk.IsOnline() { @@ -1193,7 +1216,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec // all other direct versionId references we should // ensure no dangling file is left over. - er.addPartial(bucket, version.Name, version.VersionID, -1) + er.addPartial(bucket, dobj.ObjectName, dobj.VersionID, -1) break } } diff --git a/cmd/naughty-disk_test.go b/cmd/naughty-disk_test.go index 6ceae732e..335e2bb15 100644 --- a/cmd/naughty-disk_test.go +++ b/cmd/naughty-disk_test.go @@ -225,7 +225,7 @@ func (d *naughtyDisk) Delete(ctx context.Context, volume string, path string, re return d.disk.Delete(ctx, volume, path, recursive) } -func (d *naughtyDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error { +func (d *naughtyDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions) []error { if err := d.calcError(); err != nil { errs := make([]error, len(versions)) for i := range errs { diff --git a/cmd/storage-datatypes.go b/cmd/storage-datatypes.go index c39f857b5..b4e686a75 100644 --- a/cmd/storage-datatypes.go +++ b/cmd/storage-datatypes.go @@ -80,20 +80,20 @@ type FilesInfoVersions struct { } // FileInfoVersions represent a list of versions for a given file. +//msgp:tuple FileInfoVersions +// The above means that any added/deleted fields are incompatible. type FileInfoVersions struct { // Name of the volume. - Volume string + Volume string `msg:"v,omitempty"` // Name of the file. - Name string - - IsEmptyDir bool + Name string `msg:"n,omitempty"` // Represents the latest mod time of the // latest version. - LatestModTime time.Time + LatestModTime time.Time `msg:"lm"` - Versions []FileInfo + Versions []FileInfo `msg:"vs"` } // findVersionIndex will return the version index where the version @@ -115,69 +115,74 @@ func (f *FileInfoVersions) findVersionIndex(v string) int { // The above means that any added/deleted fields are incompatible. type FileInfo struct { // Name of the volume. - Volume string + Volume string `msg:"v,omitempty"` // Name of the file. - Name string + Name string `msg:"n,omitempty"` // Version of the file. - VersionID string + VersionID string `msg:"vid,omitempty"` // Indicates if the version is the latest - IsLatest bool + IsLatest bool `msg:"is"` // Deleted is set when this FileInfo represents // a deleted marker for a versioned bucket. - Deleted bool + Deleted bool `msg:"del"` // TransitionStatus is set to Pending/Complete for transitioned // entries based on state of transition - TransitionStatus string + TransitionStatus string `msg:"ts"` // TransitionedObjName is the object name on the remote tier corresponding // to object (version) on the source tier. - TransitionedObjName string + TransitionedObjName string `msg:"to"` // TransitionTier is the storage class label assigned to remote tier. - TransitionTier string + TransitionTier string `msg:"tt"` // TransitionVersionID stores a version ID of the object associate // with the remote tier. - TransitionVersionID string + TransitionVersionID string `msg:"tv"` // ExpireRestored indicates that the restored object is to be expired. - ExpireRestored bool + ExpireRestored bool `msg:"exp"` // DataDir of the file - DataDir string + DataDir string `msg:"dd"` // Indicates if this object is still in V1 format. - XLV1 bool + XLV1 bool `msg:"v1"` // Date and time when the file was last modified, if Deleted // is 'true' this value represents when while was deleted. - ModTime time.Time + ModTime time.Time `msg:"mt"` // Total file size. - Size int64 + Size int64 `msg:"sz"` // File mode bits. - Mode uint32 + Mode uint32 `msg:"m"` // File metadata - Metadata map[string]string + Metadata map[string]string `msg:"meta"` // All the parts per object. - Parts []ObjectPartInfo + Parts []ObjectPartInfo `msg:"parts"` // Erasure info for all objects. - Erasure ErasureInfo + Erasure ErasureInfo `msg:"ei"` - MarkDeleted bool // mark this version as deleted - ReplicationState ReplicationState // Internal replication state to be passed back in ObjectInfo + MarkDeleted bool `msg:"md"` // mark this version as deleted + ReplicationState ReplicationState `msg:"rs"` // Internal replication state to be passed back in ObjectInfo - Data []byte // optionally carries object data + Data []byte `msg:"d,allownil"` // optionally carries object data - NumVersions int - SuccessorModTime time.Time + NumVersions int `msg:"nv"` + SuccessorModTime time.Time `msg:"smt"` - Fresh bool // indicates this is a first time call to write FileInfo. + Fresh bool `msg:"fr"` // indicates this is a first time call to write FileInfo. + + // Position of this version or object in a multi-object delete call, + // no other caller must set this value other than multi-object delete call. + // usage in other calls in undefined please avoid. + Idx int `msg:"i"` } // InlineData returns true if object contents are inlined alongside its metadata. diff --git a/cmd/storage-datatypes_gen.go b/cmd/storage-datatypes_gen.go index 5ef5d7663..8d0d9ac10 100644 --- a/cmd/storage-datatypes_gen.go +++ b/cmd/storage-datatypes_gen.go @@ -550,8 +550,8 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err) return } - if zb0001 != 24 { - err = msgp.ArrayError{Wanted: 24, Got: zb0001} + if zb0001 != 25 { + err = msgp.ArrayError{Wanted: 25, Got: zb0001} return } z.Volume, err = dc.ReadString() @@ -711,13 +711,18 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "Fresh") return } + z.Idx, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Idx") + return + } return } // EncodeMsg implements msgp.Encodable func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) { - // array header, size 24 - err = en.Append(0xdc, 0x0, 0x18) + // array header, size 25 + err = en.Append(0xdc, 0x0, 0x19) if err != nil { return } @@ -860,14 +865,19 @@ func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Fresh") return } + err = en.WriteInt(z.Idx) + if err != nil { + err = msgp.WrapError(err, "Idx") + return + } return } // MarshalMsg implements msgp.Marshaler func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // array header, size 24 - o = append(o, 0xdc, 0x0, 0x18) + // array header, size 25 + o = append(o, 0xdc, 0x0, 0x19) o = msgp.AppendString(o, z.Volume) o = msgp.AppendString(o, z.Name) o = msgp.AppendString(o, z.VersionID) @@ -911,6 +921,7 @@ func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.AppendInt(o, z.NumVersions) o = msgp.AppendTime(o, z.SuccessorModTime) o = msgp.AppendBool(o, z.Fresh) + o = msgp.AppendInt(o, z.Idx) return } @@ -922,8 +933,8 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err) return } - if zb0001 != 24 { - err = msgp.ArrayError{Wanted: 24, Got: zb0001} + if zb0001 != 25 { + err = msgp.ArrayError{Wanted: 25, Got: zb0001} return } z.Volume, bts, err = msgp.ReadStringBytes(bts) @@ -1083,6 +1094,11 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "Fresh") return } + z.Idx, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Idx") + return + } o = bts return } @@ -1100,87 +1116,62 @@ func (z *FileInfo) Msgsize() (s int) { for za0003 := range z.Parts { s += z.Parts[za0003].Msgsize() } - s += z.Erasure.Msgsize() + msgp.BoolSize + z.ReplicationState.Msgsize() + msgp.BytesPrefixSize + len(z.Data) + msgp.IntSize + msgp.TimeSize + msgp.BoolSize + s += z.Erasure.Msgsize() + msgp.BoolSize + z.ReplicationState.Msgsize() + msgp.BytesPrefixSize + len(z.Data) + msgp.IntSize + msgp.TimeSize + msgp.BoolSize + msgp.IntSize return } // DecodeMsg implements msgp.Decodable func (z *FileInfoVersions) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field var zb0001 uint32 - zb0001, err = dc.ReadMapHeader() + zb0001, err = dc.ReadArrayHeader() if err != nil { err = msgp.WrapError(err) return } - for zb0001 > 0 { - zb0001-- - field, err = dc.ReadMapKeyPtr() + if zb0001 != 4 { + err = msgp.ArrayError{Wanted: 4, Got: zb0001} + return + } + z.Volume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + z.Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + z.LatestModTime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "LatestModTime") + return + } + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + if cap(z.Versions) >= int(zb0002) { + z.Versions = (z.Versions)[:zb0002] + } else { + z.Versions = make([]FileInfo, zb0002) + } + for za0001 := range z.Versions { + err = z.Versions[za0001].DecodeMsg(dc) if err != nil { - err = msgp.WrapError(err) + err = msgp.WrapError(err, "Versions", za0001) return } - switch msgp.UnsafeString(field) { - case "Volume": - z.Volume, err = dc.ReadString() - if err != nil { - err = msgp.WrapError(err, "Volume") - return - } - case "Name": - z.Name, err = dc.ReadString() - if err != nil { - err = msgp.WrapError(err, "Name") - return - } - case "IsEmptyDir": - z.IsEmptyDir, err = dc.ReadBool() - if err != nil { - err = msgp.WrapError(err, "IsEmptyDir") - return - } - case "LatestModTime": - z.LatestModTime, err = dc.ReadTime() - if err != nil { - err = msgp.WrapError(err, "LatestModTime") - return - } - case "Versions": - var zb0002 uint32 - zb0002, err = dc.ReadArrayHeader() - if err != nil { - err = msgp.WrapError(err, "Versions") - return - } - if cap(z.Versions) >= int(zb0002) { - z.Versions = (z.Versions)[:zb0002] - } else { - z.Versions = make([]FileInfo, zb0002) - } - for za0001 := range z.Versions { - err = z.Versions[za0001].DecodeMsg(dc) - if err != nil { - err = msgp.WrapError(err, "Versions", za0001) - return - } - } - default: - err = dc.Skip() - if err != nil { - err = msgp.WrapError(err) - return - } - } } return } // EncodeMsg implements msgp.Encodable func (z *FileInfoVersions) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 5 - // write "Volume" - err = en.Append(0x85, 0xa6, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65) + // array header, size 4 + err = en.Append(0x94) if err != nil { return } @@ -1189,41 +1180,16 @@ func (z *FileInfoVersions) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Volume") return } - // write "Name" - err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65) - if err != nil { - return - } err = en.WriteString(z.Name) if err != nil { err = msgp.WrapError(err, "Name") return } - // write "IsEmptyDir" - err = en.Append(0xaa, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x44, 0x69, 0x72) - if err != nil { - return - } - err = en.WriteBool(z.IsEmptyDir) - if err != nil { - err = msgp.WrapError(err, "IsEmptyDir") - return - } - // write "LatestModTime" - err = en.Append(0xad, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) - if err != nil { - return - } err = en.WriteTime(z.LatestModTime) if err != nil { err = msgp.WrapError(err, "LatestModTime") return } - // write "Versions" - err = en.Append(0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73) - if err != nil { - return - } err = en.WriteArrayHeader(uint32(len(z.Versions))) if err != nil { err = msgp.WrapError(err, "Versions") @@ -1242,21 +1208,11 @@ func (z *FileInfoVersions) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *FileInfoVersions) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 5 - // string "Volume" - o = append(o, 0x85, 0xa6, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65) + // array header, size 4 + o = append(o, 0x94) o = msgp.AppendString(o, z.Volume) - // string "Name" - o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65) o = msgp.AppendString(o, z.Name) - // string "IsEmptyDir" - o = append(o, 0xaa, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x44, 0x69, 0x72) - o = msgp.AppendBool(o, z.IsEmptyDir) - // string "LatestModTime" - o = append(o, 0xad, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) o = msgp.AppendTime(o, z.LatestModTime) - // string "Versions" - o = append(o, 0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73) o = msgp.AppendArrayHeader(o, uint32(len(z.Versions))) for za0001 := range z.Versions { o, err = z.Versions[za0001].MarshalMsg(o) @@ -1270,72 +1226,48 @@ func (z *FileInfoVersions) MarshalMsg(b []byte) (o []byte, err error) { // UnmarshalMsg implements msgp.Unmarshaler func (z *FileInfoVersions) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field var zb0001 uint32 - zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) if err != nil { err = msgp.WrapError(err) return } - for zb0001 > 0 { - zb0001-- - field, bts, err = msgp.ReadMapKeyZC(bts) + if zb0001 != 4 { + err = msgp.ArrayError{Wanted: 4, Got: zb0001} + return + } + z.Volume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + z.LatestModTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LatestModTime") + return + } + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + if cap(z.Versions) >= int(zb0002) { + z.Versions = (z.Versions)[:zb0002] + } else { + z.Versions = make([]FileInfo, zb0002) + } + for za0001 := range z.Versions { + bts, err = z.Versions[za0001].UnmarshalMsg(bts) if err != nil { - err = msgp.WrapError(err) + err = msgp.WrapError(err, "Versions", za0001) return } - switch msgp.UnsafeString(field) { - case "Volume": - z.Volume, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - err = msgp.WrapError(err, "Volume") - return - } - case "Name": - z.Name, bts, err = msgp.ReadStringBytes(bts) - if err != nil { - err = msgp.WrapError(err, "Name") - return - } - case "IsEmptyDir": - z.IsEmptyDir, bts, err = msgp.ReadBoolBytes(bts) - if err != nil { - err = msgp.WrapError(err, "IsEmptyDir") - return - } - case "LatestModTime": - z.LatestModTime, bts, err = msgp.ReadTimeBytes(bts) - if err != nil { - err = msgp.WrapError(err, "LatestModTime") - return - } - case "Versions": - var zb0002 uint32 - zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err, "Versions") - return - } - if cap(z.Versions) >= int(zb0002) { - z.Versions = (z.Versions)[:zb0002] - } else { - z.Versions = make([]FileInfo, zb0002) - } - for za0001 := range z.Versions { - bts, err = z.Versions[za0001].UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Versions", za0001) - return - } - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - } } o = bts return @@ -1343,7 +1275,7 @@ func (z *FileInfoVersions) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *FileInfoVersions) Msgsize() (s int) { - s = 1 + 7 + msgp.StringPrefixSize + len(z.Volume) + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.BoolSize + 14 + msgp.TimeSize + 9 + msgp.ArrayHeaderSize + s = 1 + msgp.StringPrefixSize + len(z.Volume) + msgp.StringPrefixSize + len(z.Name) + msgp.TimeSize + msgp.ArrayHeaderSize for za0001 := range z.Versions { s += z.Versions[za0001].Msgsize() } diff --git a/cmd/storage-interface.go b/cmd/storage-interface.go index d17352c11..5f0aa5327 100644 --- a/cmd/storage-interface.go +++ b/cmd/storage-interface.go @@ -57,7 +57,7 @@ type StorageAPI interface { // Metadata operations DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool) error - DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error + DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions) []error WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) error UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo) error ReadVersion(ctx context.Context, volume, path, versionID string, readData bool) (FileInfo, error) diff --git a/cmd/storage-rest-client.go b/cmd/storage-rest-client.go index 4635a9b16..1a3bd1419 100644 --- a/cmd/storage-rest-client.go +++ b/cmd/storage-rest-client.go @@ -590,7 +590,7 @@ func (client *storageRESTClient) Delete(ctx context.Context, volume string, path } // DeleteVersions - deletes list of specified versions if present -func (client *storageRESTClient) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) (errs []error) { +func (client *storageRESTClient) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions) (errs []error) { if len(versions) == 0 { return errs } diff --git a/cmd/storage-rest-common.go b/cmd/storage-rest-common.go index af7e82e48..6229a5eed 100644 --- a/cmd/storage-rest-common.go +++ b/cmd/storage-rest-common.go @@ -18,7 +18,7 @@ package cmd const ( - storageRESTVersion = "v40" // Add ReplicationState field + storageRESTVersion = "v41" // Optimized DeleteVersions API storageRESTVersionPrefix = SlashSeparator + storageRESTVersion storageRESTPrefix = minioReservedBucketPath + "/storage" ) diff --git a/cmd/storage-rest-server.go b/cmd/storage-rest-server.go index 66d68ea5e..380b0d00e 100644 --- a/cmd/storage-rest-server.go +++ b/cmd/storage-rest-server.go @@ -643,7 +643,7 @@ func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http return } - versions := make([]FileInfo, totalVersions) + versions := make([]FileInfoVersions, totalVersions) decoder := msgp.NewReader(r.Body) for i := 0; i < totalVersions; i++ { dst := &versions[i] diff --git a/cmd/xl-storage-disk-id-check.go b/cmd/xl-storage-disk-id-check.go index 241002be5..1f08cb2ab 100644 --- a/cmd/xl-storage-disk-id-check.go +++ b/cmd/xl-storage-disk-id-check.go @@ -421,8 +421,8 @@ func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path s // DeleteVersions deletes slice of versions, it can be same object // or multiple objects. -func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) (errs []error) { - // Mererly for tracing storage +func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions) (errs []error) { + // Merely for tracing storage path := "" if len(versions) > 0 { path = versions[0].Name diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 831cdc530..13abf1285 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -821,13 +821,102 @@ func (s *xlStorage) ListDir(ctx context.Context, volume, dirPath string, count i return entries, nil } +func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis ...FileInfo) error { + buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile)) + if err != nil { + if err != errFileNotFound { + return err + } + metaDataPoolPut(buf) // Never used, return it + + buf, err = s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFileV1)) + if err != nil { + return err + } + } + + if len(buf) == 0 { + return errFileNotFound + } + + volumeDir, err := s.getVolDir(volume) + if err != nil { + return err + } + + if !isXL2V1Format(buf) { + // Delete the meta file, if there are no more versions the + // top level parent is automatically removed. + return s.deleteFile(volumeDir, pathJoin(volumeDir, path), true) + } + + var xlMeta xlMetaV2 + if err = xlMeta.Load(buf); err != nil { + return err + } + + var ( + dataDir string + lastVersion bool + ) + + for _, fi := range fis { + dataDir, lastVersion, err = xlMeta.DeleteVersion(fi) + if err != nil { + return err + } + if dataDir != "" { + versionID := fi.VersionID + if versionID == "" { + versionID = nullVersionID + } + // PR #11758 used DataDir, preserve it + // for users who might have used master + // branch + if !xlMeta.data.remove(versionID, dataDir) { + filePath := pathJoin(volumeDir, path, dataDir) + if err = checkPathLength(filePath); err != nil { + return err + } + if err = s.moveToTrash(filePath, true); err != nil { + if err != errFileNotFound { + return err + } + } + } + } + } + + if !lastVersion { + buf, err = xlMeta.AppendTo(metaDataPoolGet()) + defer metaDataPoolPut(buf) + if err != nil { + return err + } + + return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), buf) + } + + // Move xl.meta to trash + filePath := pathJoin(volumeDir, path, xlStorageFormatFile) + if err = checkPathLength(filePath); err != nil { + return err + } + + err = s.moveToTrash(filePath, false) + if err == nil || err == errFileNotFound { + s.deleteFile(volumeDir, pathJoin(volumeDir, path), false) + } + return err +} + // DeleteVersions deletes slice of versions, it can be same object // or multiple objects. -func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error { +func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions) []error { errs := make([]error, len(versions)) - for i, version := range versions { - if err := s.DeleteVersion(ctx, volume, version.Name, version, false); err != nil { + for i, fiv := range versions { + if err := s.deleteVersions(ctx, volume, fiv.Name, fiv.Versions...); err != nil { errs[i] = err } } diff --git a/go.mod b/go.mod index e5ce2da73..6d21cc496 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,7 @@ require ( github.com/nats-io/stan.go v0.8.3 github.com/ncw/directio v1.0.5 github.com/nsqio/go-nsq v1.0.8 - github.com/philhofer/fwd v1.1.1 + github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9 github.com/pierrec/lz4 v2.6.0+incompatible github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 @@ -74,7 +74,7 @@ require ( github.com/secure-io/sio-go v0.3.1 github.com/shirou/gopsutil/v3 v3.21.9 github.com/streadway/amqp v1.0.0 - github.com/tinylib/msgp v1.1.6 + github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e github.com/valyala/bytebufferpool v1.0.0 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c github.com/yargevad/filepathx v1.0.0 diff --git a/go.sum b/go.sum index 7b1a4ef80..7dde04f90 100644 --- a/go.sum +++ b/go.sum @@ -1218,8 +1218,9 @@ github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bA github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw= -github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9 h1:6ob53CVz+ja2i7easAStApZJlh7sxyq3Cm7g1Di6iqA= +github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -1430,8 +1431,9 @@ github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiff github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tinylib/msgp v1.1.3/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= -github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw= github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= +github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e h1:P5tyWbssToKowBPTA1/EzqPXwrZNc8ZeNPdjgpcDEoI= +github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e/go.mod h1:g7jEyb18KPe65d9RRhGw+ThaJr5duyBH8eaFgBUor7Y= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=