minio/cmd/storage-datatypes.go
Harshavardhana 9ccc483df6
[feat]: change erasure coding default block size from 10MiB to 1MiB (#11721)
major performance improvements in range GETs to avoid large
read amplification when ranges are tiny and random

```
-------------------
Operation: GET
Operations: 142014 -> 339421
Duration: 4m50s -> 4m56s
* Average: +139.41% (+1177.3 MiB/s) throughput, +139.11% (+658.4) obj/s
* Fastest: +125.24% (+1207.4 MiB/s) throughput, +132.32% (+612.9) obj/s
* 50% Median: +139.06% (+1175.7 MiB/s) throughput, +133.46% (+660.9) obj/s
* Slowest: +203.40% (+1267.9 MiB/s) throughput, +198.59% (+753.5) obj/s
```

TTFB from 10MiB BlockSize
```
* First Access TTFB: Avg: 81ms, Median: 61ms, Best: 20ms, Worst: 2.056s
```

TTFB from 1MiB BlockSize
```
* First Access TTFB: Avg: 22ms, Median: 21ms, Best: 8ms, Worst: 91ms
```

Full object reads however do see a slight change which won't be
noticeable in real world, so not doing any comparisons

TTFB still had improvements with full object reads with 1MiB

```
* First Access TTFB: Avg: 68ms, Median: 35ms, Best: 11ms, Worst: 1.16s
```

v/s

TTFB with 10MiB
```
* First Access TTFB: Avg: 388ms, Median: 98ms, Best: 20ms, Worst: 4.156s
```

This change should affect all new uploads, previous uploads should
continue to work with business as usual. But dramatic improvements can
be seen with these changes.
2021-03-06 14:09:34 -08:00

201 lines
5 KiB
Go

/*
* MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"time"
)
//go:generate msgp -file=$GOFILE
// DiskInfo is an extended type which returns current
// disk usage per path.
//msgp:tuple DiskInfo
// The above means that any added/deleted fields are incompatible.
type DiskInfo struct {
Total uint64
Free uint64
Used uint64
UsedInodes uint64
FSType string
RootDisk bool
Healing bool
Endpoint string
MountPath string
ID string
Error string // carries the error over the network
}
// VolsInfo is a collection of volume(bucket) information
type VolsInfo []VolInfo
// VolInfo - represents volume stat information.
//msgp:tuple VolInfo
// The above means that any added/deleted fields are incompatible.
type VolInfo struct {
// Name of the volume.
Name string
// Date and time when the volume was created.
Created time.Time
}
// FilesInfo represent a list of files, additionally
// indicates if the list is last.
type FilesInfo struct {
Files []FileInfo
IsTruncated bool
}
// FilesInfoVersions represents a list of file versions,
// additionally indicates if the list is last.
type FilesInfoVersions struct {
FilesVersions []FileInfoVersions
IsTruncated bool
}
// FileInfoVersions represent a list of versions for a given file.
type FileInfoVersions struct {
// Name of the volume.
Volume string
// Name of the file.
Name string
IsEmptyDir bool
// Represents the latest mod time of the
// latest version.
LatestModTime time.Time
Versions []FileInfo
}
// findVersionIndex will return the version index where the version
// was found. Returns -1 if not found.
func (f *FileInfoVersions) findVersionIndex(v string) int {
if f == nil || v == "" {
return -1
}
for i, ver := range f.Versions {
if ver.VersionID == v {
return i
}
}
return -1
}
// FileInfo - represents file stat information.
//msgp:tuple FileInfo
// The above means that any added/deleted fields are incompatible.
type FileInfo struct {
// Name of the volume.
Volume string
// Name of the file.
Name string
// Version of the file.
VersionID string
// Indicates if the version is the latest
IsLatest bool
// Deleted is set when this FileInfo represents
// a deleted marker for a versioned bucket.
Deleted bool
// TransitionStatus is set to Pending/Complete for transitioned
// entries based on state of transition
TransitionStatus string
// DataDir of the file
DataDir string
// Indicates if this object is still in V1 format.
XLV1 bool
// Date and time when the file was last modified, if Deleted
// is 'true' this value represents when while was deleted.
ModTime time.Time
// Total file size.
Size int64
// File mode bits.
Mode uint32
// File metadata
Metadata map[string]string
// All the parts per object.
Parts []ObjectPartInfo
// Erasure info for all objects.
Erasure ErasureInfo
// DeleteMarkerReplicationStatus is set when this FileInfo represents
// replication on a DeleteMarker
MarkDeleted bool // mark this version as deleted
DeleteMarkerReplicationStatus string
VersionPurgeStatus VersionPurgeStatusType
Data []byte // optionally carries object data
NumVersions int
SuccessorModTime time.Time
}
// VersionPurgeStatusKey denotes purge status in metadata
const VersionPurgeStatusKey = "purgestatus"
// VersionPurgeStatusType represents status of a versioned delete or permanent delete w.r.t bucket replication
type VersionPurgeStatusType string
const (
// Pending - versioned delete replication is pending.
Pending VersionPurgeStatusType = "PENDING"
// Complete - versioned delete replication is now complete, erase version on disk.
Complete VersionPurgeStatusType = "COMPLETE"
// Failed - versioned delete replication failed.
Failed VersionPurgeStatusType = "FAILED"
)
// Empty returns true if purge status was not set.
func (v VersionPurgeStatusType) Empty() bool {
return string(v) == ""
}
// Pending returns true if the version is pending purge.
func (v VersionPurgeStatusType) Pending() bool {
return v == Pending || v == Failed
}
// newFileInfo - initializes new FileInfo, allocates a fresh erasure info.
func newFileInfo(object string, dataBlocks, parityBlocks int) (fi FileInfo) {
fi.Erasure = ErasureInfo{
Algorithm: erasureAlgorithm,
DataBlocks: dataBlocks,
ParityBlocks: parityBlocks,
BlockSize: blockSizeV2,
Distribution: hashOrder(object, dataBlocks+parityBlocks),
}
return fi
}