2016-04-22 05:07:47 +02:00
|
|
|
/*
|
2020-06-13 05:04:01 +02:00
|
|
|
* MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
|
2016-04-22 05:07:47 +02:00
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-19 01:23:42 +02:00
|
|
|
package cmd
|
2016-04-08 04:01:15 +02:00
|
|
|
|
|
|
|
import (
|
|
|
|
"time"
|
|
|
|
)
|
|
|
|
|
2020-11-03 02:07:52 +01:00
|
|
|
//go:generate msgp -file=$GOFILE
|
|
|
|
|
2020-11-04 19:10:54 +01:00
|
|
|
// DiskInfo is an extended type which returns current
|
|
|
|
// disk usage per path.
|
2021-01-03 20:27:57 +01:00
|
|
|
//msgp:tuple DiskInfo
|
|
|
|
// The above means that any added/deleted fields are incompatible.
|
2020-11-04 19:10:54 +01:00
|
|
|
type DiskInfo struct {
|
2020-12-27 07:58:06 +01:00
|
|
|
Total uint64
|
|
|
|
Free uint64
|
|
|
|
Used uint64
|
|
|
|
UsedInodes uint64
|
|
|
|
FSType string
|
|
|
|
RootDisk bool
|
|
|
|
Healing bool
|
|
|
|
Endpoint string
|
|
|
|
MountPath string
|
|
|
|
ID string
|
2021-03-17 04:06:57 +01:00
|
|
|
Metrics DiskMetrics
|
2020-12-27 07:58:06 +01:00
|
|
|
Error string // carries the error over the network
|
2020-11-04 19:10:54 +01:00
|
|
|
}
|
|
|
|
|
2021-03-17 04:06:57 +01:00
|
|
|
// DiskMetrics has the information about XL Storage APIs
|
|
|
|
// the number of calls of each API and the moving average of
|
|
|
|
// the duration of each API.
|
|
|
|
type DiskMetrics struct {
|
|
|
|
APILatencies map[string]string `json:"apiLatencies,omitempty"`
|
|
|
|
APICalls map[string]uint64 `json:"apiCalls,omitempty"`
|
|
|
|
}
|
|
|
|
|
2020-11-04 19:10:54 +01:00
|
|
|
// VolsInfo is a collection of volume(bucket) information
|
|
|
|
type VolsInfo []VolInfo
|
|
|
|
|
2016-06-09 15:24:11 +02:00
|
|
|
// VolInfo - represents volume stat information.
|
2021-01-03 20:27:57 +01:00
|
|
|
//msgp:tuple VolInfo
|
|
|
|
// The above means that any added/deleted fields are incompatible.
|
2016-04-08 19:37:38 +02:00
|
|
|
type VolInfo struct {
|
2016-06-09 15:24:11 +02:00
|
|
|
// Name of the volume.
|
|
|
|
Name string
|
|
|
|
|
|
|
|
// Date and time when the volume was created.
|
2016-04-08 19:37:38 +02:00
|
|
|
Created time.Time
|
|
|
|
}
|
|
|
|
|
2019-05-14 22:49:10 +02:00
|
|
|
// FilesInfo represent a list of files, additionally
|
|
|
|
// indicates if the list is last.
|
|
|
|
type FilesInfo struct {
|
|
|
|
Files []FileInfo
|
|
|
|
IsTruncated bool
|
|
|
|
}
|
|
|
|
|
2020-06-13 05:04:01 +02:00
|
|
|
// FilesInfoVersions represents a list of file versions,
|
|
|
|
// additionally indicates if the list is last.
|
|
|
|
type FilesInfoVersions struct {
|
|
|
|
FilesVersions []FileInfoVersions
|
|
|
|
IsTruncated bool
|
|
|
|
}
|
|
|
|
|
|
|
|
// FileInfoVersions represent a list of versions for a given file.
|
|
|
|
type FileInfoVersions struct {
|
|
|
|
// Name of the volume.
|
|
|
|
Volume string
|
|
|
|
|
|
|
|
// Name of the file.
|
|
|
|
Name string
|
|
|
|
|
2021-01-27 11:19:28 +01:00
|
|
|
IsEmptyDir bool
|
|
|
|
|
2020-06-13 05:04:01 +02:00
|
|
|
// Represents the latest mod time of the
|
|
|
|
// latest version.
|
|
|
|
LatestModTime time.Time
|
|
|
|
|
|
|
|
Versions []FileInfo
|
|
|
|
}
|
|
|
|
|
2021-02-11 19:22:03 +01:00
|
|
|
// findVersionIndex will return the version index where the version
|
|
|
|
// was found. Returns -1 if not found.
|
|
|
|
func (f *FileInfoVersions) findVersionIndex(v string) int {
|
|
|
|
if f == nil || v == "" {
|
|
|
|
return -1
|
2020-10-28 17:18:35 +01:00
|
|
|
}
|
|
|
|
for i, ver := range f.Versions {
|
|
|
|
if ver.VersionID == v {
|
2021-02-11 19:22:03 +01:00
|
|
|
return i
|
2020-10-28 17:18:35 +01:00
|
|
|
}
|
|
|
|
}
|
2021-02-11 19:22:03 +01:00
|
|
|
return -1
|
2020-10-28 17:18:35 +01:00
|
|
|
}
|
|
|
|
|
2016-06-09 15:24:11 +02:00
|
|
|
// FileInfo - represents file stat information.
|
2020-11-03 02:07:52 +01:00
|
|
|
//msgp:tuple FileInfo
|
|
|
|
// The above means that any added/deleted fields are incompatible.
|
2016-04-08 04:01:15 +02:00
|
|
|
type FileInfo struct {
|
2016-06-09 15:24:11 +02:00
|
|
|
// Name of the volume.
|
|
|
|
Volume string
|
|
|
|
|
|
|
|
// Name of the file.
|
|
|
|
Name string
|
|
|
|
|
2020-06-13 05:04:01 +02:00
|
|
|
// Version of the file.
|
|
|
|
VersionID string
|
|
|
|
|
|
|
|
// Indicates if the version is the latest
|
|
|
|
IsLatest bool
|
|
|
|
|
|
|
|
// Deleted is set when this FileInfo represents
|
|
|
|
// a deleted marker for a versioned bucket.
|
|
|
|
Deleted bool
|
|
|
|
|
2020-11-12 21:12:09 +01:00
|
|
|
// TransitionStatus is set to Pending/Complete for transitioned
|
|
|
|
// entries based on state of transition
|
|
|
|
TransitionStatus string
|
|
|
|
|
2020-06-13 05:04:01 +02:00
|
|
|
// DataDir of the file
|
|
|
|
DataDir string
|
|
|
|
|
2020-07-18 02:41:29 +02:00
|
|
|
// Indicates if this object is still in V1 format.
|
|
|
|
XLV1 bool
|
|
|
|
|
2020-06-13 05:04:01 +02:00
|
|
|
// Date and time when the file was last modified, if Deleted
|
|
|
|
// is 'true' this value represents when while was deleted.
|
2016-04-08 04:01:15 +02:00
|
|
|
ModTime time.Time
|
2016-06-09 15:24:11 +02:00
|
|
|
|
|
|
|
// Total file size.
|
|
|
|
Size int64
|
|
|
|
|
|
|
|
// File mode bits.
|
2020-11-03 02:07:52 +01:00
|
|
|
Mode uint32
|
2019-05-14 22:49:10 +02:00
|
|
|
|
|
|
|
// File metadata
|
|
|
|
Metadata map[string]string
|
|
|
|
|
|
|
|
// All the parts per object.
|
|
|
|
Parts []ObjectPartInfo
|
|
|
|
|
2020-06-13 05:04:01 +02:00
|
|
|
// Erasure info for all objects.
|
|
|
|
Erasure ErasureInfo
|
2020-11-20 03:43:58 +01:00
|
|
|
|
|
|
|
// DeleteMarkerReplicationStatus is set when this FileInfo represents
|
|
|
|
// replication on a DeleteMarker
|
|
|
|
MarkDeleted bool // mark this version as deleted
|
|
|
|
DeleteMarkerReplicationStatus string
|
|
|
|
VersionPurgeStatus VersionPurgeStatusType
|
2021-01-08 04:27:31 +01:00
|
|
|
|
|
|
|
Data []byte // optionally carries object data
|
2021-02-01 18:52:11 +01:00
|
|
|
|
|
|
|
NumVersions int
|
|
|
|
SuccessorModTime time.Time
|
2020-11-20 03:43:58 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// VersionPurgeStatusKey denotes purge status in metadata
|
|
|
|
const VersionPurgeStatusKey = "purgestatus"
|
|
|
|
|
|
|
|
// VersionPurgeStatusType represents status of a versioned delete or permanent delete w.r.t bucket replication
|
|
|
|
type VersionPurgeStatusType string
|
|
|
|
|
|
|
|
const (
|
|
|
|
// Pending - versioned delete replication is pending.
|
|
|
|
Pending VersionPurgeStatusType = "PENDING"
|
|
|
|
|
|
|
|
// Complete - versioned delete replication is now complete, erase version on disk.
|
|
|
|
Complete VersionPurgeStatusType = "COMPLETE"
|
|
|
|
|
|
|
|
// Failed - versioned delete replication failed.
|
|
|
|
Failed VersionPurgeStatusType = "FAILED"
|
|
|
|
)
|
|
|
|
|
|
|
|
// Empty returns true if purge status was not set.
|
|
|
|
func (v VersionPurgeStatusType) Empty() bool {
|
|
|
|
return string(v) == ""
|
|
|
|
}
|
|
|
|
|
|
|
|
// Pending returns true if the version is pending purge.
|
|
|
|
func (v VersionPurgeStatusType) Pending() bool {
|
|
|
|
return v == Pending || v == Failed
|
2016-04-08 04:01:15 +02:00
|
|
|
}
|
2020-02-25 16:52:28 +01:00
|
|
|
|
2020-06-13 05:04:01 +02:00
|
|
|
// newFileInfo - initializes new FileInfo, allocates a fresh erasure info.
|
|
|
|
func newFileInfo(object string, dataBlocks, parityBlocks int) (fi FileInfo) {
|
|
|
|
fi.Erasure = ErasureInfo{
|
|
|
|
Algorithm: erasureAlgorithm,
|
|
|
|
DataBlocks: dataBlocks,
|
|
|
|
ParityBlocks: parityBlocks,
|
[feat]: change erasure coding default block size from 10MiB to 1MiB (#11721)
major performance improvements in range GETs to avoid large
read amplification when ranges are tiny and random
```
-------------------
Operation: GET
Operations: 142014 -> 339421
Duration: 4m50s -> 4m56s
* Average: +139.41% (+1177.3 MiB/s) throughput, +139.11% (+658.4) obj/s
* Fastest: +125.24% (+1207.4 MiB/s) throughput, +132.32% (+612.9) obj/s
* 50% Median: +139.06% (+1175.7 MiB/s) throughput, +133.46% (+660.9) obj/s
* Slowest: +203.40% (+1267.9 MiB/s) throughput, +198.59% (+753.5) obj/s
```
TTFB from 10MiB BlockSize
```
* First Access TTFB: Avg: 81ms, Median: 61ms, Best: 20ms, Worst: 2.056s
```
TTFB from 1MiB BlockSize
```
* First Access TTFB: Avg: 22ms, Median: 21ms, Best: 8ms, Worst: 91ms
```
Full object reads however do see a slight change which won't be
noticeable in real world, so not doing any comparisons
TTFB still had improvements with full object reads with 1MiB
```
* First Access TTFB: Avg: 68ms, Median: 35ms, Best: 11ms, Worst: 1.16s
```
v/s
TTFB with 10MiB
```
* First Access TTFB: Avg: 388ms, Median: 98ms, Best: 20ms, Worst: 4.156s
```
This change should affect all new uploads, previous uploads should
continue to work with business as usual. But dramatic improvements can
be seen with these changes.
2021-03-06 23:09:34 +01:00
|
|
|
BlockSize: blockSizeV2,
|
2020-06-13 05:04:01 +02:00
|
|
|
Distribution: hashOrder(object, dataBlocks+parityBlocks),
|
2020-02-25 16:52:28 +01:00
|
|
|
}
|
2020-06-13 05:04:01 +02:00
|
|
|
return fi
|
2020-02-25 16:52:28 +01:00
|
|
|
}
|