2016-07-22 02:31:14 +02:00
|
|
|
/*
|
2019-04-09 20:39:42 +02:00
|
|
|
* MinIO Cloud Storage, (C) 2016, 2017, 2017 MinIO, Inc.
|
2016-07-22 02:31:14 +02:00
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-19 01:23:42 +02:00
|
|
|
package cmd
|
2016-05-21 05:48:47 +02:00
|
|
|
|
|
|
|
import (
|
2018-04-06 00:04:40 +02:00
|
|
|
"context"
|
2018-03-28 02:23:10 +02:00
|
|
|
"encoding/hex"
|
|
|
|
"encoding/json"
|
2017-01-17 02:05:00 +01:00
|
|
|
"io"
|
|
|
|
"io/ioutil"
|
2019-02-28 20:01:25 +01:00
|
|
|
"net/http"
|
2017-01-17 02:05:00 +01:00
|
|
|
"os"
|
|
|
|
pathutil "path"
|
2019-02-28 20:01:25 +01:00
|
|
|
"time"
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2019-09-06 00:51:27 +02:00
|
|
|
jsoniter "github.com/json-iterator/go"
|
2019-10-07 07:50:24 +02:00
|
|
|
xhttp "github.com/minio/minio/cmd/http"
|
2018-04-06 00:04:40 +02:00
|
|
|
"github.com/minio/minio/cmd/logger"
|
2017-01-17 02:05:00 +01:00
|
|
|
"github.com/minio/minio/pkg/lock"
|
|
|
|
"github.com/minio/minio/pkg/mimedb"
|
2016-05-21 05:48:47 +02:00
|
|
|
)
|
|
|
|
|
2017-05-14 21:05:51 +02:00
|
|
|
// FS format, and object metadata.
|
2016-05-25 06:24:20 +02:00
|
|
|
const (
|
2017-05-14 21:05:51 +02:00
|
|
|
// fs.json object metadata.
|
|
|
|
fsMetaJSONFile = "fs.json"
|
2016-05-25 06:24:20 +02:00
|
|
|
)
|
|
|
|
|
2017-05-14 21:05:51 +02:00
|
|
|
// FS metadata constants.
|
|
|
|
const (
|
|
|
|
// FS backend meta 1.0.0 version.
|
|
|
|
fsMetaVersion100 = "1.0.0"
|
|
|
|
|
|
|
|
// FS backend meta 1.0.1 version.
|
2018-03-28 02:23:10 +02:00
|
|
|
fsMetaVersion101 = "1.0.1"
|
2017-05-14 21:05:51 +02:00
|
|
|
|
2018-03-28 02:23:10 +02:00
|
|
|
// FS backend meta 1.0.2
|
2019-04-09 20:39:42 +02:00
|
|
|
// Removed the fields "Format" and "MinIO" from fsMetaV1 as they were unused. Added "Checksum" field - to be used in future for bit-rot protection.
|
2018-03-28 02:23:10 +02:00
|
|
|
fsMetaVersion = "1.0.2"
|
2017-05-14 21:05:51 +02:00
|
|
|
|
|
|
|
// Add more constants here.
|
|
|
|
)
|
|
|
|
|
2018-03-28 02:23:10 +02:00
|
|
|
// FSChecksumInfoV1 - carries checksums of individual blocks on disk.
|
|
|
|
type FSChecksumInfoV1 struct {
|
|
|
|
Algorithm string
|
|
|
|
Blocksize int64
|
|
|
|
Hashes [][]byte
|
|
|
|
}
|
|
|
|
|
|
|
|
// MarshalJSON marshals the FSChecksumInfoV1 struct
|
|
|
|
func (c FSChecksumInfoV1) MarshalJSON() ([]byte, error) {
|
|
|
|
type checksuminfo struct {
|
|
|
|
Algorithm string `json:"algorithm"`
|
|
|
|
Blocksize int64 `json:"blocksize"`
|
|
|
|
Hashes []string `json:"hashes"`
|
|
|
|
}
|
|
|
|
var hashes []string
|
|
|
|
for _, h := range c.Hashes {
|
|
|
|
hashes = append(hashes, hex.EncodeToString(h))
|
|
|
|
}
|
|
|
|
info := checksuminfo{
|
|
|
|
Algorithm: c.Algorithm,
|
|
|
|
Hashes: hashes,
|
|
|
|
Blocksize: c.Blocksize,
|
|
|
|
}
|
|
|
|
return json.Marshal(info)
|
|
|
|
}
|
|
|
|
|
|
|
|
// UnmarshalJSON unmarshals the the given data into the FSChecksumInfoV1 struct
|
|
|
|
func (c *FSChecksumInfoV1) UnmarshalJSON(data []byte) error {
|
|
|
|
type checksuminfo struct {
|
|
|
|
Algorithm string `json:"algorithm"`
|
|
|
|
Blocksize int64 `json:"blocksize"`
|
|
|
|
Hashes []string `json:"hashes"`
|
|
|
|
}
|
|
|
|
|
|
|
|
var info checksuminfo
|
|
|
|
err := json.Unmarshal(data, &info)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
c.Algorithm = info.Algorithm
|
|
|
|
c.Blocksize = info.Blocksize
|
|
|
|
var hashes [][]byte
|
|
|
|
for _, hashStr := range info.Hashes {
|
|
|
|
h, err := hex.DecodeString(hashStr)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
hashes = append(hashes, h)
|
|
|
|
}
|
|
|
|
c.Hashes = hashes
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-05-21 05:48:47 +02:00
|
|
|
// A fsMetaV1 represents a metadata header mapping keys to sets of values.
|
|
|
|
type fsMetaV1 struct {
|
|
|
|
Version string `json:"version"`
|
2018-03-28 02:23:10 +02:00
|
|
|
// checksums of blocks on disk.
|
|
|
|
Checksum FSChecksumInfoV1 `json:"checksum,omitempty"`
|
|
|
|
// Metadata map for current object.
|
|
|
|
Meta map[string]string `json:"meta,omitempty"`
|
|
|
|
// parts info for current object - used in encryption.
|
2019-01-05 23:16:43 +01:00
|
|
|
Parts []ObjectPartInfo `json:"parts,omitempty"`
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
|
|
|
|
2017-05-14 21:05:51 +02:00
|
|
|
// IsValid - tells if the format is sane by validating the version
|
|
|
|
// string and format style.
|
|
|
|
func (m fsMetaV1) IsValid() bool {
|
2018-03-28 02:23:10 +02:00
|
|
|
return isFSMetaValid(m.Version)
|
2017-05-14 21:05:51 +02:00
|
|
|
}
|
|
|
|
|
2018-03-28 02:23:10 +02:00
|
|
|
// Verifies if the backend format metadata is same by validating
|
|
|
|
// the version string.
|
|
|
|
func isFSMetaValid(version string) bool {
|
|
|
|
return (version == fsMetaVersion || version == fsMetaVersion100 || version == fsMetaVersion101)
|
2017-05-14 21:05:51 +02:00
|
|
|
}
|
|
|
|
|
2017-01-17 02:05:00 +01:00
|
|
|
// Converts metadata to object info.
|
|
|
|
func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo {
|
|
|
|
if len(m.Meta) == 0 {
|
|
|
|
m.Meta = make(map[string]string)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Guess content-type from the extension if possible.
|
|
|
|
if m.Meta["content-type"] == "" {
|
2018-10-02 08:18:17 +02:00
|
|
|
m.Meta["content-type"] = mimedb.TypeByExtension(pathutil.Ext(object))
|
2017-01-17 02:05:00 +01:00
|
|
|
}
|
|
|
|
|
2019-12-06 08:16:06 +01:00
|
|
|
if HasSuffix(object, SlashSeparator) {
|
2017-10-17 02:20:54 +02:00
|
|
|
m.Meta["etag"] = emptyETag // For directories etag is d41d8cd98f00b204e9800998ecf8427e
|
|
|
|
m.Meta["content-type"] = "application/octet-stream"
|
|
|
|
}
|
|
|
|
|
2017-01-17 02:05:00 +01:00
|
|
|
objInfo := ObjectInfo{
|
|
|
|
Bucket: bucket,
|
|
|
|
Name: object,
|
|
|
|
}
|
|
|
|
|
2017-04-04 18:14:03 +02:00
|
|
|
// We set file info only if its valid.
|
2017-01-17 02:05:00 +01:00
|
|
|
objInfo.ModTime = timeSentinel
|
|
|
|
if fi != nil {
|
|
|
|
objInfo.ModTime = fi.ModTime()
|
|
|
|
objInfo.Size = fi.Size()
|
2017-10-17 02:20:54 +02:00
|
|
|
if fi.IsDir() {
|
|
|
|
// Directory is always 0 bytes in S3 API, treat it as such.
|
|
|
|
objInfo.Size = 0
|
|
|
|
objInfo.IsDir = fi.IsDir()
|
|
|
|
}
|
2017-01-17 02:05:00 +01:00
|
|
|
}
|
|
|
|
|
2017-05-14 21:05:51 +02:00
|
|
|
objInfo.ETag = extractETag(m.Meta)
|
2017-01-17 02:05:00 +01:00
|
|
|
objInfo.ContentType = m.Meta["content-type"]
|
|
|
|
objInfo.ContentEncoding = m.Meta["content-encoding"]
|
2019-10-07 07:50:24 +02:00
|
|
|
if storageClass, ok := m.Meta[xhttp.AmzStorageClass]; ok {
|
2018-04-06 17:20:02 +02:00
|
|
|
objInfo.StorageClass = storageClass
|
|
|
|
} else {
|
|
|
|
objInfo.StorageClass = globalMinioDefaultStorageClass
|
|
|
|
}
|
2019-02-28 20:01:25 +01:00
|
|
|
var (
|
|
|
|
t time.Time
|
|
|
|
e error
|
|
|
|
)
|
|
|
|
if exp, ok := m.Meta["expires"]; ok {
|
|
|
|
if t, e = time.Parse(http.TimeFormat, exp); e == nil {
|
|
|
|
objInfo.Expires = t.UTC()
|
|
|
|
}
|
|
|
|
}
|
2017-05-14 21:05:51 +02:00
|
|
|
// etag/md5Sum has already been extracted. We need to
|
|
|
|
// remove to avoid it from appearing as part of
|
|
|
|
// response headers. e.g, X-Minio-* or X-Amz-*.
|
2018-01-04 07:14:45 +01:00
|
|
|
objInfo.UserDefined = cleanMetadata(m.Meta)
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2018-03-01 20:37:57 +01:00
|
|
|
// All the parts per object.
|
|
|
|
objInfo.Parts = m.Parts
|
|
|
|
|
2017-01-17 02:05:00 +01:00
|
|
|
// Success..
|
|
|
|
return objInfo
|
|
|
|
}
|
|
|
|
|
2017-01-25 21:29:06 +01:00
|
|
|
func (m *fsMetaV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
|
2018-02-07 00:37:48 +01:00
|
|
|
if err = jsonSave(lk, m); err != nil {
|
|
|
|
return 0, err
|
2017-01-17 02:05:00 +01:00
|
|
|
}
|
2018-02-07 00:37:48 +01:00
|
|
|
fi, err := lk.Stat()
|
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
2018-02-07 00:37:48 +01:00
|
|
|
return fi.Size(), nil
|
2016-05-21 05:48:47 +02:00
|
|
|
}
|
|
|
|
|
2018-04-06 00:04:40 +02:00
|
|
|
func (m *fsMetaV1) ReadFrom(ctx context.Context, lk *lock.LockedFile) (n int64, err error) {
|
2017-04-04 18:14:03 +02:00
|
|
|
var fsMetaBuf []byte
|
2017-01-25 21:29:06 +01:00
|
|
|
fi, err := lk.Stat()
|
|
|
|
if err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return 0, err
|
2017-01-25 21:29:06 +01:00
|
|
|
}
|
|
|
|
|
2017-04-04 18:14:03 +02:00
|
|
|
fsMetaBuf, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
|
2016-08-31 22:42:57 +02:00
|
|
|
if err != nil {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.LogIf(ctx, err)
|
|
|
|
return 0, err
|
2016-08-31 22:42:57 +02:00
|
|
|
}
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2017-04-04 18:14:03 +02:00
|
|
|
if len(fsMetaBuf) == 0 {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.LogIf(ctx, io.EOF)
|
|
|
|
return 0, io.EOF
|
2016-08-25 18:39:01 +02:00
|
|
|
}
|
2017-01-17 02:05:00 +01:00
|
|
|
|
2019-09-06 00:51:27 +02:00
|
|
|
var json = jsoniter.ConfigCompatibleWithStandardLibrary
|
|
|
|
if err = json.Unmarshal(fsMetaBuf, m); err != nil {
|
2019-08-19 20:35:52 +02:00
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
|
2017-05-14 21:05:51 +02:00
|
|
|
// Verify if the format is valid, return corrupted format
|
|
|
|
// for unrecognized formats.
|
2018-03-28 02:23:10 +02:00
|
|
|
if !isFSMetaValid(m.Version) {
|
2018-04-06 00:04:40 +02:00
|
|
|
logger.GetReqInfo(ctx).AppendTags("file", lk.Name())
|
|
|
|
logger.LogIf(ctx, errCorruptedFormat)
|
|
|
|
return 0, errCorruptedFormat
|
2017-05-14 21:05:51 +02:00
|
|
|
}
|
|
|
|
|
2017-01-17 02:05:00 +01:00
|
|
|
// Success.
|
2017-04-04 18:14:03 +02:00
|
|
|
return int64(len(fsMetaBuf)), nil
|
2016-08-31 22:42:57 +02:00
|
|
|
}
|
|
|
|
|
2016-05-26 12:15:01 +02:00
|
|
|
// newFSMetaV1 - initializes new fsMetaV1.
|
|
|
|
func newFSMetaV1() (fsMeta fsMetaV1) {
|
|
|
|
fsMeta = fsMetaV1{}
|
2017-01-18 21:24:34 +01:00
|
|
|
fsMeta.Version = fsMetaVersion
|
2016-05-26 12:15:01 +02:00
|
|
|
return fsMeta
|
|
|
|
}
|