fix: implement readMetadata common function for re-use (#12353)
Previous PR #12351 added functions to read from the reader stream to reduce memory usage, use the same technique in few other places where we are not interested in reading the data part.
This commit is contained in:
Harshavardhana
GitHub
parent
9d1b6fb37d
commit
0287711dc9
4
Makefile
4
Makefile
|
|
@ -17,8 +17,8 @@ checks:
|
|||
getdeps:
|
||||
@mkdir -p ${GOPATH}/bin
|
||||
@which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0)
|
||||
@which msgp 1>/dev/null || (echo "Installing msgp" && go get github.com/tinylib/msgp@v1.1.3)
|
||||
@which stringer 1>/dev/null || (echo "Installing stringer" && go get golang.org/x/tools/cmd/stringer)
|
||||
@which msgp 1>/dev/null || (echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.3)
|
||||
@which stringer 1>/dev/null || (echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer)
|
||||
|
||||
crosscompile:
|
||||
@(env bash $(PWD)/buildscripts/cross-compile.sh)
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
|
|||
// Fast exit track to check if we are listing an object with
|
||||
// a trailing slash, this will avoid to list the object content.
|
||||
if HasSuffix(opts.BaseDir, SlashSeparator) {
|
||||
metadata, err := xioutil.ReadFile(pathJoin(volumeDir,
|
||||
metadata, err := s.readMetadata(pathJoin(volumeDir,
|
||||
opts.BaseDir[:len(opts.BaseDir)-1]+globalDirSuffix,
|
||||
xlStorageFormatFile))
|
||||
if err == nil {
|
||||
|
|
@ -97,7 +97,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
|
|||
// behavior.
|
||||
out <- metaCacheEntry{
|
||||
name: opts.BaseDir,
|
||||
metadata: xlMetaV2TrimData(metadata),
|
||||
metadata: metadata,
|
||||
}
|
||||
} else {
|
||||
if st, err := os.Lstat(pathJoin(volumeDir, opts.BaseDir, xlStorageFormatFile)); err == nil && st.Mode().IsRegular() {
|
||||
|
|
@ -154,25 +154,11 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
|
|||
// If root was an object return it as such.
|
||||
if HasSuffix(entry, xlStorageFormatFile) {
|
||||
var meta metaCacheEntry
|
||||
f, err := os.OpenFile(pathJoin(volumeDir, current, entry), readMode, 0)
|
||||
meta.metadata, err = s.readMetadata(pathJoin(volumeDir, current, entry))
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
continue
|
||||
}
|
||||
stat, err := f.Stat()
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
f.Close()
|
||||
continue
|
||||
}
|
||||
meta.metadata, err = readXLMetaNoData(f, stat.Size())
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
f.Close()
|
||||
continue
|
||||
}
|
||||
f.Close()
|
||||
meta.metadata = xlMetaV2TrimData(meta.metadata)
|
||||
meta.name = strings.TrimSuffix(entry, xlStorageFormatFile)
|
||||
meta.name = strings.TrimSuffix(meta.name, SlashSeparator)
|
||||
meta.name = pathJoin(current, meta.name)
|
||||
|
|
@ -233,7 +219,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
|
|||
meta.name = meta.name[:len(meta.name)-1] + globalDirSuffixWithSlash
|
||||
}
|
||||
|
||||
meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFile))
|
||||
meta.metadata, err = s.readMetadata(pathJoin(volumeDir, meta.name, xlStorageFormatFile))
|
||||
switch {
|
||||
case err == nil:
|
||||
// It was an object
|
||||
|
|
|
|||
|
|
@ -1420,7 +1420,7 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) {
|
|||
err = readMore(size)
|
||||
return buf, err
|
||||
case 1, 2:
|
||||
sz, tmp, err := ReadBytesHeader(tmp)
|
||||
sz, tmp, err := msgp.ReadBytesHeader(tmp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -1459,51 +1459,3 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) {
|
|||
return nil, errors.New("unknown major metadata version")
|
||||
}
|
||||
}
|
||||
|
||||
// ReadBytesHeader reads the 'bin' header size
|
||||
// off of 'b' and returns the size and remaining bytes.
|
||||
// Possible errors:
|
||||
// - ErrShortBytes (too few bytes)
|
||||
// - TypeError{} (not a bin object)
|
||||
// TODO: Replace when https://github.com/tinylib/msgp/pull/289 is merged.
|
||||
func ReadBytesHeader(b []byte) (sz uint32, o []byte, err error) {
|
||||
if len(b) < 1 {
|
||||
return 0, nil, msgp.ErrShortBytes
|
||||
}
|
||||
var big = binary.BigEndian
|
||||
|
||||
const (
|
||||
mbin8 uint8 = 0xc4
|
||||
mbin16 uint8 = 0xc5
|
||||
mbin32 uint8 = 0xc6
|
||||
)
|
||||
switch b[0] {
|
||||
case mbin8:
|
||||
if len(b) < 2 {
|
||||
err = msgp.ErrShortBytes
|
||||
return
|
||||
}
|
||||
sz = uint32(b[1])
|
||||
o = b[2:]
|
||||
return
|
||||
case mbin16:
|
||||
if len(b) < 3 {
|
||||
err = msgp.ErrShortBytes
|
||||
return
|
||||
}
|
||||
sz = uint32(big.Uint16(b[1:]))
|
||||
o = b[3:]
|
||||
return
|
||||
case mbin32:
|
||||
if len(b) < 5 {
|
||||
err = msgp.ErrShortBytes
|
||||
return
|
||||
}
|
||||
sz = big.Uint32(b[1:])
|
||||
o = b[5:]
|
||||
return
|
||||
default:
|
||||
err = msgp.TypeError{Method: msgp.BinType, Encoded: msgp.NextType(b)}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -384,6 +384,19 @@ func (s *xlStorage) Healing() *healingTracker {
|
|||
return &h
|
||||
}
|
||||
|
||||
func (s *xlStorage) readMetadata(itemPath string) ([]byte, error) {
|
||||
f, err := os.OpenFile(itemPath, readMode, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
stat, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return readXLMetaNoData(f, stat.Size())
|
||||
}
|
||||
|
||||
func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry) (dataUsageCache, error) {
|
||||
// Updates must be closed before we return.
|
||||
defer close(updates)
|
||||
|
|
@ -421,29 +434,13 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
|
|||
return sizeSummary{}, errSkipFile
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(item.Path, readMode, 0)
|
||||
buf, err := s.readMetadata(item.Path)
|
||||
if err != nil {
|
||||
if intDataUpdateTracker.debug {
|
||||
console.Debugf(color.Green("scannerBucket:")+" object path missing: %v: %w\n", item.Path, err)
|
||||
}
|
||||
return sizeSummary{}, errSkipFile
|
||||
}
|
||||
defer f.Close()
|
||||
stat, err := f.Stat()
|
||||
if err != nil {
|
||||
if intDataUpdateTracker.debug {
|
||||
console.Debugf(color.Green("scannerBucket:")+" stat failed: %v: %w\n", item.Path, err)
|
||||
}
|
||||
return sizeSummary{}, errSkipFile
|
||||
}
|
||||
|
||||
buf, err := readXLMetaNoData(f, stat.Size())
|
||||
if err != nil {
|
||||
if intDataUpdateTracker.debug {
|
||||
console.Debugf(color.Green("scannerBucket:")+" readXLMetaNoData: %v: %w\n", item.Path, err)
|
||||
}
|
||||
return sizeSummary{}, errSkipFile
|
||||
}
|
||||
|
||||
// Remove filename which is the meta file.
|
||||
item.transformMetaDir()
|
||||
|
|
|
|||
3
go.mod
3
go.mod
|
|
@ -77,8 +77,7 @@ require (
|
|||
github.com/shirou/gopsutil/v3 v3.21.3
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/streadway/amqp v1.0.0
|
||||
github.com/tinylib/msgp v1.1.3
|
||||
github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 // indirect
|
||||
github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402
|
||||
github.com/valyala/bytebufferpool v1.0.0
|
||||
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
|
||||
github.com/willf/bloom v2.0.3+incompatible
|
||||
|
|
|
|||
6
go.sum
6
go.sum
|
|
@ -686,16 +686,15 @@ github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV
|
|||
github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y=
|
||||
github.com/tidwall/sjson v1.1.6 h1:8fDdlahON04OZBlTQCIatW8FstSFJz8oxidj5h0rmSQ=
|
||||
github.com/tidwall/sjson v1.1.6/go.mod h1:KN3FZ7odvXIHPbJdhNorK/M9lWweVUbXsXXhrJ/kGOA=
|
||||
github.com/tinylib/msgp v1.1.3 h1:3giwAkmtaEDLSV0MdO1lDLuPgklgPzmk8H9+So2BVfA=
|
||||
github.com/tinylib/msgp v1.1.3/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
|
||||
github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402 h1:x5VlSgDgIGXNegkO4gigpYmb/RFkKGgy12Kkrbif7XE=
|
||||
github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
|
||||
github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M=
|
||||
github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek=
|
||||
github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc=
|
||||
github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 h1:OXcKh35JaYsGMRzpvFkLv/MEyPuL49CThT1pZ8aSml4=
|
||||
github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q=
|
||||
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
|
||||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
|
|
@ -989,6 +988,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc
|
|||
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20201105001634-bc3cf281b174/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
|
|
|
|||
Loading…
Reference in a new issue