From 0287711dc9665525e3ec2e911b220f5a7a3b40ed Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 21 May 2021 11:41:25 -0700 Subject: [PATCH] fix: implement readMetadata common function for re-use (#12353) Previous PR #12351 added functions to read from the reader stream to reduce memory usage, use the same technique in few other places where we are not interested in reading the data part. --- Makefile | 4 +-- cmd/metacache-walk.go | 22 +++------------- cmd/xl-storage-format-v2.go | 50 +------------------------------------ cmd/xl-storage.go | 31 +++++++++++------------ go.mod | 3 +-- go.sum | 6 ++--- 6 files changed, 25 insertions(+), 91 deletions(-) diff --git a/Makefile b/Makefile index feadc3251..cea8ad39b 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,8 @@ checks: getdeps: @mkdir -p ${GOPATH}/bin @which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0) - @which msgp 1>/dev/null || (echo "Installing msgp" && go get github.com/tinylib/msgp@v1.1.3) - @which stringer 1>/dev/null || (echo "Installing stringer" && go get golang.org/x/tools/cmd/stringer) + @which msgp 1>/dev/null || (echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.3) + @which stringer 1>/dev/null || (echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer) crosscompile: @(env bash $(PWD)/buildscripts/cross-compile.sh) diff --git a/cmd/metacache-walk.go b/cmd/metacache-walk.go index c8d5d3f0a..056cc40f8 100644 --- a/cmd/metacache-walk.go +++ b/cmd/metacache-walk.go @@ -88,7 +88,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ // Fast exit track to check if we are listing an object with // a trailing slash, this will avoid to list the object content. if HasSuffix(opts.BaseDir, SlashSeparator) { - metadata, err := xioutil.ReadFile(pathJoin(volumeDir, + metadata, err := s.readMetadata(pathJoin(volumeDir, opts.BaseDir[:len(opts.BaseDir)-1]+globalDirSuffix, xlStorageFormatFile)) if err == nil { @@ -97,7 +97,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ // behavior. out <- metaCacheEntry{ name: opts.BaseDir, - metadata: xlMetaV2TrimData(metadata), + metadata: metadata, } } else { if st, err := os.Lstat(pathJoin(volumeDir, opts.BaseDir, xlStorageFormatFile)); err == nil && st.Mode().IsRegular() { @@ -154,25 +154,11 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ // If root was an object return it as such. if HasSuffix(entry, xlStorageFormatFile) { var meta metaCacheEntry - f, err := os.OpenFile(pathJoin(volumeDir, current, entry), readMode, 0) + meta.metadata, err = s.readMetadata(pathJoin(volumeDir, current, entry)) if err != nil { logger.LogIf(ctx, err) continue } - stat, err := f.Stat() - if err != nil { - logger.LogIf(ctx, err) - f.Close() - continue - } - meta.metadata, err = readXLMetaNoData(f, stat.Size()) - if err != nil { - logger.LogIf(ctx, err) - f.Close() - continue - } - f.Close() - meta.metadata = xlMetaV2TrimData(meta.metadata) meta.name = strings.TrimSuffix(entry, xlStorageFormatFile) meta.name = strings.TrimSuffix(meta.name, SlashSeparator) meta.name = pathJoin(current, meta.name) @@ -233,7 +219,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ meta.name = meta.name[:len(meta.name)-1] + globalDirSuffixWithSlash } - meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFile)) + meta.metadata, err = s.readMetadata(pathJoin(volumeDir, meta.name, xlStorageFormatFile)) switch { case err == nil: // It was an object diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index cf18ba1a7..533c392c9 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -1420,7 +1420,7 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) { err = readMore(size) return buf, err case 1, 2: - sz, tmp, err := ReadBytesHeader(tmp) + sz, tmp, err := msgp.ReadBytesHeader(tmp) if err != nil { return nil, err } @@ -1459,51 +1459,3 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) { return nil, errors.New("unknown major metadata version") } } - -// ReadBytesHeader reads the 'bin' header size -// off of 'b' and returns the size and remaining bytes. -// Possible errors: -// - ErrShortBytes (too few bytes) -// - TypeError{} (not a bin object) -// TODO: Replace when https://github.com/tinylib/msgp/pull/289 is merged. -func ReadBytesHeader(b []byte) (sz uint32, o []byte, err error) { - if len(b) < 1 { - return 0, nil, msgp.ErrShortBytes - } - var big = binary.BigEndian - - const ( - mbin8 uint8 = 0xc4 - mbin16 uint8 = 0xc5 - mbin32 uint8 = 0xc6 - ) - switch b[0] { - case mbin8: - if len(b) < 2 { - err = msgp.ErrShortBytes - return - } - sz = uint32(b[1]) - o = b[2:] - return - case mbin16: - if len(b) < 3 { - err = msgp.ErrShortBytes - return - } - sz = uint32(big.Uint16(b[1:])) - o = b[3:] - return - case mbin32: - if len(b) < 5 { - err = msgp.ErrShortBytes - return - } - sz = big.Uint32(b[1:]) - o = b[5:] - return - default: - err = msgp.TypeError{Method: msgp.BinType, Encoded: msgp.NextType(b)} - return - } -} diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index f23ece729..eb2873911 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -384,6 +384,19 @@ func (s *xlStorage) Healing() *healingTracker { return &h } +func (s *xlStorage) readMetadata(itemPath string) ([]byte, error) { + f, err := os.OpenFile(itemPath, readMode, 0) + if err != nil { + return nil, err + } + defer f.Close() + stat, err := f.Stat() + if err != nil { + return nil, err + } + return readXLMetaNoData(f, stat.Size()) +} + func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry) (dataUsageCache, error) { // Updates must be closed before we return. defer close(updates) @@ -421,29 +434,13 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates return sizeSummary{}, errSkipFile } - f, err := os.OpenFile(item.Path, readMode, 0) + buf, err := s.readMetadata(item.Path) if err != nil { if intDataUpdateTracker.debug { console.Debugf(color.Green("scannerBucket:")+" object path missing: %v: %w\n", item.Path, err) } return sizeSummary{}, errSkipFile } - defer f.Close() - stat, err := f.Stat() - if err != nil { - if intDataUpdateTracker.debug { - console.Debugf(color.Green("scannerBucket:")+" stat failed: %v: %w\n", item.Path, err) - } - return sizeSummary{}, errSkipFile - } - - buf, err := readXLMetaNoData(f, stat.Size()) - if err != nil { - if intDataUpdateTracker.debug { - console.Debugf(color.Green("scannerBucket:")+" readXLMetaNoData: %v: %w\n", item.Path, err) - } - return sizeSummary{}, errSkipFile - } // Remove filename which is the meta file. item.transformMetaDir() diff --git a/go.mod b/go.mod index 4e2b40263..55768eaee 100644 --- a/go.mod +++ b/go.mod @@ -77,8 +77,7 @@ require ( github.com/shirou/gopsutil/v3 v3.21.3 github.com/spf13/pflag v1.0.5 // indirect github.com/streadway/amqp v1.0.0 - github.com/tinylib/msgp v1.1.3 - github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 // indirect + github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402 github.com/valyala/bytebufferpool v1.0.0 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a github.com/willf/bloom v2.0.3+incompatible diff --git a/go.sum b/go.sum index 234207658..1c8fd6257 100644 --- a/go.sum +++ b/go.sum @@ -686,16 +686,15 @@ github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/tidwall/sjson v1.1.6 h1:8fDdlahON04OZBlTQCIatW8FstSFJz8oxidj5h0rmSQ= github.com/tidwall/sjson v1.1.6/go.mod h1:KN3FZ7odvXIHPbJdhNorK/M9lWweVUbXsXXhrJ/kGOA= -github.com/tinylib/msgp v1.1.3 h1:3giwAkmtaEDLSV0MdO1lDLuPgklgPzmk8H9+So2BVfA= github.com/tinylib/msgp v1.1.3/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402 h1:x5VlSgDgIGXNegkO4gigpYmb/RFkKGgy12Kkrbif7XE= +github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M= github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc= github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 h1:OXcKh35JaYsGMRzpvFkLv/MEyPuL49CThT1pZ8aSml4= -github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -989,6 +988,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201105001634-bc3cf281b174/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=