fix: implement readMetadata common function for re-use (#12353)

Previous PR #12351 added functions to read from the reader
stream to reduce memory usage, use the same technique in
few other places where we are not interested in reading the
data part.

Makefile

@@ -17,8 +17,8 @@ checks:
 getdeps:
 	@mkdir -p ${GOPATH}/bin
 	@which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0)
-	@which msgp 1>/dev/null || (echo "Installing msgp" && go get github.com/tinylib/msgp@v1.1.3)
+	@which msgp 1>/dev/null || (echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.3)
-	@which stringer 1>/dev/null || (echo "Installing stringer" && go get golang.org/x/tools/cmd/stringer)
+	@which stringer 1>/dev/null || (echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer)
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)

cmd/metacache-walk.go

@@ -88,7 +88,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 	// Fast exit track to check if we are listing an object with
 	// a trailing slash, this will avoid to list the object content.
 	if HasSuffix(opts.BaseDir, SlashSeparator) {
-		metadata, err := xioutil.ReadFile(pathJoin(volumeDir,
+		metadata, err := s.readMetadata(pathJoin(volumeDir,
 			opts.BaseDir[:len(opts.BaseDir)-1]+globalDirSuffix,
 			xlStorageFormatFile))
 		if err == nil {
@@ -97,7 +97,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			// behavior.
 			out <- metaCacheEntry{
 				name:     opts.BaseDir,
-				metadata: xlMetaV2TrimData(metadata),
+				metadata: metadata,
 			}
 		} else {
 			if st, err := os.Lstat(pathJoin(volumeDir, opts.BaseDir, xlStorageFormatFile)); err == nil && st.Mode().IsRegular() {
@@ -154,25 +154,11 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			// If root was an object return it as such.
 			if HasSuffix(entry, xlStorageFormatFile) {
 				var meta metaCacheEntry
-				f, err := os.OpenFile(pathJoin(volumeDir, current, entry), readMode, 0)
+				meta.metadata, err = s.readMetadata(pathJoin(volumeDir, current, entry))
 				if err != nil {
 					logger.LogIf(ctx, err)
 					continue
 				}
-				stat, err := f.Stat()
-				if err != nil {
-					logger.LogIf(ctx, err)
-					f.Close()
-					continue
-				}
-				meta.metadata, err = readXLMetaNoData(f, stat.Size())
-				if err != nil {
-					logger.LogIf(ctx, err)
-					f.Close()
-					continue
-				}
-				f.Close()
-				meta.metadata = xlMetaV2TrimData(meta.metadata)
 				meta.name = strings.TrimSuffix(entry, xlStorageFormatFile)
 				meta.name = strings.TrimSuffix(meta.name, SlashSeparator)
 				meta.name = pathJoin(current, meta.name)
@@ -233,7 +219,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				meta.name = meta.name[:len(meta.name)-1] + globalDirSuffixWithSlash
 			}
 
-			meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFile))
+			meta.metadata, err = s.readMetadata(pathJoin(volumeDir, meta.name, xlStorageFormatFile))
 			switch {
 			case err == nil:
 				// It was an object

cmd/xl-storage-format-v2.go

@@ -1420,7 +1420,7 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) {
 			err = readMore(size)
 			return buf, err
 		case 1, 2:
-			sz, tmp, err := ReadBytesHeader(tmp)
+			sz, tmp, err := msgp.ReadBytesHeader(tmp)
 			if err != nil {
 				return nil, err
 			}
@@ -1459,51 +1459,3 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) {
 		return nil, errors.New("unknown major metadata version")
 	}
 }
-
-// ReadBytesHeader reads the 'bin' header size
-// off of 'b' and returns the size and remaining bytes.
-// Possible errors:
-// - ErrShortBytes (too few bytes)
-// - TypeError{} (not a bin object)
-// TODO: Replace when https://github.com/tinylib/msgp/pull/289 is merged.
-func ReadBytesHeader(b []byte) (sz uint32, o []byte, err error) {
-	if len(b) < 1 {
-		return 0, nil, msgp.ErrShortBytes
-	}
-	var big = binary.BigEndian
-
-	const (
-		mbin8  uint8 = 0xc4
-		mbin16 uint8 = 0xc5
-		mbin32 uint8 = 0xc6
-	)
-	switch b[0] {
-	case mbin8:
-		if len(b) < 2 {
-			err = msgp.ErrShortBytes
-			return
-		}
-		sz = uint32(b[1])
-		o = b[2:]
-		return
-	case mbin16:
-		if len(b) < 3 {
-			err = msgp.ErrShortBytes
-			return
-		}
-		sz = uint32(big.Uint16(b[1:]))
-		o = b[3:]
-		return
-	case mbin32:
-		if len(b) < 5 {
-			err = msgp.ErrShortBytes
-			return
-		}
-		sz = big.Uint32(b[1:])
-		o = b[5:]
-		return
-	default:
-		err = msgp.TypeError{Method: msgp.BinType, Encoded: msgp.NextType(b)}
-		return
-	}
-}

cmd/xl-storage.go

@@ -384,6 +384,19 @@ func (s *xlStorage) Healing() *healingTracker {
 	return &h
 }
 
+func (s *xlStorage) readMetadata(itemPath string) ([]byte, error) {
+	f, err := os.OpenFile(itemPath, readMode, 0)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	stat, err := f.Stat()
+	if err != nil {
+		return nil, err
+	}
+	return readXLMetaNoData(f, stat.Size())
+}
+
 func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry) (dataUsageCache, error) {
 	// Updates must be closed before we return.
 	defer close(updates)
@@ -421,29 +434,13 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 			return sizeSummary{}, errSkipFile
 		}
 
-		f, err := os.OpenFile(item.Path, readMode, 0)
+		buf, err := s.readMetadata(item.Path)
 		if err != nil {
 			if intDataUpdateTracker.debug {
 				console.Debugf(color.Green("scannerBucket:")+" object path missing: %v: %w\n", item.Path, err)
 			}
 			return sizeSummary{}, errSkipFile
 		}
-		defer f.Close()
-		stat, err := f.Stat()
-		if err != nil {
-			if intDataUpdateTracker.debug {
-				console.Debugf(color.Green("scannerBucket:")+" stat failed: %v: %w\n", item.Path, err)
-			}
-			return sizeSummary{}, errSkipFile
-		}
-
-		buf, err := readXLMetaNoData(f, stat.Size())
-		if err != nil {
-			if intDataUpdateTracker.debug {
-				console.Debugf(color.Green("scannerBucket:")+" readXLMetaNoData: %v: %w\n", item.Path, err)
-			}
-			return sizeSummary{}, errSkipFile
-		}
 
 		// Remove filename which is the meta file.
 		item.transformMetaDir()

go.mod

@@ -77,8 +77,7 @@ require (
 	github.com/shirou/gopsutil/v3 v3.21.3
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/streadway/amqp v1.0.0
-	github.com/tinylib/msgp v1.1.3
+	github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402
-	github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 // indirect
 	github.com/valyala/bytebufferpool v1.0.0
 	github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
 	github.com/willf/bloom v2.0.3+incompatible

go.sum

@@ -686,16 +686,15 @@ github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV
 github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y=
 github.com/tidwall/sjson v1.1.6 h1:8fDdlahON04OZBlTQCIatW8FstSFJz8oxidj5h0rmSQ=
 github.com/tidwall/sjson v1.1.6/go.mod h1:KN3FZ7odvXIHPbJdhNorK/M9lWweVUbXsXXhrJ/kGOA=
-github.com/tinylib/msgp v1.1.3 h1:3giwAkmtaEDLSV0MdO1lDLuPgklgPzmk8H9+So2BVfA=
 github.com/tinylib/msgp v1.1.3/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402 h1:x5VlSgDgIGXNegkO4gigpYmb/RFkKGgy12Kkrbif7XE=
+github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
 github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M=
 github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek=
 github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc=
 github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 h1:OXcKh35JaYsGMRzpvFkLv/MEyPuL49CThT1pZ8aSml4=
-github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
@@ -989,6 +988,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201105001634-bc3cf281b174/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=