From 92bb2928e427fe35e43f2b026118786df872630e Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Mon, 16 Aug 2021 20:55:07 +0200 Subject: [PATCH] Compress better on amd64 (#12974) Since S2 has amd64 assembly, it now operates at a reasonable speed to use by default. Here are some examples of stream compression speed, 16 cores: ``` nyc-taxi-data-10M.csv s2 1 3325605752 -> 1095998837 312ms 10139.07MB/s 67.04% reduction nyc-taxi-data-10M.csv s2 2 3325605752 -> 917905514 428ms 7393.74MB/s 72.40% github-june-2days-2019.json s2 1 6273951764 -> 1043196283 391ms 15301.99 MB/s 83.37% github-june-2days-2019.json s2 2 6273951764 -> 955924506 519ms 11510.81MB/s 84.76% github-ranks-backup.bin s2 1 1862623243 -> 623911363 146ms 12133MB/s 66.50% github-ranks-backup.bin s2 2 1862623243 -> 563752759 230ms 7705.26MB/s 69.73% ``` We keep non-assembly platforms on the faster, but less efficient mode. --- cmd/object-api-utils.go | 12 +++++++++++- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cmd/object-api-utils.go b/cmd/object-api-utils.go index a71cea3bf..961be2f7b 100644 --- a/cmd/object-api-utils.go +++ b/cmd/object-api-utils.go @@ -884,6 +884,16 @@ func CleanMinioInternalMetadataKeys(metadata map[string]string) map[string]strin return newMeta } +// compressOpts are the options for writing compressed data. +var compressOpts []s2.WriterOption + +func init() { + if runtime.GOARCH == "amd64" { + // On amd64 we have assembly and can use stronger compression. + compressOpts = append(compressOpts, s2.WriterBetterCompression()) + } +} + // newS2CompressReader will read data from r, compress it and return the compressed data as a Reader. // Use Close to ensure resources are released on incomplete streams. // @@ -894,7 +904,7 @@ func newS2CompressReader(r io.Reader, on int64) io.ReadCloser { pr, pw := io.Pipe() // Copy input to compressor go func() { - comp := s2.NewWriter(pw) + comp := s2.NewWriter(pw, compressOpts...) cn, err := io.Copy(comp, r) if err != nil { comp.Close() diff --git a/go.mod b/go.mod index dd8ccf46a..666f80fdf 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/jcmturner/gokrb5/v8 v8.4.2 github.com/json-iterator/go v1.1.11 - github.com/klauspost/compress v1.13.3 + github.com/klauspost/compress v1.13.4 github.com/klauspost/cpuid/v2 v2.0.4 github.com/klauspost/pgzip v1.2.5 github.com/klauspost/readahead v1.3.1 diff --git a/go.sum b/go.sum index cb7df0d64..ddc076e24 100644 --- a/go.sum +++ b/go.sum @@ -881,8 +881,8 @@ github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.13.3 h1:BtAvtV1+h0YwSVwWoYXMREPpYu9VzTJ9QDI1TEg/iQQ= -github.com/klauspost/compress v1.13.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= +github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=