2016-03-28 06:52:38 +02:00
|
|
|
/*
|
2020-06-13 05:04:01 +02:00
|
|
|
* MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
|
2016-03-28 06:52:38 +02:00
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-19 01:23:42 +02:00
|
|
|
package cmd
|
2016-03-28 06:52:38 +02:00
|
|
|
|
2016-06-01 05:23:31 +02:00
|
|
|
import (
|
2018-04-06 00:04:40 +02:00
|
|
|
"context"
|
2020-04-01 21:14:00 +02:00
|
|
|
"errors"
|
2016-06-19 22:35:26 +02:00
|
|
|
"io"
|
2019-01-18 16:48:25 +01:00
|
|
|
"sync"
|
2020-04-01 21:14:00 +02:00
|
|
|
"sync/atomic"
|
2017-11-25 20:58:29 +01:00
|
|
|
|
2018-04-06 00:04:40 +02:00
|
|
|
"github.com/minio/minio/cmd/logger"
|
2016-06-01 05:23:31 +02:00
|
|
|
)
|
2016-03-28 06:52:38 +02:00
|
|
|
|
2019-01-17 13:58:18 +01:00
|
|
|
// Reads in parallel from readers.
|
2018-08-07 00:14:08 +02:00
|
|
|
type parallelReader struct {
|
2019-01-17 13:58:18 +01:00
|
|
|
readers []io.ReaderAt
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
orgReaders []io.ReaderAt
|
2018-08-07 00:14:08 +02:00
|
|
|
dataBlocks int
|
2021-01-02 19:35:57 +01:00
|
|
|
errs []error
|
2018-08-07 00:14:08 +02:00
|
|
|
offset int64
|
|
|
|
shardSize int64
|
|
|
|
shardFileSize int64
|
2019-01-17 13:58:18 +01:00
|
|
|
buf [][]byte
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
readerToBuf []int
|
2018-03-04 23:16:45 +01:00
|
|
|
}
|
|
|
|
|
2018-08-07 00:14:08 +02:00
|
|
|
// newParallelReader returns parallelReader.
|
2019-01-17 13:58:18 +01:00
|
|
|
func newParallelReader(readers []io.ReaderAt, e Erasure, offset, totalLength int64) *parallelReader {
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
r2b := make([]int, len(readers))
|
|
|
|
for i := range r2b {
|
|
|
|
r2b[i] = i
|
|
|
|
}
|
2018-08-07 00:14:08 +02:00
|
|
|
return ¶llelReader{
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
readers: readers,
|
|
|
|
orgReaders: readers,
|
2021-01-02 19:35:57 +01:00
|
|
|
errs: make([]error, len(readers)),
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
dataBlocks: e.dataBlocks,
|
|
|
|
offset: (offset / e.blockSize) * e.ShardSize(),
|
|
|
|
shardSize: e.ShardSize(),
|
|
|
|
shardFileSize: e.ShardFileSize(totalLength),
|
|
|
|
buf: make([][]byte, len(readers)),
|
|
|
|
readerToBuf: r2b,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// preferReaders can mark readers as preferred.
|
|
|
|
// These will be chosen before others.
|
|
|
|
func (p *parallelReader) preferReaders(prefer []bool) {
|
|
|
|
if len(prefer) != len(p.orgReaders) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
// Copy so we don't change our input.
|
|
|
|
tmp := make([]io.ReaderAt, len(p.orgReaders))
|
|
|
|
copy(tmp, p.orgReaders)
|
|
|
|
p.readers = tmp
|
|
|
|
// next is the next non-preferred index.
|
|
|
|
next := 0
|
|
|
|
for i, ok := range prefer {
|
|
|
|
if !ok || p.readers[i] == nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if i == next {
|
|
|
|
next++
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
// Move reader with index i to index next.
|
|
|
|
// Do this by swapping next and i
|
|
|
|
p.readers[next], p.readers[i] = p.readers[i], p.readers[next]
|
|
|
|
p.readerToBuf[next] = i
|
|
|
|
p.readerToBuf[i] = next
|
|
|
|
next++
|
2018-03-04 23:16:45 +01:00
|
|
|
}
|
2018-08-07 00:14:08 +02:00
|
|
|
}
|
2018-03-04 23:16:45 +01:00
|
|
|
|
2018-08-07 00:14:08 +02:00
|
|
|
// Returns if buf can be erasure decoded.
|
|
|
|
func (p *parallelReader) canDecode(buf [][]byte) bool {
|
|
|
|
bufCount := 0
|
|
|
|
for _, b := range buf {
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
if len(b) > 0 {
|
2018-08-07 00:14:08 +02:00
|
|
|
bufCount++
|
2018-03-04 23:16:45 +01:00
|
|
|
}
|
|
|
|
}
|
2018-08-07 00:14:08 +02:00
|
|
|
return bufCount >= p.dataBlocks
|
2018-03-04 23:16:45 +01:00
|
|
|
}
|
|
|
|
|
2019-01-17 13:58:18 +01:00
|
|
|
// Read reads from readers in parallel. Returns p.dataBlocks number of bufs.
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
|
|
|
|
newBuf := dst
|
|
|
|
if len(dst) != len(p.readers) {
|
|
|
|
newBuf = make([][]byte, len(p.readers))
|
|
|
|
} else {
|
|
|
|
for i := range newBuf {
|
|
|
|
newBuf[i] = newBuf[i][:0]
|
|
|
|
}
|
|
|
|
}
|
2019-01-18 16:48:25 +01:00
|
|
|
var newBufLK sync.RWMutex
|
2018-03-04 23:16:45 +01:00
|
|
|
|
2018-08-07 00:14:08 +02:00
|
|
|
if p.offset+p.shardSize > p.shardFileSize {
|
|
|
|
p.shardSize = p.shardFileSize - p.offset
|
2018-03-04 23:16:45 +01:00
|
|
|
}
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
if p.shardSize == 0 {
|
|
|
|
return newBuf, nil
|
|
|
|
}
|
2018-03-04 23:16:45 +01:00
|
|
|
|
2019-01-18 16:48:25 +01:00
|
|
|
readTriggerCh := make(chan bool, len(p.readers))
|
|
|
|
for i := 0; i < p.dataBlocks; i++ {
|
|
|
|
// Setup read triggers for p.dataBlocks number of reads so that it reads in parallel.
|
|
|
|
readTriggerCh <- true
|
2018-03-04 23:16:45 +01:00
|
|
|
}
|
|
|
|
|
2021-01-27 19:21:14 +01:00
|
|
|
bitrotHeal := int32(0) // Atomic bool flag.
|
|
|
|
missingPartsHeal := int32(0) // Atomic bool flag.
|
2019-01-18 16:48:25 +01:00
|
|
|
readerIndex := 0
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
// if readTrigger is true, it implies next disk.ReadAt() should be tried
|
|
|
|
// if readTrigger is false, it implies previous disk.ReadAt() was successful and there is no need
|
|
|
|
// to try reading the next disk.
|
|
|
|
for readTrigger := range readTriggerCh {
|
|
|
|
newBufLK.RLock()
|
|
|
|
canDecode := p.canDecode(newBuf)
|
|
|
|
newBufLK.RUnlock()
|
|
|
|
if canDecode {
|
|
|
|
break
|
2018-08-07 00:14:08 +02:00
|
|
|
}
|
2019-01-18 16:48:25 +01:00
|
|
|
if readerIndex == len(p.readers) {
|
2018-08-07 00:14:08 +02:00
|
|
|
break
|
2018-03-04 23:16:45 +01:00
|
|
|
}
|
2019-01-18 16:48:25 +01:00
|
|
|
if !readTrigger {
|
2018-08-07 00:14:08 +02:00
|
|
|
continue
|
2016-06-25 03:00:34 +02:00
|
|
|
}
|
2019-01-18 16:48:25 +01:00
|
|
|
wg.Add(1)
|
|
|
|
go func(i int) {
|
|
|
|
defer wg.Done()
|
2020-06-26 08:20:12 +02:00
|
|
|
rr := p.readers[i]
|
|
|
|
if rr == nil {
|
|
|
|
// Since reader is nil, trigger another read.
|
2019-01-18 16:48:25 +01:00
|
|
|
readTriggerCh <- true
|
|
|
|
return
|
2018-03-04 23:16:45 +01:00
|
|
|
}
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
bufIdx := p.readerToBuf[i]
|
|
|
|
if p.buf[bufIdx] == nil {
|
2019-01-18 16:48:25 +01:00
|
|
|
// Reading first time on this disk, hence the buffer needs to be allocated.
|
|
|
|
// Subsequent reads will re-use this buffer.
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
p.buf[bufIdx] = make([]byte, p.shardSize)
|
2019-01-18 16:48:25 +01:00
|
|
|
}
|
|
|
|
// For the last shard, the shardsize might be less than previous shard sizes.
|
|
|
|
// Hence the following statement ensures that the buffer size is reset to the right size.
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
p.buf[bufIdx] = p.buf[bufIdx][:p.shardSize]
|
2021-03-16 04:03:13 +01:00
|
|
|
n, err := rr.ReadAt(p.buf[bufIdx], p.offset)
|
2019-01-18 16:48:25 +01:00
|
|
|
if err != nil {
|
2021-01-27 19:21:14 +01:00
|
|
|
if errors.Is(err, errFileNotFound) {
|
|
|
|
atomic.StoreInt32(&missingPartsHeal, 1)
|
|
|
|
} else if errors.Is(err, errFileCorrupt) {
|
|
|
|
atomic.StoreInt32(&bitrotHeal, 1)
|
2020-04-01 21:14:00 +02:00
|
|
|
}
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
|
|
|
|
// This will be communicated upstream.
|
|
|
|
p.orgReaders[bufIdx] = nil
|
2019-01-18 16:48:25 +01:00
|
|
|
p.readers[i] = nil
|
2021-01-02 19:35:57 +01:00
|
|
|
p.errs[i] = err
|
|
|
|
|
2019-01-18 16:48:25 +01:00
|
|
|
// Since ReadAt returned error, trigger another read.
|
|
|
|
readTriggerCh <- true
|
|
|
|
return
|
|
|
|
}
|
|
|
|
newBufLK.Lock()
|
2021-03-16 04:03:13 +01:00
|
|
|
newBuf[bufIdx] = p.buf[bufIdx][:n]
|
2019-01-18 16:48:25 +01:00
|
|
|
newBufLK.Unlock()
|
|
|
|
// Since ReadAt returned success, there is no need to trigger another read.
|
|
|
|
readTriggerCh <- false
|
|
|
|
}(readerIndex)
|
|
|
|
readerIndex++
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
if p.canDecode(newBuf) {
|
|
|
|
p.offset += p.shardSize
|
2021-01-27 19:21:14 +01:00
|
|
|
if atomic.LoadInt32(&missingPartsHeal) == 1 {
|
|
|
|
return newBuf, errFileNotFound
|
|
|
|
} else if atomic.LoadInt32(&bitrotHeal) == 1 {
|
|
|
|
return newBuf, errFileCorrupt
|
2020-04-01 21:14:00 +02:00
|
|
|
}
|
2019-01-18 16:48:25 +01:00
|
|
|
return newBuf, nil
|
2018-08-07 00:14:08 +02:00
|
|
|
}
|
2018-03-04 23:16:45 +01:00
|
|
|
|
2021-01-02 19:35:57 +01:00
|
|
|
return nil, reduceReadQuorumErrs(context.Background(), p.errs, objectOpIgnoredErrs, p.dataBlocks)
|
2018-08-07 00:14:08 +02:00
|
|
|
}
|
2018-03-04 23:16:45 +01:00
|
|
|
|
2018-08-24 08:35:37 +02:00
|
|
|
// Decode reads from readers, reconstructs data if needed and writes the data to the writer.
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
// A set of preferred drives can be supplied. In that case they will be used and the data reconstructed.
|
2021-01-27 19:21:14 +01:00
|
|
|
func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.ReaderAt, offset, length, totalLength int64, prefer []bool) (written int64, derr error) {
|
2018-08-07 00:14:08 +02:00
|
|
|
if offset < 0 || length < 0 {
|
|
|
|
logger.LogIf(ctx, errInvalidArgument)
|
2021-01-27 19:21:14 +01:00
|
|
|
return -1, errInvalidArgument
|
2018-08-07 00:14:08 +02:00
|
|
|
}
|
|
|
|
if offset+length > totalLength {
|
|
|
|
logger.LogIf(ctx, errInvalidArgument)
|
2021-01-27 19:21:14 +01:00
|
|
|
return -1, errInvalidArgument
|
2018-08-07 00:14:08 +02:00
|
|
|
}
|
2021-01-27 19:21:14 +01:00
|
|
|
|
2018-08-07 00:14:08 +02:00
|
|
|
if length == 0 {
|
2021-01-27 19:21:14 +01:00
|
|
|
return 0, nil
|
2018-08-07 00:14:08 +02:00
|
|
|
}
|
2018-03-04 23:16:45 +01:00
|
|
|
|
2019-01-17 13:58:18 +01:00
|
|
|
reader := newParallelReader(readers, e, offset, totalLength)
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
if len(prefer) == len(readers) {
|
|
|
|
reader.preferReaders(prefer)
|
|
|
|
}
|
2018-08-07 00:14:08 +02:00
|
|
|
|
2018-08-24 08:35:37 +02:00
|
|
|
startBlock := offset / e.blockSize
|
|
|
|
endBlock := (offset + length) / e.blockSize
|
2018-08-07 00:14:08 +02:00
|
|
|
|
|
|
|
var bytesWritten int64
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
var bufs [][]byte
|
2018-08-07 00:14:08 +02:00
|
|
|
for block := startBlock; block <= endBlock; block++ {
|
|
|
|
var blockOffset, blockLength int64
|
|
|
|
switch {
|
|
|
|
case startBlock == endBlock:
|
2018-08-24 08:35:37 +02:00
|
|
|
blockOffset = offset % e.blockSize
|
2018-08-07 00:14:08 +02:00
|
|
|
blockLength = length
|
|
|
|
case block == startBlock:
|
2018-08-24 08:35:37 +02:00
|
|
|
blockOffset = offset % e.blockSize
|
|
|
|
blockLength = e.blockSize - blockOffset
|
2018-08-07 00:14:08 +02:00
|
|
|
case block == endBlock:
|
|
|
|
blockOffset = 0
|
2018-08-24 08:35:37 +02:00
|
|
|
blockLength = (offset + length) % e.blockSize
|
2018-08-07 00:14:08 +02:00
|
|
|
default:
|
|
|
|
blockOffset = 0
|
2018-08-24 08:35:37 +02:00
|
|
|
blockLength = e.blockSize
|
2016-06-25 03:00:34 +02:00
|
|
|
}
|
2018-08-07 00:14:08 +02:00
|
|
|
if blockLength == 0 {
|
|
|
|
break
|
|
|
|
}
|
2021-01-27 19:21:14 +01:00
|
|
|
|
Prefer local disks when fetching data blocks (#9563)
If the requested server is part of the set this will always read
from the local disk, even if the disk contains a parity shard.
In default setup there is a 50% chance that at least
one shard that otherwise would have been fetched remotely
will be read locally instead.
It basically trades RPC call overhead for reed-solomon.
On distributed localhost this seems to be fairly break-even,
with a very small gain in throughput and latency.
However on networked servers this should be a bigger
1MB objects, before:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 76257:
* Avg: 25ms 50%: 24ms 90%: 32ms 99%: 42ms Fastest: 7ms Slowest: 67ms
* First Byte: Average: 23ms, Median: 22ms, Best: 5ms, Worst: 65ms
Throughput:
* Average: 1213.68 MiB/s, 1272.63 obj/s (59.948s, starting 14:45:44 CEST)
```
After:
```
Operation: GET. Concurrency: 32. Hosts: 4.
Requests considered: 78845:
* Avg: 24ms 50%: 24ms 90%: 31ms 99%: 39ms Fastest: 8ms Slowest: 62ms
* First Byte: Average: 22ms, Median: 21ms, Best: 6ms, Worst: 57ms
Throughput:
* Average: 1255.11 MiB/s, 1316.08 obj/s (59.938s, starting 14:43:58 CEST)
```
Bonus fix: Only ask for heal once on an object.
2020-05-27 01:47:23 +02:00
|
|
|
var err error
|
|
|
|
bufs, err = reader.Read(bufs)
|
2021-01-27 19:21:14 +01:00
|
|
|
if len(bufs) > 0 {
|
|
|
|
// Set only if there are be enough data for reconstruction.
|
|
|
|
// and only for expected errors, also set once.
|
|
|
|
if errors.Is(err, errFileNotFound) || errors.Is(err, errFileCorrupt) {
|
|
|
|
if derr == nil {
|
|
|
|
derr = err
|
|
|
|
}
|
2020-04-01 21:14:00 +02:00
|
|
|
}
|
2021-01-27 19:21:14 +01:00
|
|
|
} else if err != nil {
|
|
|
|
// For all errors that cannot be reconstructed fail the read operation.
|
|
|
|
return -1, err
|
2017-08-15 03:08:42 +02:00
|
|
|
}
|
2020-11-12 21:12:09 +01:00
|
|
|
|
2018-08-24 08:35:37 +02:00
|
|
|
if err = e.DecodeDataBlocks(bufs); err != nil {
|
2018-08-07 00:14:08 +02:00
|
|
|
logger.LogIf(ctx, err)
|
2021-01-27 19:21:14 +01:00
|
|
|
return -1, err
|
2016-06-27 22:24:55 +02:00
|
|
|
}
|
2020-11-12 21:12:09 +01:00
|
|
|
|
2018-08-24 08:35:37 +02:00
|
|
|
n, err := writeDataBlocks(ctx, writer, bufs, e.dataBlocks, blockOffset, blockLength)
|
2018-08-07 00:14:08 +02:00
|
|
|
if err != nil {
|
2021-01-27 19:21:14 +01:00
|
|
|
return -1, err
|
2018-08-07 00:14:08 +02:00
|
|
|
}
|
2021-01-27 19:21:14 +01:00
|
|
|
|
2018-08-07 00:14:08 +02:00
|
|
|
bytesWritten += n
|
|
|
|
}
|
2021-01-27 19:21:14 +01:00
|
|
|
|
2018-08-07 00:14:08 +02:00
|
|
|
if bytesWritten != length {
|
|
|
|
logger.LogIf(ctx, errLessData)
|
2021-01-27 19:21:14 +01:00
|
|
|
return bytesWritten, errLessData
|
2016-06-27 22:24:55 +02:00
|
|
|
}
|
2020-04-01 21:14:00 +02:00
|
|
|
|
2021-01-27 19:21:14 +01:00
|
|
|
return bytesWritten, derr
|
2016-06-27 22:24:55 +02:00
|
|
|
}
|