maxmustermann/minio
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Move to Set() not Append() due to large memory reference copy

Browse source
This commit is contained in:
Harshavardhana 2015-06-28 10:09:22 -07:00
parent 22abe1b397
commit 91e5f648cb
1 changed files with 11 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
pkg/storage/drivers/donut/donut.go
View file

@ -500,9 +500,8 @@ func (d donutDriver) ListObjects(bucketName string, resources drivers.BucketReso
} }
type proxyReader struct { type proxyReader struct {
reader io.Reader reader io.Reader
driver donutDriver readBytes []byte
object string
} }
func (r *proxyReader) free(p []byte) { func (r *proxyReader) free(p []byte) {
@ -513,24 +512,18 @@ func (r *proxyReader) Read(p []byte) (n int, err error) {
defer r.free(p) defer r.free(p)
n, err = r.reader.Read(p) n, err = r.reader.Read(p)
if err == io.EOF || err == io.ErrUnexpectedEOF { if err == io.EOF || err == io.ErrUnexpectedEOF {
ok := r.driver.objects.Append(r.object, p[0:n]) r.readBytes = append(r.readBytes, p[0:n]...)
if !ok {
return n, io.ErrShortBuffer
}
return return
} }
if err != nil { if err != nil {
return return
} }
ok := r.driver.objects.Append(r.object, p[0:n]) r.readBytes = append(r.readBytes, p[0:n]...)
if !ok {
return n, io.ErrShortBuffer
}
return return
} }
func newProxyReader(r io.Reader, d donutDriver, k string) *proxyReader { func newProxyReader(r io.Reader) *proxyReader {
return &proxyReader{reader: r, driver: d, object: k} return &proxyReader{reader: r, readBytes: nil}
} }
// CreateObject creates a new object // CreateObject creates a new object
@ -578,7 +571,7 @@ func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedM
} }
expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes) expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
} }
newReader := newProxyReader(reader, d, objectKey) newReader := newProxyReader(reader)
calculatedMD5Sum, err := d.donut.PutObject(bucketName, objectName, expectedMD5Sum, newReader, metadata) calculatedMD5Sum, err := d.donut.PutObject(bucketName, objectName, expectedMD5Sum, newReader, metadata)
if err != nil { if err != nil {
switch iodine.ToError(err).(type) { switch iodine.ToError(err).(type) {
@ -587,6 +580,10 @@ func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedM
} }
return "", iodine.New(err, errParams) return "", iodine.New(err, errParams)
} }
d.objects.Set(objectKey, newReader.readBytes)
// free up
newReader.readBytes = nil
go debug.FreeOSMemory()
objectMetadata, err := d.donut.GetObjectMetadata(bucketName, objectName) objectMetadata, err := d.donut.GetObjectMetadata(bucketName, objectName)
if err != nil { if err != nil {
return "", iodine.New(err, nil) return "", iodine.New(err, nil)