Update s3 library again

This commit is contained in:
Alexander Neumann 2016-01-02 14:09:06 +01:00
parent a73c4bd5a7
commit f319354174
3 changed files with 26 additions and 11 deletions

4
Godeps/Godeps.json generated
View File

@ -24,8 +24,8 @@
}, },
{ {
"ImportPath": "github.com/minio/minio-go", "ImportPath": "github.com/minio/minio-go",
"Comment": "v0.2.5-185-g654a97a", "Comment": "v0.2.5-187-gad1597d",
"Rev": "654a97a4d165dabf422bec2ef6673bcd9d3daf00" "Rev": "ad1597d864f56f608f8a1694ae9b5970fef57eb6"
}, },
{ {
"ImportPath": "github.com/pkg/sftp", "ImportPath": "github.com/pkg/sftp",

View File

@ -17,7 +17,6 @@
package minio package minio
import ( import (
"bytes"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@ -186,7 +185,7 @@ func (c Client) GetObjectPartial(bucketName, objectName string) (ReadAtCloser, O
// Get shortest length. // Get shortest length.
// NOTE: Last remaining bytes are usually smaller than // NOTE: Last remaining bytes are usually smaller than
// req.Buffer size. Use that as the final length. // req.Buffer size. Use that as the final length.
length := math.Min(float64(req.Buffer.Len()), float64(objectStat.Size-req.Offset)) length := math.Min(float64(len(req.Buffer)), float64(objectStat.Size-req.Offset))
httpReader, _, err := c.getObject(bucketName, objectName, req.Offset, int64(length)) httpReader, _, err := c.getObject(bucketName, objectName, req.Offset, int64(length))
if err != nil { if err != nil {
resCh <- readAtResponse{ resCh <- readAtResponse{
@ -194,7 +193,12 @@ func (c Client) GetObjectPartial(bucketName, objectName string) (ReadAtCloser, O
} }
return return
} }
size, err := io.CopyN(req.Buffer, httpReader, int64(length)) size, err := io.ReadFull(httpReader, req.Buffer)
if err == io.ErrUnexpectedEOF {
// If an EOF happens after reading some but not all the bytes
// ReadFull returns ErrUnexpectedEOF
err = io.EOF
}
resCh <- readAtResponse{ resCh <- readAtResponse{
Size: int(size), Size: int(size),
Error: err, Error: err,
@ -214,7 +218,7 @@ type readAtResponse struct {
// request message container to communicate with internal go-routine. // request message container to communicate with internal go-routine.
type readAtRequest struct { type readAtRequest struct {
Buffer *bytes.Buffer Buffer []byte
Offset int64 // readAt offset. Offset int64 // readAt offset.
} }
@ -267,7 +271,7 @@ func (r *objectReadAtCloser) ReadAt(b []byte, offset int64) (int, error) {
reqMsg := readAtRequest{} reqMsg := readAtRequest{}
// Send the current offset and bytes requested. // Send the current offset and bytes requested.
reqMsg.Buffer = bytes.NewBuffer(b) reqMsg.Buffer = b
reqMsg.Offset = offset reqMsg.Offset = offset
// Send read request over the control channel. // Send read request over the control channel.

View File

@ -18,6 +18,7 @@ package minio_test
import ( import (
"bytes" "bytes"
crand "crypto/rand"
"io" "io"
"io/ioutil" "io/ioutil"
"math/rand" "math/rand"
@ -83,8 +84,13 @@ func TestGetObjectPartialFunctional(t *testing.T) {
t.Fatal("Error:", err, bucketName) t.Fatal("Error:", err, bucketName)
} }
// generate data // generate data more than 32K
buf := make([]byte, rand.Intn(1<<20)) buf := make([]byte, rand.Intn(1<<20)+32*1024)
_, err = io.ReadFull(crand.Reader, buf)
if err != nil {
t.Fatal("Error:", err)
}
// save the data // save the data
objectName := randString(60, rand.NewSource(time.Now().UnixNano())) objectName := randString(60, rand.NewSource(time.Now().UnixNano()))
@ -122,6 +128,10 @@ func TestGetObjectPartialFunctional(t *testing.T) {
if m != len(buf2) { if m != len(buf2) {
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2)) t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2))
} }
if !bytes.Equal(buf2, buf[offset:offset+512]) {
t.Fatal("Error: Incorrect read between two ReadAt from same offset.")
}
offset += 512
m, err = r.ReadAt(buf3, offset) m, err = r.ReadAt(buf3, offset)
if err != nil { if err != nil {
t.Fatal("Error:", err, st.Size, len(buf3), offset) t.Fatal("Error:", err, st.Size, len(buf3), offset)
@ -129,9 +139,10 @@ func TestGetObjectPartialFunctional(t *testing.T) {
if m != len(buf3) { if m != len(buf3) {
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3)) t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3))
} }
if !bytes.Equal(buf2, buf3) { if !bytes.Equal(buf3, buf[offset:offset+512]) {
t.Fatal("Error: Incorrect read between two ReadAt from same offset.") t.Fatal("Error: Incorrect read between two ReadAt from same offset.")
} }
offset += 512
m, err = r.ReadAt(buf4, offset) m, err = r.ReadAt(buf4, offset)
if err != nil { if err != nil {
t.Fatal("Error:", err, st.Size, len(buf4), offset) t.Fatal("Error:", err, st.Size, len(buf4), offset)
@ -139,7 +150,7 @@ func TestGetObjectPartialFunctional(t *testing.T) {
if m != len(buf4) { if m != len(buf4) {
t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4)) t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4))
} }
if !bytes.Equal(buf2, buf4) { if !bytes.Equal(buf4, buf[offset:offset+512]) {
t.Fatal("Error: Incorrect read between two ReadAt from same offset.") t.Fatal("Error: Incorrect read between two ReadAt from same offset.")
} }