From a61016cb55d38c41f2eadb88bf615225e6486c1a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 8 Jun 2017 19:40:06 +0200 Subject: [PATCH 1/2] Update github.com/minio/minio-go --- vendor/manifest | 2 +- .../minio/minio-go/api-get-object.go | 20 +++-- .../src/github.com/minio/minio-go/api-stat.go | 11 ++- .../minio/minio-go/api_functional_v4_test.go | 1 + .../src/github.com/minio/minio-go/docs/API.md | 7 +- .../examples/s3/get-encrypted-object.go | 6 +- .../examples/s3/put-encrypted-object.go | 5 +- .../examples/s3/putobject-getobject-sse.go | 87 +++++++++++++++++++ .../minio/minio-go/pkg/encrypt/cbc.go | 9 ++ .../minio/minio-go/pkg/encrypt/interface.go | 3 + .../minio/minio-go/request-headers.go | 4 +- .../src/github.com/minio/minio-go/s3-error.go | 2 +- 12 files changed, 133 insertions(+), 24 deletions(-) create mode 100644 vendor/src/github.com/minio/minio-go/examples/s3/putobject-getobject-sse.go diff --git a/vendor/manifest b/vendor/manifest index 40f904099..e53bc0309 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -46,7 +46,7 @@ { "importpath": "github.com/minio/minio-go", "repository": "https://github.com/minio/minio-go", - "revision": "85f15b007f08e11a62c769abe65299b812fd2e0d", + "revision": "f2362d9e7d8daf89594ee0a079be2424eaf360be", "branch": "master" }, { diff --git a/vendor/src/github.com/minio/minio-go/api-get-object.go b/vendor/src/github.com/minio/minio-go/api-get-object.go index 2abd4608e..36d7f5049 100644 --- a/vendor/src/github.com/minio/minio-go/api-get-object.go +++ b/vendor/src/github.com/minio/minio-go/api-get-object.go @@ -28,9 +28,9 @@ import ( "github.com/minio/minio-go/pkg/encrypt" ) -// GetEncryptedObject deciphers and streams data stored in the server after applying a specifed encryption materiels -func (c Client) GetEncryptedObject(bucketName, objectName string, encryptMaterials encrypt.Materials) (io.Reader, error) { - +// GetEncryptedObject deciphers and streams data stored in the server after applying a specifed encryption materials, +// returned stream should be closed by the caller. +func (c Client) GetEncryptedObject(bucketName, objectName string, encryptMaterials encrypt.Materials) (io.ReadCloser, error) { if encryptMaterials == nil { return nil, ErrInvalidArgument("Unable to recognize empty encryption properties") } @@ -328,14 +328,14 @@ func (o *Object) setOffset(bytesRead int64) error { // Update the currentOffset. o.currOffset += bytesRead - if o.currOffset >= o.objectInfo.Size { + if o.objectInfo.Size > -1 && o.currOffset >= o.objectInfo.Size { return io.EOF } return nil } // Read reads up to len(b) bytes into b. It returns the number of -// bytes read (0 <= n <= len(p)) and any error encountered. Returns +// bytes read (0 <= n <= len(b)) and any error encountered. Returns // io.EOF upon end of file. func (o *Object) Read(b []byte) (n int, err error) { if o == nil { @@ -442,7 +442,7 @@ func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) { if o.objectInfoSet { // If offset is negative than we return io.EOF. // If offset is greater than or equal to object size we return io.EOF. - if offset >= o.objectInfo.Size || offset < 0 { + if (o.objectInfo.Size > -1 && offset >= o.objectInfo.Size) || offset < 0 { return 0, io.EOF } } @@ -542,16 +542,20 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { default: return 0, ErrInvalidArgument(fmt.Sprintf("Invalid whence %d", whence)) case 0: - if offset > o.objectInfo.Size { + if o.objectInfo.Size > -1 && offset > o.objectInfo.Size { return 0, io.EOF } o.currOffset = offset case 1: - if o.currOffset+offset > o.objectInfo.Size { + if o.objectInfo.Size > -1 && o.currOffset+offset > o.objectInfo.Size { return 0, io.EOF } o.currOffset += offset case 2: + // If we don't know the object size return an error for io.SeekEnd + if o.objectInfo.Size < 0 { + return 0, ErrInvalidArgument("Whence END is not supported when the object size is unknown") + } // Seeking to positive offset is valid for whence '2', but // since we are backing a Reader we have reached 'EOF' if // offset is positive. diff --git a/vendor/src/github.com/minio/minio-go/api-stat.go b/vendor/src/github.com/minio/minio-go/api-stat.go index 5b3dfe1b4..94f35db2a 100644 --- a/vendor/src/github.com/minio/minio-go/api-stat.go +++ b/vendor/src/github.com/minio/minio-go/api-stat.go @@ -21,8 +21,6 @@ import ( "strconv" "strings" "time" - - "github.com/minio/minio-go/pkg/s3utils" ) // BucketExists verify if bucket exists and you have permission to access it. @@ -126,12 +124,13 @@ func (c Client) statObject(bucketName, objectName string, reqHeaders RequestHead md5sum := strings.TrimPrefix(resp.Header.Get("ETag"), "\"") md5sum = strings.TrimSuffix(md5sum, "\"") - // Content-Length is not valid for Google Cloud Storage, do not verify. + // Parse content length is exists var size int64 = -1 - if !s3utils.IsGoogleEndpoint(c.endpointURL) { - // Parse content length. - size, err = strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) + contentLengthStr := resp.Header.Get("Content-Length") + if contentLengthStr != "" { + size, err = strconv.ParseInt(contentLengthStr, 10, 64) if err != nil { + // Content-Length is not valid return ObjectInfo{}, ErrorResponse{ Code: "InternalError", Message: "Content-Length is invalid. " + reportIssue, diff --git a/vendor/src/github.com/minio/minio-go/api_functional_v4_test.go b/vendor/src/github.com/minio/minio-go/api_functional_v4_test.go index a553ea2cd..80ddf80bd 100644 --- a/vendor/src/github.com/minio/minio-go/api_functional_v4_test.go +++ b/vendor/src/github.com/minio/minio-go/api_functional_v4_test.go @@ -1962,6 +1962,7 @@ func TestEncryptionPutGet(t *testing.T) { if err != nil { t.Fatalf("Test %d, error: %v %v %v", i+1, err, bucketName, objectName) } + defer r.Close() // Compare the sent object with the received one recvBuffer := bytes.NewBuffer([]byte{}) diff --git a/vendor/src/github.com/minio/minio-go/docs/API.md b/vendor/src/github.com/minio/minio-go/docs/API.md index 06735427e..0782aa509 100644 --- a/vendor/src/github.com/minio/minio-go/docs/API.md +++ b/vendor/src/github.com/minio/minio-go/docs/API.md @@ -771,7 +771,7 @@ if err != nil { ``` -### GetEncryptedObject(bucketName, objectName string, encryptMaterials minio.EncryptionMaterials) (io.Reader, error) +### GetEncryptedObject(bucketName, objectName string, encryptMaterials minio.EncryptionMaterials) (io.ReadCloser, error) Returns the decrypted stream of the object data based of the given encryption materiels. Most of the common errors occur when reading the stream. @@ -788,7 +788,7 @@ __Return Value__ |Param |Type |Description | |:---|:---| :---| -|`stream` | _io.Reader_ | Returns the deciphered object reader. | +|`stream` | _io.ReadCloser_ | Returns the deciphered object reader, caller should close after reading. | |`err` | _error | Returns errors. | @@ -810,11 +810,14 @@ if err != nil { fmt.Println(err) return } +defer object.Close() + localFile, err := os.Create("/tmp/local-file.jpg") if err != nil { fmt.Println(err) return } + if _, err = io.Copy(localFile, object); err != nil { fmt.Println(err) return diff --git a/vendor/src/github.com/minio/minio-go/examples/s3/get-encrypted-object.go b/vendor/src/github.com/minio/minio-go/examples/s3/get-encrypted-object.go index e997140be..8f51f26ae 100644 --- a/vendor/src/github.com/minio/minio-go/examples/s3/get-encrypted-object.go +++ b/vendor/src/github.com/minio/minio-go/examples/s3/get-encrypted-object.go @@ -24,6 +24,7 @@ import ( "os" "github.com/minio/minio-go" + "github.com/minio/minio-go/pkg/encrypt" ) func main() { @@ -59,10 +60,10 @@ func main() { //// // Build a symmetric key - symmetricKey := minio.NewSymmetricKey([]byte("my-secret-key-00")) + symmetricKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) // Build encryption materials which will encrypt uploaded data - cbcMaterials, err := minio.NewCBCSecureMaterials(symmetricKey) + cbcMaterials, err := encrypt.NewCBCSecureMaterials(symmetricKey) if err != nil { log.Fatalln(err) } @@ -72,6 +73,7 @@ func main() { if err != nil { log.Fatalln(err) } + defer reader.Close() // Local file which holds plain data localFile, err := os.Create("my-testfile") diff --git a/vendor/src/github.com/minio/minio-go/examples/s3/put-encrypted-object.go b/vendor/src/github.com/minio/minio-go/examples/s3/put-encrypted-object.go index f03f82147..b8f7e12f2 100644 --- a/vendor/src/github.com/minio/minio-go/examples/s3/put-encrypted-object.go +++ b/vendor/src/github.com/minio/minio-go/examples/s3/put-encrypted-object.go @@ -23,6 +23,7 @@ import ( "os" "github.com/minio/minio-go" + "github.com/minio/minio-go/pkg/encrypt" ) func main() { @@ -65,10 +66,10 @@ func main() { //// // Build a symmetric key - symmetricKey := minio.NewSymmetricKey([]byte("my-secret-key-00")) + symmetricKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) // Build encryption materials which will encrypt uploaded data - cbcMaterials, err := minio.NewCBCSecureMaterials(symmetricKey) + cbcMaterials, err := encrypt.NewCBCSecureMaterials(symmetricKey) if err != nil { log.Fatalln(err) } diff --git a/vendor/src/github.com/minio/minio-go/examples/s3/putobject-getobject-sse.go b/vendor/src/github.com/minio/minio-go/examples/s3/putobject-getobject-sse.go new file mode 100644 index 000000000..92e6a4840 --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/examples/s3/putobject-getobject-sse.go @@ -0,0 +1,87 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "io/ioutil" + "log" + "net/http" + + minio "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + minioClient, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + content := bytes.NewReader([]byte("Hello again")) + key := []byte("32byteslongsecretkeymustprovided") + h := md5.New() + h.Write(key) + encryptionKey := base64.StdEncoding.EncodeToString(key) + encryptionKeyMD5 := base64.StdEncoding.EncodeToString(h.Sum(nil)) + + // Amazon S3 does not store the encryption key you provide. + // Instead S3 stores a randomly salted HMAC value of the + // encryption key in order to validate future requests. + // The salted HMAC value cannot be used to derive the value + // of the encryption key or to decrypt the contents of the + // encrypted object. That means, if you lose the encryption + // key, you lose the object. + var metadata = map[string][]string{ + "x-amz-server-side-encryption-customer-algorithm": []string{"AES256"}, + "x-amz-server-side-encryption-customer-key": []string{encryptionKey}, + "x-amz-server-side-encryption-customer-key-MD5": []string{encryptionKeyMD5}, + } + + // minioClient.TraceOn(os.Stderr) // Enable to debug. + _, err = minioClient.PutObjectWithMetadata("mybucket", "my-encrypted-object.txt", content, metadata, nil) + if err != nil { + log.Fatalln(err) + } + + var reqHeaders = minio.RequestHeaders{Header: http.Header{}} + for k, v := range metadata { + reqHeaders.Set(k, v[0]) + } + coreClient := minio.Core{minioClient} + reader, _, err := coreClient.GetObject("mybucket", "my-encrypted-object.txt", reqHeaders) + if err != nil { + log.Fatalln(err) + } + defer reader.Close() + + decBytes, err := ioutil.ReadAll(reader) + if err != nil { + log.Fatalln(err) + } + if !bytes.Equal(decBytes, []byte("Hello again")) { + log.Fatalln("Expected \"Hello, world\", got %s", string(decBytes)) + } +} diff --git a/vendor/src/github.com/minio/minio-go/pkg/encrypt/cbc.go b/vendor/src/github.com/minio/minio-go/pkg/encrypt/cbc.go index 7670e68f4..be45e52f4 100644 --- a/vendor/src/github.com/minio/minio-go/pkg/encrypt/cbc.go +++ b/vendor/src/github.com/minio/minio-go/pkg/encrypt/cbc.go @@ -89,6 +89,15 @@ func NewCBCSecureMaterials(key Key) (*CBCSecureMaterials, error) { } +// Close implements closes the internal stream. +func (s *CBCSecureMaterials) Close() error { + closer, ok := s.stream.(io.Closer) + if ok { + return closer.Close() + } + return nil +} + // SetupEncryptMode - tells CBC that we are going to encrypt data func (s *CBCSecureMaterials) SetupEncryptMode(stream io.Reader) error { // Set mode to encrypt diff --git a/vendor/src/github.com/minio/minio-go/pkg/encrypt/interface.go b/vendor/src/github.com/minio/minio-go/pkg/encrypt/interface.go index 2fd75033f..8b8554336 100644 --- a/vendor/src/github.com/minio/minio-go/pkg/encrypt/interface.go +++ b/vendor/src/github.com/minio/minio-go/pkg/encrypt/interface.go @@ -25,6 +25,9 @@ import "io" // Materials - provides generic interface to encrypt any stream of data. type Materials interface { + // Closes the wrapped stream properly, initiated by the caller. + Close() error + // Returns encrypted/decrypted data, io.Reader compatible. Read(b []byte) (int, error) diff --git a/vendor/src/github.com/minio/minio-go/request-headers.go b/vendor/src/github.com/minio/minio-go/request-headers.go index fa23b2fe3..76c87202d 100644 --- a/vendor/src/github.com/minio/minio-go/request-headers.go +++ b/vendor/src/github.com/minio/minio-go/request-headers.go @@ -48,7 +48,7 @@ func (c RequestHeaders) SetMatchETag(etag string) error { if etag == "" { return ErrInvalidArgument("ETag cannot be empty.") } - c.Set("If-Match", etag) + c.Set("If-Match", "\""+etag+"\"") return nil } @@ -57,7 +57,7 @@ func (c RequestHeaders) SetMatchETagExcept(etag string) error { if etag == "" { return ErrInvalidArgument("ETag cannot be empty.") } - c.Set("If-None-Match", etag) + c.Set("If-None-Match", "\""+etag+"\"") return nil } diff --git a/vendor/src/github.com/minio/minio-go/s3-error.go b/vendor/src/github.com/minio/minio-go/s3-error.go index 11b40a0f8..c5aff9bbc 100644 --- a/vendor/src/github.com/minio/minio-go/s3-error.go +++ b/vendor/src/github.com/minio/minio-go/s3-error.go @@ -25,7 +25,7 @@ var s3ErrorResponseMap = map[string]string{ "EntityTooLarge": "Your proposed upload exceeds the maximum allowed object size.", "IncompleteBody": "You did not provide the number of bytes specified by the Content-Length HTTP header.", "InternalError": "We encountered an internal error, please try again.", - "InvalidAccessKeyID": "The access key ID you provided does not exist in our records.", + "InvalidAccessKeyId": "The access key ID you provided does not exist in our records.", "InvalidBucketName": "The specified bucket is not valid.", "InvalidDigest": "The Content-Md5 you specified is not valid.", "InvalidRange": "The requested range is not satisfiable", From a325a20fb42ef610340e0b5d8891642dfa839b2a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 8 Jun 2017 20:50:56 +0200 Subject: [PATCH 2/2] s3: Increase wait time for minio server --- src/restic/backend/s3/s3_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/restic/backend/s3/s3_test.go b/src/restic/backend/s3/s3_test.go index 7736ab858..470a7ab5f 100644 --- a/src/restic/backend/s3/s3_test.go +++ b/src/restic/backend/s3/s3_test.go @@ -49,7 +49,7 @@ func runMinio(ctx context.Context, t testing.TB, dir, key, secret string) func() // wait until the TCP port is reachable var success bool - for i := 0; i < 10; i++ { + for i := 0; i < 100; i++ { time.Sleep(200 * time.Millisecond) c, err := net.Dial("tcp", "localhost:9000")