From 1b50d55d0c56ed5fbfcd261b6b06d29fd0fbad99 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 8 May 2016 11:24:24 +0200 Subject: [PATCH] Update minio-go --- vendor/manifest | 2 +- vendor/src/github.com/minio/minio-go/API.md | 91 +++-- .../minio/minio-go/api-error-response_test.go | 277 +++++++++++++++ .../minio/minio-go/api-get-policy.go | 2 +- .../minio/minio-go/api-get-policy_test.go | 50 +-- .../src/github.com/minio/minio-go/api-list.go | 4 +- .../minio/minio-go/api-put-bucket_test.go | 108 +++--- vendor/src/github.com/minio/minio-go/api.go | 32 +- .../github.com/minio/minio-go/bucket-cache.go | 19 +- .../minio/minio-go/bucket-cache_test.go | 320 ++++++++++++++++++ .../minio-go/examples/s3/listobjects-N.go | 76 +++++ vendor/src/github.com/minio/minio-go/retry.go | 33 +- .../minio/minio-go/test-utils_test.go | 64 ++++ 13 files changed, 912 insertions(+), 166 deletions(-) create mode 100644 vendor/src/github.com/minio/minio-go/api-error-response_test.go create mode 100644 vendor/src/github.com/minio/minio-go/bucket-cache_test.go create mode 100644 vendor/src/github.com/minio/minio-go/examples/s3/listobjects-N.go create mode 100644 vendor/src/github.com/minio/minio-go/test-utils_test.go diff --git a/vendor/manifest b/vendor/manifest index bca4ae098..1b55eeb43 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -28,7 +28,7 @@ { "importpath": "github.com/minio/minio-go", "repository": "https://github.com/minio/minio-go", - "revision": "17b4ebd52505bde655e3b14df732e31850641bb7", + "revision": "867b27701ad16db4a9f4dad40d28187ca8433ec9", "branch": "master" }, { diff --git a/vendor/src/github.com/minio/minio-go/API.md b/vendor/src/github.com/minio/minio-go/API.md index 24429d835..8e1c82dc9 100644 --- a/vendor/src/github.com/minio/minio-go/API.md +++ b/vendor/src/github.com/minio/minio-go/API.md @@ -60,10 +60,10 @@ s3Client can be used to perform operations on S3 storage. APIs are described bel ### Bucket operations --------------------------------------- -#### MakeBucket(bucketName, location) +#### MakeBucket(bucketName string, location string) error Create a new bucket. -__Arguments__ +__Parameters__ * `bucketName` _string_ - Name of the bucket. * `location` _string_ - region valid values are _us-west-1_, _us-west-2_, _eu-west-1_, _eu-central-1_, _ap-southeast-1_, _ap-northeast-1_, _ap-southeast-2_, _sa-east-1_ @@ -78,10 +78,10 @@ fmt.Println("Successfully created mybucket.") ``` --------------------------------------- -#### ListBuckets() -List all buckets. +#### ListBuckets() ([]BucketInfo, error) +Lists all buckets. -`bucketList` emits bucket with the format: +`bucketList` lists bucket in the format: * `bucket.Name` _string_: bucket name * `bucket.CreationDate` time.Time : date when bucket was created @@ -98,10 +98,10 @@ for _, bucket := range buckets { ``` --------------------------------------- -#### BucketExists(bucketName) +#### BucketExists(bucketName string) error Check if bucket exists. -__Arguments__ +__Parameters__ * `bucketName` _string_ : name of the bucket __Example__ @@ -114,10 +114,10 @@ if err != nil { ``` --------------------------------------- -#### RemoveBucket(bucketName) +#### RemoveBucket(bucketName string) error Remove a bucket. -__Arguments__ +__Parameters__ * `bucketName` _string_ : name of the bucket __Example__ @@ -130,16 +130,16 @@ if err != nil { ``` --------------------------------------- -#### GetBucketPolicy(bucketName, objectPrefix) +#### GetBucketPolicy(bucketName string, objectPrefix string) error Get access permissions on a bucket or a prefix. -__Arguments__ +__Parameters__ * `bucketName` _string_ : name of the bucket * `objectPrefix` _string_ : name of the object prefix __Example__ ```go -bucketPolicy, err := s3Client.GetBucketPolicy("mybucket") +bucketPolicy, err := s3Client.GetBucketPolicy("mybucket", "") if err != nil { fmt.Println(err) return @@ -148,13 +148,13 @@ fmt.Println("Access permissions for mybucket is", bucketPolicy) ``` --------------------------------------- -#### SetBucketPolicy(bucketname, objectPrefix, policy) +#### SetBucketPolicy(bucketname string, objectPrefix string, policy BucketPolicy) error Set access permissions on bucket or an object prefix. -__Arguments__ +__Parameters__ * `bucketName` _string_: name of the bucket * `objectPrefix` _string_ : name of the object prefix -* `policy` _BucketPolicy_: policy can be _non_, _readonly_, _readwrite_, _writeonly_ +* `policy` _BucketPolicy_: policy can be _BucketPolicyNone_, _BucketPolicyReadOnly_, _BucketPolicyReadWrite_, _BucketPolicyWriteOnly_ __Example__ ```go @@ -166,10 +166,10 @@ if err != nil { ``` --------------------------------------- -#### RemoveBucketPolicy(bucketname, objectPrefix) +#### RemoveBucketPolicy(bucketname string, objectPrefix string) error Remove existing permissions on bucket or an object prefix. -__Arguments__ +__Parameters__ * `bucketName` _string_: name of the bucket * `objectPrefix` _string_ : name of the object prefix @@ -184,10 +184,10 @@ if err != nil { --------------------------------------- -#### ListObjects(bucketName, prefix, recursive, doneCh) +#### ListObjects(bucketName string, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectInfo List objects in a bucket. -__Arguments__ +__Parameters__ * `bucketName` _string_: name of the bucket * `objectPrefix` _string_: the prefix of the objects that should be listed * `recursive` _bool_: `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/' @@ -222,10 +222,10 @@ for object := range objectCh { --------------------------------------- -#### ListIncompleteUploads(bucketName, prefix, recursive) +#### ListIncompleteUploads(bucketName string, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectMultipartInfo List partially uploaded objects in a bucket. -__Arguments__ +__Parameters__ * `bucketname` _string_: name of the bucket * `prefix` _string_: prefix of the object names that are partially uploaded * `recursive` bool: directory style listing when false, recursive listing when true @@ -259,15 +259,15 @@ for multiPartObject := range multiPartObjectCh { --------------------------------------- ### Object operations -#### GetObject(bucketName, objectName) +#### GetObject(bucketName string, objectName string) *Object Download an object. -__Arguments__ +__Parameters__ * `bucketName` _string_: name of the bucket * `objectName` _string_: name of the object __Return Value__ -* `object` _*minio.Object_ : _minio.Object_ represents object reader. +* `object` _*Object_ : _Object_ represents object reader. __Example__ ```go @@ -285,10 +285,10 @@ if _, err := io.Copy(localFile, object); err != nil { --------------------------------------- --------------------------------------- -#### FGetObject(bucketName, objectName, filePath) +#### FGetObject(bucketName string, objectName string, filePath string) error Callback is called with `error` in case of error or `null` in case of success -__Arguments__ +__Parameters__ * `bucketName` _string_: name of the bucket * `objectName` _string_: name of the object * `filePath` _string_: path to which the object data will be written to @@ -303,11 +303,10 @@ if err != nil { ``` --------------------------------------- -#### PutObject(bucketName, objectName, reader, contentType) -Upload an object. +#### PutObject(bucketName string, objectName string, reader io.Reader, contentType string) (n int, err error) +Upload contents from `io.Reader` to objectName. -Uploading a stream -__Arguments__ +__Parameters__ * `bucketName` _string_: name of the bucket * `objectName` _string_: name of the object * `reader` _io.Reader_: Any golang object implementing io.Reader @@ -331,10 +330,10 @@ if err != nil { --------------------------------------- -#### CopyObject(bucketName, objectName, objectSource, conditions) +#### CopyObject(bucketName string, objectName string, objectSource string, conditions CopyConditions) error Copy a source object into a new object with the provided name in the provided bucket. -__Arguments__ +__Parameters__ * `bucketName` _string_: name of the bucket * `objectName` _string_: name of the object * `objectSource` _string_: name of the object source. @@ -367,10 +366,10 @@ if err != nil { --------------------------------------- -#### FPutObject(bucketName, objectName, filePath, contentType) +#### FPutObject(bucketName string, objectName string, filePath string, contentType string) error Uploads the object using contents from a file -__Arguments__ +__Parameters__ * `bucketName` _string_: name of the bucket * `objectName` _string_: name of the object * `filePath` _string_: file path of the file to be uploaded @@ -386,10 +385,10 @@ if err != nil { ``` --------------------------------------- -#### StatObject(bucketName, objectName) +#### StatObject(bucketName string, objectName string) (ObjectInfo, error) Get metadata of an object. -__Arguments__ +__Parameters__ * `bucketName` _string_: name of the bucket * `objectName` _string_: name of the object @@ -411,10 +410,10 @@ fmt.Println(objInfo) ``` --------------------------------------- -#### RemoveObject(bucketName, objectName) +#### RemoveObject(bucketName string, objectName string) error Remove an object. -__Arguments__ +__Parameters__ * `bucketName` _string_: name of the bucket * `objectName` _string_: name of the object @@ -428,10 +427,10 @@ if err != nil { ``` --------------------------------------- -#### RemoveIncompleteUpload(bucketName, objectName) +#### RemoveIncompleteUpload(bucketName string, objectName string) error Remove an partially uploaded object. -__Arguments__ +__Parameters__ * `bucketName` _string_: name of the bucket * `objectName` _string_: name of the object @@ -447,14 +446,14 @@ if err != nil { ### Presigned operations --------------------------------------- -#### PresignedGetObject(bucketName, objectName, expiry) +#### PresignedGetObject(bucketName, objectName string, expiry time.Duration, reqParams url.Values) error Generate a presigned URL for GET. -__Arguments__ +__Parameters__ * `bucketName` _string_: name of the bucket. * `objectName` _string_: name of the object. * `expiry` _time.Duration_: expiry in seconds. - `reqParams` _url.Values_ : additional response header overrides supports _response-expires_, _response-content-type_, _response-cache-control_, _response-content-disposition_ +* `reqParams` _url.Values_ : additional response header overrides supports _response-expires_, _response-content-type_, _response-cache-control_, _response-content-disposition_ __Example__ ```go @@ -472,13 +471,13 @@ if err != nil { --------------------------------------- -#### PresignedPutObject(bucketName, objectName, expiry) +#### PresignedPutObject(bucketName string, objectName string, expiry time.Duration) (string, error) Generate a presigned URL for PUT.
NOTE: you can upload to S3 only with specified object name.
-__Arguments__ +__Parameters__ * `bucketName` _string_: name of the bucket * `objectName` _string_: name of the object * `expiry` _time.Duration_: expiry in seconds @@ -495,7 +494,7 @@ if err != nil { --------------------------------------- -#### PresignedPostPolicy +#### PresignedPostPolicy(policy PostPolicy) (map[string]string, error) PresignedPostPolicy we can provide policies specifying conditions restricting what you want to allow in a POST request, such as bucket name where objects can be uploaded, key name prefixes that you want to allow for the object being created and more. diff --git a/vendor/src/github.com/minio/minio-go/api-error-response_test.go b/vendor/src/github.com/minio/minio-go/api-error-response_test.go new file mode 100644 index 000000000..a4e5bdc0f --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/api-error-response_test.go @@ -0,0 +1,277 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required bZy applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "encoding/xml" + "fmt" + "io/ioutil" + "net/http" + "reflect" + "strconv" + "testing" +) + +// Tests validate the Error generator function for http response with error. +func TestHttpRespToErrorResponse(t *testing.T) { + // 'genAPIErrorResponse' generates ErrorResponse for given APIError. + // provides a encodable populated response values. + genAPIErrorResponse := func(err APIError, bucketName string) ErrorResponse { + var errResp = ErrorResponse{} + errResp.Code = err.Code + errResp.Message = err.Description + errResp.BucketName = bucketName + return errResp + } + + // Encodes the response headers into XML format. + encodeErr := func(response interface{}) []byte { + var bytesBuffer bytes.Buffer + bytesBuffer.WriteString(xml.Header) + encode := xml.NewEncoder(&bytesBuffer) + encode.Encode(response) + return bytesBuffer.Bytes() + } + + // `createAPIErrorResponse` Mocks XML error response from the server. + createAPIErrorResponse := func(APIErr APIError, bucketName string) *http.Response { + // generate error response. + // response body contains the XML error message. + resp := &http.Response{} + errorResponse := genAPIErrorResponse(APIErr, bucketName) + encodedErrorResponse := encodeErr(errorResponse) + // write Header. + resp.StatusCode = APIErr.HTTPStatusCode + resp.Body = ioutil.NopCloser(bytes.NewBuffer(encodedErrorResponse)) + + return resp + } + + // 'genErrResponse' contructs error response based http Status Code + genErrResponse := func(resp *http.Response, code, message, bucketName, objectName string) ErrorResponse { + errResp := ErrorResponse{ + Code: code, + Message: message, + BucketName: bucketName, + Key: objectName, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), + } + return errResp + } + + // Generate invalid argument error. + genInvalidError := func(message string) error { + errResp := ErrorResponse{ + Code: "InvalidArgument", + Message: message, + RequestID: "minio", + } + return errResp + } + + // Set common http response headers. + setCommonHeaders := func(resp *http.Response) *http.Response { + // set headers. + resp.Header = make(http.Header) + resp.Header.Set("x-amz-request-id", "xyz") + resp.Header.Set("x-amz-id-2", "abc") + resp.Header.Set("x-amz-bucket-region", "us-east-1") + return resp + } + + // Generate http response with empty body. + // Set the StatusCode to the arugment supplied. + // Sets common headers. + genEmptyBodyResponse := func(statusCode int) *http.Response { + resp := &http.Response{} + // set empty response body. + resp.Body = ioutil.NopCloser(bytes.NewBuffer([]byte(""))) + // set headers. + setCommonHeaders(resp) + // set status code. + resp.StatusCode = statusCode + return resp + } + + // Decode XML error message from the http response body. + decodeXMLError := func(resp *http.Response, t *testing.T) error { + var errResp ErrorResponse + err := xmlDecoder(resp.Body, &errResp) + if err != nil { + t.Fatal("XML decoding of response body failed") + } + return errResp + } + + // List of APIErrors used to generate/mock server side XML error response. + APIErrors := []APIError{ + { + Code: "NoSuchBucketPolicy", + Description: "The specified bucket does not have a bucket policy.", + HTTPStatusCode: http.StatusNotFound, + }, + } + + // List of expected response. + // Used for asserting the actual response. + expectedErrResponse := []error{ + genInvalidError("Response is empty. " + "Please report this issue at https://github.com/minio/minio-go/issues."), + decodeXMLError(createAPIErrorResponse(APIErrors[0], "minio-bucket"), t), + genErrResponse(setCommonHeaders(&http.Response{}), "NoSuchBucket", "The specified bucket does not exist.", "minio-bucket", ""), + genErrResponse(setCommonHeaders(&http.Response{}), "NoSuchKey", "The specified key does not exist.", "minio-bucket", "Asia/"), + genErrResponse(setCommonHeaders(&http.Response{}), "AccessDenied", "Access Denied.", "minio-bucket", ""), + genErrResponse(setCommonHeaders(&http.Response{}), "Conflict", "Bucket not empty.", "minio-bucket", ""), + genErrResponse(setCommonHeaders(&http.Response{}), "Bad Request", "Bad Request", "minio-bucket", ""), + } + + // List of http response to be used as input. + inputResponses := []*http.Response{ + nil, + createAPIErrorResponse(APIErrors[0], "minio-bucket"), + genEmptyBodyResponse(http.StatusNotFound), + genEmptyBodyResponse(http.StatusNotFound), + genEmptyBodyResponse(http.StatusForbidden), + genEmptyBodyResponse(http.StatusConflict), + genEmptyBodyResponse(http.StatusBadRequest), + } + + testCases := []struct { + bucketName string + objectName string + inputHTTPResp *http.Response + // expected results. + expectedResult error + // flag indicating whether tests should pass. + + }{ + {"minio-bucket", "", inputResponses[0], expectedErrResponse[0]}, + {"minio-bucket", "", inputResponses[1], expectedErrResponse[1]}, + {"minio-bucket", "", inputResponses[2], expectedErrResponse[2]}, + {"minio-bucket", "Asia/", inputResponses[3], expectedErrResponse[3]}, + {"minio-bucket", "", inputResponses[4], expectedErrResponse[4]}, + {"minio-bucket", "", inputResponses[5], expectedErrResponse[5]}, + } + + for i, testCase := range testCases { + actualResult := httpRespToErrorResponse(testCase.inputHTTPResp, testCase.bucketName, testCase.objectName) + if !reflect.DeepEqual(testCase.expectedResult, actualResult) { + t.Errorf("Test %d: Expected result to be '%+v', but instead got '%+v'", i+1, testCase.expectedResult, actualResult) + } + } +} + +// Test validates 'ErrEntityTooLarge' error response. +func TestErrEntityTooLarge(t *testing.T) { + msg := fmt.Sprintf("Your proposed upload size ‘%d’ exceeds the maximum allowed object size ‘%d’ for single PUT operation.", 1000000, 99999) + expectedResult := ErrorResponse{ + Code: "EntityTooLarge", + Message: msg, + BucketName: "minio-bucket", + Key: "Asia/", + } + actualResult := ErrEntityTooLarge(1000000, 99999, "minio-bucket", "Asia/") + if !reflect.DeepEqual(expectedResult, actualResult) { + t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult) + } +} + +// Test validates 'ErrEntityTooSmall' error response. +func TestErrEntityTooSmall(t *testing.T) { + msg := fmt.Sprintf("Your proposed upload size ‘%d’ is below the minimum allowed object size '0B' for single PUT operation.", -1) + expectedResult := ErrorResponse{ + Code: "EntityTooLarge", + Message: msg, + BucketName: "minio-bucket", + Key: "Asia/", + } + actualResult := ErrEntityTooSmall(-1, "minio-bucket", "Asia/") + if !reflect.DeepEqual(expectedResult, actualResult) { + t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult) + } +} + +// Test validates 'ErrUnexpectedEOF' error response. +func TestErrUnexpectedEOF(t *testing.T) { + msg := fmt.Sprintf("Data read ‘%s’ is not equal to the size ‘%s’ of the input Reader.", + strconv.FormatInt(100, 10), strconv.FormatInt(101, 10)) + expectedResult := ErrorResponse{ + Code: "UnexpectedEOF", + Message: msg, + BucketName: "minio-bucket", + Key: "Asia/", + } + actualResult := ErrUnexpectedEOF(100, 101, "minio-bucket", "Asia/") + if !reflect.DeepEqual(expectedResult, actualResult) { + t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult) + } +} + +// Test validates 'ErrInvalidBucketName' error response. +func TestErrInvalidBucketName(t *testing.T) { + expectedResult := ErrorResponse{ + Code: "InvalidBucketName", + Message: "Invalid Bucket name", + RequestID: "minio", + } + actualResult := ErrInvalidBucketName("Invalid Bucket name") + if !reflect.DeepEqual(expectedResult, actualResult) { + t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult) + } +} + +// Test validates 'ErrInvalidObjectName' error response. +func TestErrInvalidObjectName(t *testing.T) { + expectedResult := ErrorResponse{ + Code: "NoSuchKey", + Message: "Invalid Object Key", + RequestID: "minio", + } + actualResult := ErrInvalidObjectName("Invalid Object Key") + if !reflect.DeepEqual(expectedResult, actualResult) { + t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult) + } +} + +// Test validates 'ErrInvalidParts' error response. +func TestErrInvalidParts(t *testing.T) { + msg := fmt.Sprintf("Unexpected number of parts found Want %d, Got %d", 10, 9) + expectedResult := ErrorResponse{ + Code: "InvalidParts", + Message: msg, + RequestID: "minio", + } + actualResult := ErrInvalidParts(10, 9) + if !reflect.DeepEqual(expectedResult, actualResult) { + t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult) + } +} + +// Test validates 'ErrInvalidArgument' response. +func TestErrInvalidArgument(t *testing.T) { + expectedResult := ErrorResponse{ + Code: "InvalidArgument", + Message: "Invalid Argument", + RequestID: "minio", + } + actualResult := ErrInvalidArgument("Invalid Argument") + if !reflect.DeepEqual(expectedResult, actualResult) { + t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult) + } +} diff --git a/vendor/src/github.com/minio/minio-go/api-get-policy.go b/vendor/src/github.com/minio/minio-go/api-get-policy.go index bc86ebd04..1004461b2 100644 --- a/vendor/src/github.com/minio/minio-go/api-get-policy.go +++ b/vendor/src/github.com/minio/minio-go/api-get-policy.go @@ -61,7 +61,7 @@ func (c Client) getBucketPolicy(bucketName string, objectPrefix string) (BucketA } -// processes the GetPolicy http resposne from the server. +// processes the GetPolicy http response from the server. func processBucketPolicyResponse(bucketName string, resp *http.Response) (BucketAccessPolicy, error) { if resp != nil { if resp.StatusCode != http.StatusOK { diff --git a/vendor/src/github.com/minio/minio-go/api-get-policy_test.go b/vendor/src/github.com/minio/minio-go/api-get-policy_test.go index 83ae81e03..a15f53577 100644 --- a/vendor/src/github.com/minio/minio-go/api-get-policy_test.go +++ b/vendor/src/github.com/minio/minio-go/api-get-policy_test.go @@ -19,65 +19,24 @@ package minio import ( "bytes" "encoding/json" - "encoding/xml" "io/ioutil" "net/http" "reflect" "testing" ) -type APIError struct { - Code string - Description string - HTTPStatusCode int -} - -// Mocks XML error response from the server. -func generateErrorResponse(resp *http.Response, APIErr APIError, bucketName string) *http.Response { - // generate error response. - errorResponse := getAPIErrorResponse(APIErr, bucketName) - encodedErrorResponse := encodeResponse(errorResponse) - // write Header. - resp.StatusCode = APIErr.HTTPStatusCode - resp.Body = ioutil.NopCloser(bytes.NewBuffer(encodedErrorResponse)) - - return resp -} - -// getErrorResponse gets in standard error and resource value and -// provides a encodable populated response values. -func getAPIErrorResponse(err APIError, bucketName string) ErrorResponse { - var data = ErrorResponse{} - data.Code = err.Code - data.Message = err.Description - - data.BucketName = bucketName - // TODO implement this in future - return data -} - -// Encodes the response headers into XML format. -func encodeResponse(response interface{}) []byte { - var bytesBuffer bytes.Buffer - bytesBuffer.WriteString(xml.Header) - encode := xml.NewEncoder(&bytesBuffer) - encode.Encode(response) - return bytesBuffer.Bytes() -} - // Mocks valid http response containing bucket policy from server. func generatePolicyResponse(resp *http.Response, policy BucketAccessPolicy) (*http.Response, error) { - policyBytes, e := json.Marshal(policy) - if e != nil { - return nil, e + policyBytes, err := json.Marshal(policy) + if err != nil { + return nil, err } resp.StatusCode = http.StatusOK resp.Body = ioutil.NopCloser(bytes.NewBuffer(policyBytes)) return resp, nil - } -// Tests the processing of GetPolicy resposne from server. +// Tests the processing of GetPolicy response from server. func TestProcessBucketPolicyResopnse(t *testing.T) { bucketAccesPolicies := []BucketAccessPolicy{ {Version: "1.0"}, @@ -139,6 +98,5 @@ func TestProcessBucketPolicyResopnse(t *testing.T) { t.Errorf("Test %d: The expected BucketPolicy doesnt match the actual BucketPolicy", i+1) } } - } } diff --git a/vendor/src/github.com/minio/minio-go/api-list.go b/vendor/src/github.com/minio/minio-go/api-list.go index b17a51acc..e61763db9 100644 --- a/vendor/src/github.com/minio/minio-go/api-list.go +++ b/vendor/src/github.com/minio/minio-go/api-list.go @@ -77,7 +77,7 @@ func (c Client) ListBuckets() ([]BucketInfo, error) { // func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { // Allocate new list objects channel. - objectStatCh := make(chan ObjectInfo, 1000) + objectStatCh := make(chan ObjectInfo) // Default listing is delimited at "/" delimiter := "/" if recursive { @@ -254,7 +254,7 @@ func (c Client) ListIncompleteUploads(bucketName, objectPrefix string, recursive // listIncompleteUploads lists all incomplete uploads. func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive, aggregateSize bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { // Allocate channel for multipart uploads. - objectMultipartStatCh := make(chan ObjectMultipartInfo, 1000) + objectMultipartStatCh := make(chan ObjectMultipartInfo) // Delimiter is set to "/" by default. delimiter := "/" if recursive { diff --git a/vendor/src/github.com/minio/minio-go/api-put-bucket_test.go b/vendor/src/github.com/minio/minio-go/api-put-bucket_test.go index ce8b27dd0..050feed0e 100644 --- a/vendor/src/github.com/minio/minio-go/api-put-bucket_test.go +++ b/vendor/src/github.com/minio/minio-go/api-put-bucket_test.go @@ -27,71 +27,71 @@ import ( "testing" ) -// Generates expected http request for bucket creation. -// Used for asserting with the actual request generated. -func createExpectedRequest(c *Client, bucketName string, location string, req *http.Request) (*http.Request, error) { +// Tests validate http request formulated for creation of bucket. +func TestMakeBucketRequest(t *testing.T) { + // Generates expected http request for bucket creation. + // Used for asserting with the actual request generated. + createExpectedRequest := func(c *Client, bucketName string, location string, req *http.Request) (*http.Request, error) { - targetURL := *c.endpointURL - targetURL.Path = "/" + bucketName + "/" + targetURL := *c.endpointURL + targetURL.Path = "/" + bucketName + "/" - // get a new HTTP request for the method. - req, err := http.NewRequest("PUT", targetURL.String(), nil) - if err != nil { - return nil, err - } - - // set UserAgent for the request. - c.setUserAgent(req) - - // set sha256 sum for signature calculation only with signature version '4'. - if c.signature.isV4() { - req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{}))) - } - - // If location is not 'us-east-1' create bucket location config. - if location != "us-east-1" && location != "" { - createBucketConfig := createBucketConfiguration{} - createBucketConfig.Location = location - var createBucketConfigBytes []byte - createBucketConfigBytes, err = xml.Marshal(createBucketConfig) + // get a new HTTP request for the method. + req, err := http.NewRequest("PUT", targetURL.String(), nil) if err != nil { return nil, err } - createBucketConfigBuffer := bytes.NewBuffer(createBucketConfigBytes) - req.Body = ioutil.NopCloser(createBucketConfigBuffer) - req.ContentLength = int64(len(createBucketConfigBytes)) - // Set content-md5. - req.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(sumMD5(createBucketConfigBytes))) + + // set UserAgent for the request. + c.setUserAgent(req) + + // set sha256 sum for signature calculation only with signature version '4'. if c.signature.isV4() { - // Set sha256. - req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256(createBucketConfigBytes))) + req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{}))) } + + // If location is not 'us-east-1' create bucket location config. + if location != "us-east-1" && location != "" { + createBucketConfig := createBucketConfiguration{} + createBucketConfig.Location = location + var createBucketConfigBytes []byte + createBucketConfigBytes, err = xml.Marshal(createBucketConfig) + if err != nil { + return nil, err + } + createBucketConfigBuffer := bytes.NewBuffer(createBucketConfigBytes) + req.Body = ioutil.NopCloser(createBucketConfigBuffer) + req.ContentLength = int64(len(createBucketConfigBytes)) + // Set content-md5. + req.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(sumMD5(createBucketConfigBytes))) + if c.signature.isV4() { + // Set sha256. + req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256(createBucketConfigBytes))) + } + } + + // Sign the request. + if c.signature.isV4() { + // Signature calculated for MakeBucket request should be for 'us-east-1', + // regardless of the bucket's location constraint. + req = signV4(*req, c.accessKeyID, c.secretAccessKey, "us-east-1") + } else if c.signature.isV2() { + req = signV2(*req, c.accessKeyID, c.secretAccessKey) + } + + // Return signed request. + return req, nil } - // Sign the request. - if c.signature.isV4() { - // Signature calculated for MakeBucket request should be for 'us-east-1', - // regardless of the bucket's location constraint. - req = signV4(*req, c.accessKeyID, c.secretAccessKey, "us-east-1") - } else if c.signature.isV2() { - req = signV2(*req, c.accessKeyID, c.secretAccessKey) + // Get Request body. + getReqBody := func(reqBody io.ReadCloser) (string, error) { + contents, err := ioutil.ReadAll(reqBody) + if err != nil { + return "", err + } + return string(contents), nil } - // Return signed request. - return req, nil -} - -// Get Request body. -func getReqBody(reqBody io.ReadCloser) (string, error) { - contents, err := ioutil.ReadAll(reqBody) - if err != nil { - return "", err - } - return string(contents), nil -} - -// Tests validate http request formulated for creation of bucket. -func TestMakeBucketRequest(t *testing.T) { // Info for 'Client' creation. // Will be used as arguments for 'NewClient'. type infoForClient struct { diff --git a/vendor/src/github.com/minio/minio-go/api.go b/vendor/src/github.com/minio/minio-go/api.go index a47cadd1e..93c0a6d23 100644 --- a/vendor/src/github.com/minio/minio-go/api.go +++ b/vendor/src/github.com/minio/minio-go/api.go @@ -419,11 +419,20 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt bodySeeker, isRetryable = metadata.contentBody.(io.Seeker) } - // Retry executes the following function body if request has an - // error until maxRetries have been exhausted, retry attempts are - // performed after waiting for a given period of time in a - // binomial fashion. - for range c.newRetryTimer(MaxRetry, time.Second, time.Second*30, MaxJitter) { + // Create a done channel to control 'ListObjects' go routine. + doneCh := make(chan struct{}, 1) + + // Indicate to our routine to exit cleanly upon return. + defer close(doneCh) + + // Blank indentifier is kept here on purpose since 'range' without + // blank identifiers is only supported since go1.4 + // https://golang.org/doc/go1.4#forrange. + for _ = range c.newRetryTimer(MaxRetry, time.Second, time.Second*30, MaxJitter, doneCh) { + // Retry executes the following function body if request has an + // error until maxRetries have been exhausted, retry attempts are + // performed after waiting for a given period of time in a + // binomial fashion. if isRetryable { // Seek back to beginning for each attempt. if _, err = bodySeeker.Seek(0, 0); err != nil { @@ -505,8 +514,18 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R if method == "" { method = "POST" } + + // Default all requests to "us-east-1" or "cn-north-1" (china region) + location := "us-east-1" + if isAmazonChinaEndpoint(c.endpointURL) { + // For china specifically we need to set everything to + // cn-north-1 for now, there is no easier way until AWS S3 + // provides a cleaner compatible API across "us-east-1" and + // China region. + location = "cn-north-1" + } + // Gather location only if bucketName is present. - location := "us-east-1" // Default all other requests to "us-east-1". if metadata.bucketName != "" { location, err = c.getBucketLocation(metadata.bucketName) if err != nil { @@ -648,6 +667,5 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que if err != nil { return nil, err } - return u, nil } diff --git a/vendor/src/github.com/minio/minio-go/bucket-cache.go b/vendor/src/github.com/minio/minio-go/bucket-cache.go index 50679a380..14ac17044 100644 --- a/vendor/src/github.com/minio/minio-go/bucket-cache.go +++ b/vendor/src/github.com/minio/minio-go/bucket-cache.go @@ -72,6 +72,14 @@ func (c Client) getBucketLocation(bucketName string) (string, error) { return location, nil } + if isAmazonChinaEndpoint(c.endpointURL) { + // For china specifically we need to set everything to + // cn-north-1 for now, there is no easier way until AWS S3 + // provides a cleaner compatible API across "us-east-1" and + // China region. + return "cn-north-1", nil + } + // Initialize a new request. req, err := c.getBucketLocationRequest(bucketName) if err != nil { @@ -84,6 +92,16 @@ func (c Client) getBucketLocation(bucketName string) (string, error) { if err != nil { return "", err } + location, err := processBucketLocationResponse(resp, bucketName) + if err != nil { + return "", err + } + c.bucketLocCache.Set(bucketName, location) + return location, nil +} + +// processes the getBucketLocation http response from the server. +func processBucketLocationResponse(resp *http.Response, bucketName string) (bucketLocation string, err error) { if resp != nil { if resp.StatusCode != http.StatusOK { err = httpRespToErrorResponse(resp, bucketName, "") @@ -117,7 +135,6 @@ func (c Client) getBucketLocation(bucketName string) (string, error) { } // Save the location into cache. - c.bucketLocCache.Set(bucketName, location) // Return. return location, nil diff --git a/vendor/src/github.com/minio/minio-go/bucket-cache_test.go b/vendor/src/github.com/minio/minio-go/bucket-cache_test.go new file mode 100644 index 000000000..57b0e9e34 --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/bucket-cache_test.go @@ -0,0 +1,320 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2016, 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "encoding/hex" + "encoding/xml" + "io/ioutil" + "net/http" + "net/url" + "path" + "reflect" + "testing" +) + +// Test validates `newBucketLocationCache`. +func TestNewBucketLocationCache(t *testing.T) { + expectedBucketLocationcache := &bucketLocationCache{ + items: make(map[string]string), + } + actualBucketLocationCache := newBucketLocationCache() + + if !reflect.DeepEqual(actualBucketLocationCache, expectedBucketLocationcache) { + t.Errorf("Unexpected return value") + } +} + +// Tests validate bucketLocationCache operations. +func TestBucketLocationCacheOps(t *testing.T) { + testBucketLocationCache := newBucketLocationCache() + expectedBucketName := "minio-bucket" + expectedLocation := "us-east-1" + testBucketLocationCache.Set(expectedBucketName, expectedLocation) + actualLocation, ok := testBucketLocationCache.Get(expectedBucketName) + if !ok { + t.Errorf("Bucket location cache not set") + } + if expectedLocation != actualLocation { + t.Errorf("Bucket location cache not set to expected value") + } + testBucketLocationCache.Delete(expectedBucketName) + _, ok = testBucketLocationCache.Get(expectedBucketName) + if ok { + t.Errorf("Bucket location cache not deleted as expected") + } +} + +// Tests validate http request generation for 'getBucketLocation'. +func TestGetBucketLocationRequest(t *testing.T) { + // Generates expected http request for getBucketLocation. + // Used for asserting with the actual request generated. + createExpectedRequest := func(c *Client, bucketName string, req *http.Request) (*http.Request, error) { + // Set location query. + urlValues := make(url.Values) + urlValues.Set("location", "") + + // Set get bucket location always as path style. + targetURL := c.endpointURL + targetURL.Path = path.Join(bucketName, "") + "/" + targetURL.RawQuery = urlValues.Encode() + + // Get a new HTTP request for the method. + req, err := http.NewRequest("GET", targetURL.String(), nil) + if err != nil { + return nil, err + } + + // Set UserAgent for the request. + c.setUserAgent(req) + + // Set sha256 sum for signature calculation only with signature version '4'. + if c.signature.isV4() { + req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{}))) + } + + // Sign the request. + if c.signature.isV4() { + req = signV4(*req, c.accessKeyID, c.secretAccessKey, "us-east-1") + } else if c.signature.isV2() { + req = signV2(*req, c.accessKeyID, c.secretAccessKey) + } + return req, nil + + } + // Info for 'Client' creation. + // Will be used as arguments for 'NewClient'. + type infoForClient struct { + endPoint string + accessKey string + secretKey string + enableInsecure bool + } + // dataset for 'NewClient' call. + info := []infoForClient{ + // endpoint localhost. + // both access-key and secret-key are empty. + {"localhost:9000", "", "", false}, + // both access-key are secret-key exists. + {"localhost:9000", "my-access-key", "my-secret-key", false}, + // one of acess-key and secret-key are empty. + {"localhost:9000", "", "my-secret-key", false}, + + // endpoint amazon s3. + {"s3.amazonaws.com", "", "", false}, + {"s3.amazonaws.com", "my-access-key", "my-secret-key", false}, + {"s3.amazonaws.com", "my-acess-key", "", false}, + + // endpoint google cloud storage. + {"storage.googleapis.com", "", "", false}, + {"storage.googleapis.com", "my-access-key", "my-secret-key", false}, + {"storage.googleapis.com", "", "my-secret-key", false}, + + // endpoint custom domain running Minio server. + {"play.minio.io", "", "", false}, + {"play.minio.io", "my-access-key", "my-secret-key", false}, + {"play.minio.io", "my-acess-key", "", false}, + } + testCases := []struct { + bucketName string + // data for new client creation. + info infoForClient + // error in the output. + err error + // flag indicating whether tests should pass. + shouldPass bool + }{ + // Client is constructed using the info struct. + // case with empty location. + {"my-bucket", info[0], nil, true}, + // case with location set to standard 'us-east-1'. + {"my-bucket", info[0], nil, true}, + // case with location set to a value different from 'us-east-1'. + {"my-bucket", info[0], nil, true}, + + {"my-bucket", info[1], nil, true}, + {"my-bucket", info[1], nil, true}, + {"my-bucket", info[1], nil, true}, + + {"my-bucket", info[2], nil, true}, + {"my-bucket", info[2], nil, true}, + {"my-bucket", info[2], nil, true}, + + {"my-bucket", info[3], nil, true}, + {"my-bucket", info[3], nil, true}, + {"my-bucket", info[3], nil, true}, + + {"my-bucket", info[4], nil, true}, + {"my-bucket", info[4], nil, true}, + {"my-bucket", info[4], nil, true}, + + {"my-bucket", info[5], nil, true}, + {"my-bucket", info[5], nil, true}, + {"my-bucket", info[5], nil, true}, + + {"my-bucket", info[6], nil, true}, + {"my-bucket", info[6], nil, true}, + {"my-bucket", info[6], nil, true}, + + {"my-bucket", info[7], nil, true}, + {"my-bucket", info[7], nil, true}, + {"my-bucket", info[7], nil, true}, + + {"my-bucket", info[8], nil, true}, + {"my-bucket", info[8], nil, true}, + {"my-bucket", info[8], nil, true}, + + {"my-bucket", info[9], nil, true}, + {"my-bucket", info[9], nil, true}, + {"my-bucket", info[9], nil, true}, + + {"my-bucket", info[10], nil, true}, + {"my-bucket", info[10], nil, true}, + {"my-bucket", info[10], nil, true}, + + {"my-bucket", info[11], nil, true}, + {"my-bucket", info[11], nil, true}, + {"my-bucket", info[11], nil, true}, + } + for i, testCase := range testCases { + // cannot create a newclient with empty endPoint value. + // validates and creates a new client only if the endPoint value is not empty. + client := &Client{} + var err error + if testCase.info.endPoint != "" { + + client, err = New(testCase.info.endPoint, testCase.info.accessKey, testCase.info.secretKey, testCase.info.enableInsecure) + if err != nil { + t.Fatalf("Test %d: Failed to create new Client: %s", i+1, err.Error()) + } + } + + actualReq, err := client.getBucketLocationRequest(testCase.bucketName) + if err != nil && testCase.shouldPass { + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) + } + if err == nil && !testCase.shouldPass { + t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if err != nil && !testCase.shouldPass { + if err.Error() != testCase.err.Error() { + t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) + } + } + + // Test passes as expected, but the output values are verified for correctness here. + if err == nil && testCase.shouldPass { + expectedReq := &http.Request{} + expectedReq, err = createExpectedRequest(client, testCase.bucketName, expectedReq) + if err != nil { + t.Fatalf("Test %d: Expected request Creation failed", i+1) + } + if expectedReq.Method != actualReq.Method { + t.Errorf("Test %d: The expected Request method doesn't match with the actual one", i+1) + } + if expectedReq.URL.String() != actualReq.URL.String() { + t.Errorf("Test %d: Expected the request URL to be '%s', but instead found '%s'", i+1, expectedReq.URL.String(), actualReq.URL.String()) + } + if expectedReq.ContentLength != actualReq.ContentLength { + t.Errorf("Test %d: Expected the request body Content-Length to be '%d', but found '%d' instead", i+1, expectedReq.ContentLength, actualReq.ContentLength) + } + + if expectedReq.Header.Get("X-Amz-Content-Sha256") != actualReq.Header.Get("X-Amz-Content-Sha256") { + t.Errorf("Test %d: 'X-Amz-Content-Sha256' header of the expected request doesn't match with that of the actual request", i+1) + } + if expectedReq.Header.Get("User-Agent") != actualReq.Header.Get("User-Agent") { + t.Errorf("Test %d: Expected 'User-Agent' header to be \"%s\",but found \"%s\" instead", i+1, expectedReq.Header.Get("User-Agent"), actualReq.Header.Get("User-Agent")) + } + } + } +} + +// generates http response with bucket location set in the body. +func generateLocationResponse(resp *http.Response, bodyContent []byte) (*http.Response, error) { + resp.StatusCode = http.StatusOK + resp.Body = ioutil.NopCloser(bytes.NewBuffer(bodyContent)) + return resp, nil +} + +// Tests the processing of GetPolicy response from server. +func TestProcessBucketLocationResponse(t *testing.T) { + // LocationResponse - format for location response. + type LocationResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"` + Location string `xml:",chardata"` + } + + APIErrors := []APIError{ + { + Code: "AccessDenied", + Description: "Access Denied", + HTTPStatusCode: http.StatusUnauthorized, + }, + } + testCases := []struct { + bucketName string + inputLocation string + isAPIError bool + apiErr APIError + // expected results. + expectedResult string + err error + // flag indicating whether tests should pass. + shouldPass bool + }{ + {"my-bucket", "", true, APIErrors[0], "us-east-1", nil, true}, + {"my-bucket", "", false, APIError{}, "us-east-1", nil, true}, + {"my-bucket", "EU", false, APIError{}, "eu-west-1", nil, true}, + {"my-bucket", "eu-central-1", false, APIError{}, "eu-central-1", nil, true}, + {"my-bucket", "us-east-1", false, APIError{}, "us-east-1", nil, true}, + } + + for i, testCase := range testCases { + inputResponse := &http.Response{} + var err error + if testCase.isAPIError { + inputResponse = generateErrorResponse(inputResponse, testCase.apiErr, testCase.bucketName) + } else { + inputResponse, err = generateLocationResponse(inputResponse, encodeResponse(LocationResponse{ + Location: testCase.inputLocation, + })) + if err != nil { + t.Fatalf("Test %d: Creation of valid response failed", i+1) + } + } + actualResult, err := processBucketLocationResponse(inputResponse, "my-bucket") + if err != nil && testCase.shouldPass { + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) + } + if err == nil && !testCase.shouldPass { + t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if err != nil && !testCase.shouldPass { + if err.Error() != testCase.err.Error() { + t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) + } + } + if err == nil && testCase.shouldPass { + if !reflect.DeepEqual(testCase.expectedResult, actualResult) { + t.Errorf("Test %d: The expected BucketPolicy doesnt match the actual BucketPolicy", i+1) + } + } + } +} diff --git a/vendor/src/github.com/minio/minio-go/examples/s3/listobjects-N.go b/vendor/src/github.com/minio/minio-go/examples/s3/listobjects-N.go new file mode 100644 index 000000000..81136f92e --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/examples/s3/listobjects-N.go @@ -0,0 +1,76 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname + // are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set insecure=true to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", false) + if err != nil { + fmt.Println(err) + return + } + + // List 'N' number of objects from a bucket-name with a matching prefix. + listObjectsN := func(bucket, prefix string, recursive bool, N int) (objsInfo []minio.ObjectInfo, err error) { + // Create a done channel to control 'ListObjects' go routine. + doneCh := make(chan struct{}, 1) + + // Free the channel upon return. + defer close(doneCh) + + i := 1 + for object := range s3Client.ListObjects(bucket, prefix, recursive, doneCh) { + if object.Err != nil { + return nil, object.Err + } + i++ + // Verify if we have printed N objects. + if i == N { + // Indicate ListObjects go-routine to exit and stop + // feeding the objectInfo channel. + doneCh <- struct{}{} + } + objsInfo = append(objsInfo, object) + } + return objsInfo, nil + } + + // List recursively first 100 entries for prefix 'my-prefixname'. + recursive := true + objsInfo, err := listObjectsN("my-bucketname", "my-prefixname", recursive, 100) + if err != nil { + fmt.Println(err) + } + + // Print all the entries. + fmt.Println(objsInfo) +} diff --git a/vendor/src/github.com/minio/minio-go/retry.go b/vendor/src/github.com/minio/minio-go/retry.go index d9fbe12f5..41b70e474 100644 --- a/vendor/src/github.com/minio/minio-go/retry.go +++ b/vendor/src/github.com/minio/minio-go/retry.go @@ -35,7 +35,7 @@ const NoJitter = 0.0 // newRetryTimer creates a timer with exponentially increasing delays // until the maximum retry attempts are reached. -func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64) <-chan int { +func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int { attemptCh := make(chan int) // computes the exponential backoff duration according to @@ -63,7 +63,13 @@ func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duratio go func() { defer close(attemptCh) for i := 0; i < maxRetry; i++ { - attemptCh <- i + 1 // Attempts start from 1. + select { + // Attempts start from 1. + case attemptCh <- i + 1: + case <-doneCh: + // Stop the routine. + return + } time.Sleep(exponentialBackoffWait(i)) } }() @@ -73,13 +79,24 @@ func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duratio // isNetErrorRetryable - is network error retryable. func isNetErrorRetryable(err error) bool { switch err.(type) { - case *net.DNSError, *net.OpError, net.UnknownNetworkError: - return true - case *url.Error: - // For a URL error, where it replies back "connection closed" - // retry again. - if strings.Contains(err.Error(), "Connection closed by foreign host") { + case net.Error: + switch err.(type) { + case *net.DNSError, *net.OpError, net.UnknownNetworkError: return true + case *url.Error: + // For a URL error, where it replies back "connection closed" + // retry again. + if strings.Contains(err.Error(), "Connection closed by foreign host") { + return true + } + default: + if strings.Contains(err.Error(), "net/http: TLS handshake timeout") { + // If error is - tlsHandshakeTimeoutError, retry. + return true + } else if strings.Contains(err.Error(), "i/o timeout") { + // If error is - tcp timeoutError, retry. + return true + } } } return false diff --git a/vendor/src/github.com/minio/minio-go/test-utils_test.go b/vendor/src/github.com/minio/minio-go/test-utils_test.go new file mode 100644 index 000000000..179c28a23 --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/test-utils_test.go @@ -0,0 +1,64 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "encoding/xml" + "io/ioutil" + "net/http" +) + +// Contains common used utilities for tests. + +// APIError Used for mocking error response from server. +type APIError struct { + Code string + Description string + HTTPStatusCode int +} + +// Mocks XML error response from the server. +func generateErrorResponse(resp *http.Response, APIErr APIError, bucketName string) *http.Response { + // generate error response. + errorResponse := getAPIErrorResponse(APIErr, bucketName) + encodedErrorResponse := encodeResponse(errorResponse) + // write Header. + resp.StatusCode = APIErr.HTTPStatusCode + resp.Body = ioutil.NopCloser(bytes.NewBuffer(encodedErrorResponse)) + + return resp +} + +// getErrorResponse gets in standard error and resource value and +// provides a encodable populated response values. +func getAPIErrorResponse(err APIError, bucketName string) ErrorResponse { + var errResp = ErrorResponse{} + errResp.Code = err.Code + errResp.Message = err.Description + errResp.BucketName = bucketName + return errResp +} + +// Encodes the response headers into XML format. +func encodeResponse(response interface{}) []byte { + var bytesBuffer bytes.Buffer + bytesBuffer.WriteString(xml.Header) + encode := xml.NewEncoder(&bytesBuffer) + encode.Encode(response) + return bytesBuffer.Bytes() +}