2
2
mirror of https://github.com/octoleo/restic.git synced 2024-11-26 14:56:29 +00:00

Update minio-go

This commit is contained in:
Alexander Neumann 2016-05-08 11:24:24 +02:00
parent 7dc7f0d295
commit 1b50d55d0c
13 changed files with 912 additions and 166 deletions

2
vendor/manifest vendored
View File

@ -28,7 +28,7 @@
{ {
"importpath": "github.com/minio/minio-go", "importpath": "github.com/minio/minio-go",
"repository": "https://github.com/minio/minio-go", "repository": "https://github.com/minio/minio-go",
"revision": "17b4ebd52505bde655e3b14df732e31850641bb7", "revision": "867b27701ad16db4a9f4dad40d28187ca8433ec9",
"branch": "master" "branch": "master"
}, },
{ {

View File

@ -60,10 +60,10 @@ s3Client can be used to perform operations on S3 storage. APIs are described bel
### Bucket operations ### Bucket operations
--------------------------------------- ---------------------------------------
<a name="MakeBucket"> <a name="MakeBucket">
#### MakeBucket(bucketName, location) #### MakeBucket(bucketName string, location string) error
Create a new bucket. Create a new bucket.
__Arguments__ __Parameters__
* `bucketName` _string_ - Name of the bucket. * `bucketName` _string_ - Name of the bucket.
* `location` _string_ - region valid values are _us-west-1_, _us-west-2_, _eu-west-1_, _eu-central-1_, _ap-southeast-1_, _ap-northeast-1_, _ap-southeast-2_, _sa-east-1_ * `location` _string_ - region valid values are _us-west-1_, _us-west-2_, _eu-west-1_, _eu-central-1_, _ap-southeast-1_, _ap-northeast-1_, _ap-southeast-2_, _sa-east-1_
@ -78,10 +78,10 @@ fmt.Println("Successfully created mybucket.")
``` ```
--------------------------------------- ---------------------------------------
<a name="ListBuckets"> <a name="ListBuckets">
#### ListBuckets() #### ListBuckets() ([]BucketInfo, error)
List all buckets. Lists all buckets.
`bucketList` emits bucket with the format: `bucketList` lists bucket in the format:
* `bucket.Name` _string_: bucket name * `bucket.Name` _string_: bucket name
* `bucket.CreationDate` time.Time : date when bucket was created * `bucket.CreationDate` time.Time : date when bucket was created
@ -98,10 +98,10 @@ for _, bucket := range buckets {
``` ```
--------------------------------------- ---------------------------------------
<a name="BucketExists"> <a name="BucketExists">
#### BucketExists(bucketName) #### BucketExists(bucketName string) error
Check if bucket exists. Check if bucket exists.
__Arguments__ __Parameters__
* `bucketName` _string_ : name of the bucket * `bucketName` _string_ : name of the bucket
__Example__ __Example__
@ -114,10 +114,10 @@ if err != nil {
``` ```
--------------------------------------- ---------------------------------------
<a name="RemoveBucket"> <a name="RemoveBucket">
#### RemoveBucket(bucketName) #### RemoveBucket(bucketName string) error
Remove a bucket. Remove a bucket.
__Arguments__ __Parameters__
* `bucketName` _string_ : name of the bucket * `bucketName` _string_ : name of the bucket
__Example__ __Example__
@ -130,16 +130,16 @@ if err != nil {
``` ```
--------------------------------------- ---------------------------------------
<a name="GetBucketPolicy"> <a name="GetBucketPolicy">
#### GetBucketPolicy(bucketName, objectPrefix) #### GetBucketPolicy(bucketName string, objectPrefix string) error
Get access permissions on a bucket or a prefix. Get access permissions on a bucket or a prefix.
__Arguments__ __Parameters__
* `bucketName` _string_ : name of the bucket * `bucketName` _string_ : name of the bucket
* `objectPrefix` _string_ : name of the object prefix * `objectPrefix` _string_ : name of the object prefix
__Example__ __Example__
```go ```go
bucketPolicy, err := s3Client.GetBucketPolicy("mybucket") bucketPolicy, err := s3Client.GetBucketPolicy("mybucket", "")
if err != nil { if err != nil {
fmt.Println(err) fmt.Println(err)
return return
@ -148,13 +148,13 @@ fmt.Println("Access permissions for mybucket is", bucketPolicy)
``` ```
--------------------------------------- ---------------------------------------
<a name="SetBucketPolicy"> <a name="SetBucketPolicy">
#### SetBucketPolicy(bucketname, objectPrefix, policy) #### SetBucketPolicy(bucketname string, objectPrefix string, policy BucketPolicy) error
Set access permissions on bucket or an object prefix. Set access permissions on bucket or an object prefix.
__Arguments__ __Parameters__
* `bucketName` _string_: name of the bucket * `bucketName` _string_: name of the bucket
* `objectPrefix` _string_ : name of the object prefix * `objectPrefix` _string_ : name of the object prefix
* `policy` _BucketPolicy_: policy can be _non_, _readonly_, _readwrite_, _writeonly_ * `policy` _BucketPolicy_: policy can be _BucketPolicyNone_, _BucketPolicyReadOnly_, _BucketPolicyReadWrite_, _BucketPolicyWriteOnly_
__Example__ __Example__
```go ```go
@ -166,10 +166,10 @@ if err != nil {
``` ```
--------------------------------------- ---------------------------------------
<a name="RemoveBucketPolicy"> <a name="RemoveBucketPolicy">
#### RemoveBucketPolicy(bucketname, objectPrefix) #### RemoveBucketPolicy(bucketname string, objectPrefix string) error
Remove existing permissions on bucket or an object prefix. Remove existing permissions on bucket or an object prefix.
__Arguments__ __Parameters__
* `bucketName` _string_: name of the bucket * `bucketName` _string_: name of the bucket
* `objectPrefix` _string_ : name of the object prefix * `objectPrefix` _string_ : name of the object prefix
@ -184,10 +184,10 @@ if err != nil {
--------------------------------------- ---------------------------------------
<a name="ListObjects"> <a name="ListObjects">
#### ListObjects(bucketName, prefix, recursive, doneCh) #### ListObjects(bucketName string, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectInfo
List objects in a bucket. List objects in a bucket.
__Arguments__ __Parameters__
* `bucketName` _string_: name of the bucket * `bucketName` _string_: name of the bucket
* `objectPrefix` _string_: the prefix of the objects that should be listed * `objectPrefix` _string_: the prefix of the objects that should be listed
* `recursive` _bool_: `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/' * `recursive` _bool_: `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'
@ -222,10 +222,10 @@ for object := range objectCh {
--------------------------------------- ---------------------------------------
<a name="ListIncompleteUploads"> <a name="ListIncompleteUploads">
#### ListIncompleteUploads(bucketName, prefix, recursive) #### ListIncompleteUploads(bucketName string, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectMultipartInfo
List partially uploaded objects in a bucket. List partially uploaded objects in a bucket.
__Arguments__ __Parameters__
* `bucketname` _string_: name of the bucket * `bucketname` _string_: name of the bucket
* `prefix` _string_: prefix of the object names that are partially uploaded * `prefix` _string_: prefix of the object names that are partially uploaded
* `recursive` bool: directory style listing when false, recursive listing when true * `recursive` bool: directory style listing when false, recursive listing when true
@ -259,15 +259,15 @@ for multiPartObject := range multiPartObjectCh {
--------------------------------------- ---------------------------------------
### Object operations ### Object operations
<a name="GetObject"> <a name="GetObject">
#### GetObject(bucketName, objectName) #### GetObject(bucketName string, objectName string) *Object
Download an object. Download an object.
__Arguments__ __Parameters__
* `bucketName` _string_: name of the bucket * `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object * `objectName` _string_: name of the object
__Return Value__ __Return Value__
* `object` _*minio.Object_ : _minio.Object_ represents object reader. * `object` _*Object_ : _Object_ represents object reader.
__Example__ __Example__
```go ```go
@ -285,10 +285,10 @@ if _, err := io.Copy(localFile, object); err != nil {
--------------------------------------- ---------------------------------------
--------------------------------------- ---------------------------------------
<a name="FGetObject"> <a name="FGetObject">
#### FGetObject(bucketName, objectName, filePath) #### FGetObject(bucketName string, objectName string, filePath string) error
Callback is called with `error` in case of error or `null` in case of success Callback is called with `error` in case of error or `null` in case of success
__Arguments__ __Parameters__
* `bucketName` _string_: name of the bucket * `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object * `objectName` _string_: name of the object
* `filePath` _string_: path to which the object data will be written to * `filePath` _string_: path to which the object data will be written to
@ -303,11 +303,10 @@ if err != nil {
``` ```
--------------------------------------- ---------------------------------------
<a name="PutObject"> <a name="PutObject">
#### PutObject(bucketName, objectName, reader, contentType) #### PutObject(bucketName string, objectName string, reader io.Reader, contentType string) (n int, err error)
Upload an object. Upload contents from `io.Reader` to objectName.
Uploading a stream __Parameters__
__Arguments__
* `bucketName` _string_: name of the bucket * `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object * `objectName` _string_: name of the object
* `reader` _io.Reader_: Any golang object implementing io.Reader * `reader` _io.Reader_: Any golang object implementing io.Reader
@ -331,10 +330,10 @@ if err != nil {
--------------------------------------- ---------------------------------------
<a name="CopyObject"> <a name="CopyObject">
#### CopyObject(bucketName, objectName, objectSource, conditions) #### CopyObject(bucketName string, objectName string, objectSource string, conditions CopyConditions) error
Copy a source object into a new object with the provided name in the provided bucket. Copy a source object into a new object with the provided name in the provided bucket.
__Arguments__ __Parameters__
* `bucketName` _string_: name of the bucket * `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object * `objectName` _string_: name of the object
* `objectSource` _string_: name of the object source. * `objectSource` _string_: name of the object source.
@ -367,10 +366,10 @@ if err != nil {
--------------------------------------- ---------------------------------------
<a name="FPutObject"> <a name="FPutObject">
#### FPutObject(bucketName, objectName, filePath, contentType) #### FPutObject(bucketName string, objectName string, filePath string, contentType string) error
Uploads the object using contents from a file Uploads the object using contents from a file
__Arguments__ __Parameters__
* `bucketName` _string_: name of the bucket * `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object * `objectName` _string_: name of the object
* `filePath` _string_: file path of the file to be uploaded * `filePath` _string_: file path of the file to be uploaded
@ -386,10 +385,10 @@ if err != nil {
``` ```
--------------------------------------- ---------------------------------------
<a name="StatObject"> <a name="StatObject">
#### StatObject(bucketName, objectName) #### StatObject(bucketName string, objectName string) (ObjectInfo, error)
Get metadata of an object. Get metadata of an object.
__Arguments__ __Parameters__
* `bucketName` _string_: name of the bucket * `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object * `objectName` _string_: name of the object
@ -411,10 +410,10 @@ fmt.Println(objInfo)
``` ```
--------------------------------------- ---------------------------------------
<a name="RemoveObject"> <a name="RemoveObject">
#### RemoveObject(bucketName, objectName) #### RemoveObject(bucketName string, objectName string) error
Remove an object. Remove an object.
__Arguments__ __Parameters__
* `bucketName` _string_: name of the bucket * `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object * `objectName` _string_: name of the object
@ -428,10 +427,10 @@ if err != nil {
``` ```
--------------------------------------- ---------------------------------------
<a name="RemoveIncompleteUpload"> <a name="RemoveIncompleteUpload">
#### RemoveIncompleteUpload(bucketName, objectName) #### RemoveIncompleteUpload(bucketName string, objectName string) error
Remove an partially uploaded object. Remove an partially uploaded object.
__Arguments__ __Parameters__
* `bucketName` _string_: name of the bucket * `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object * `objectName` _string_: name of the object
@ -447,14 +446,14 @@ if err != nil {
### Presigned operations ### Presigned operations
--------------------------------------- ---------------------------------------
<a name="PresignedGetObject"> <a name="PresignedGetObject">
#### PresignedGetObject(bucketName, objectName, expiry) #### PresignedGetObject(bucketName, objectName string, expiry time.Duration, reqParams url.Values) error
Generate a presigned URL for GET. Generate a presigned URL for GET.
__Arguments__ __Parameters__
* `bucketName` _string_: name of the bucket. * `bucketName` _string_: name of the bucket.
* `objectName` _string_: name of the object. * `objectName` _string_: name of the object.
* `expiry` _time.Duration_: expiry in seconds. * `expiry` _time.Duration_: expiry in seconds.
`reqParams` _url.Values_ : additional response header overrides supports _response-expires_, _response-content-type_, _response-cache-control_, _response-content-disposition_ * `reqParams` _url.Values_ : additional response header overrides supports _response-expires_, _response-content-type_, _response-cache-control_, _response-content-disposition_
__Example__ __Example__
```go ```go
@ -472,13 +471,13 @@ if err != nil {
--------------------------------------- ---------------------------------------
<a name="PresignedPutObject"> <a name="PresignedPutObject">
#### PresignedPutObject(bucketName, objectName, expiry) #### PresignedPutObject(bucketName string, objectName string, expiry time.Duration) (string, error)
Generate a presigned URL for PUT. Generate a presigned URL for PUT.
<blockquote> <blockquote>
NOTE: you can upload to S3 only with specified object name. NOTE: you can upload to S3 only with specified object name.
</blockquote> </blockquote>
__Arguments__ __Parameters__
* `bucketName` _string_: name of the bucket * `bucketName` _string_: name of the bucket
* `objectName` _string_: name of the object * `objectName` _string_: name of the object
* `expiry` _time.Duration_: expiry in seconds * `expiry` _time.Duration_: expiry in seconds
@ -495,7 +494,7 @@ if err != nil {
--------------------------------------- ---------------------------------------
<a name="PresignedPostPolicy"> <a name="PresignedPostPolicy">
#### PresignedPostPolicy #### PresignedPostPolicy(policy PostPolicy) (map[string]string, error)
PresignedPostPolicy we can provide policies specifying conditions restricting PresignedPostPolicy we can provide policies specifying conditions restricting
what you want to allow in a POST request, such as bucket name where objects can be what you want to allow in a POST request, such as bucket name where objects can be
uploaded, key name prefixes that you want to allow for the object being created and more. uploaded, key name prefixes that you want to allow for the object being created and more.

View File

@ -0,0 +1,277 @@
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required bZy applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"bytes"
"encoding/xml"
"fmt"
"io/ioutil"
"net/http"
"reflect"
"strconv"
"testing"
)
// Tests validate the Error generator function for http response with error.
func TestHttpRespToErrorResponse(t *testing.T) {
// 'genAPIErrorResponse' generates ErrorResponse for given APIError.
// provides a encodable populated response values.
genAPIErrorResponse := func(err APIError, bucketName string) ErrorResponse {
var errResp = ErrorResponse{}
errResp.Code = err.Code
errResp.Message = err.Description
errResp.BucketName = bucketName
return errResp
}
// Encodes the response headers into XML format.
encodeErr := func(response interface{}) []byte {
var bytesBuffer bytes.Buffer
bytesBuffer.WriteString(xml.Header)
encode := xml.NewEncoder(&bytesBuffer)
encode.Encode(response)
return bytesBuffer.Bytes()
}
// `createAPIErrorResponse` Mocks XML error response from the server.
createAPIErrorResponse := func(APIErr APIError, bucketName string) *http.Response {
// generate error response.
// response body contains the XML error message.
resp := &http.Response{}
errorResponse := genAPIErrorResponse(APIErr, bucketName)
encodedErrorResponse := encodeErr(errorResponse)
// write Header.
resp.StatusCode = APIErr.HTTPStatusCode
resp.Body = ioutil.NopCloser(bytes.NewBuffer(encodedErrorResponse))
return resp
}
// 'genErrResponse' contructs error response based http Status Code
genErrResponse := func(resp *http.Response, code, message, bucketName, objectName string) ErrorResponse {
errResp := ErrorResponse{
Code: code,
Message: message,
BucketName: bucketName,
Key: objectName,
RequestID: resp.Header.Get("x-amz-request-id"),
HostID: resp.Header.Get("x-amz-id-2"),
Region: resp.Header.Get("x-amz-bucket-region"),
}
return errResp
}
// Generate invalid argument error.
genInvalidError := func(message string) error {
errResp := ErrorResponse{
Code: "InvalidArgument",
Message: message,
RequestID: "minio",
}
return errResp
}
// Set common http response headers.
setCommonHeaders := func(resp *http.Response) *http.Response {
// set headers.
resp.Header = make(http.Header)
resp.Header.Set("x-amz-request-id", "xyz")
resp.Header.Set("x-amz-id-2", "abc")
resp.Header.Set("x-amz-bucket-region", "us-east-1")
return resp
}
// Generate http response with empty body.
// Set the StatusCode to the arugment supplied.
// Sets common headers.
genEmptyBodyResponse := func(statusCode int) *http.Response {
resp := &http.Response{}
// set empty response body.
resp.Body = ioutil.NopCloser(bytes.NewBuffer([]byte("")))
// set headers.
setCommonHeaders(resp)
// set status code.
resp.StatusCode = statusCode
return resp
}
// Decode XML error message from the http response body.
decodeXMLError := func(resp *http.Response, t *testing.T) error {
var errResp ErrorResponse
err := xmlDecoder(resp.Body, &errResp)
if err != nil {
t.Fatal("XML decoding of response body failed")
}
return errResp
}
// List of APIErrors used to generate/mock server side XML error response.
APIErrors := []APIError{
{
Code: "NoSuchBucketPolicy",
Description: "The specified bucket does not have a bucket policy.",
HTTPStatusCode: http.StatusNotFound,
},
}
// List of expected response.
// Used for asserting the actual response.
expectedErrResponse := []error{
genInvalidError("Response is empty. " + "Please report this issue at https://github.com/minio/minio-go/issues."),
decodeXMLError(createAPIErrorResponse(APIErrors[0], "minio-bucket"), t),
genErrResponse(setCommonHeaders(&http.Response{}), "NoSuchBucket", "The specified bucket does not exist.", "minio-bucket", ""),
genErrResponse(setCommonHeaders(&http.Response{}), "NoSuchKey", "The specified key does not exist.", "minio-bucket", "Asia/"),
genErrResponse(setCommonHeaders(&http.Response{}), "AccessDenied", "Access Denied.", "minio-bucket", ""),
genErrResponse(setCommonHeaders(&http.Response{}), "Conflict", "Bucket not empty.", "minio-bucket", ""),
genErrResponse(setCommonHeaders(&http.Response{}), "Bad Request", "Bad Request", "minio-bucket", ""),
}
// List of http response to be used as input.
inputResponses := []*http.Response{
nil,
createAPIErrorResponse(APIErrors[0], "minio-bucket"),
genEmptyBodyResponse(http.StatusNotFound),
genEmptyBodyResponse(http.StatusNotFound),
genEmptyBodyResponse(http.StatusForbidden),
genEmptyBodyResponse(http.StatusConflict),
genEmptyBodyResponse(http.StatusBadRequest),
}
testCases := []struct {
bucketName string
objectName string
inputHTTPResp *http.Response
// expected results.
expectedResult error
// flag indicating whether tests should pass.
}{
{"minio-bucket", "", inputResponses[0], expectedErrResponse[0]},
{"minio-bucket", "", inputResponses[1], expectedErrResponse[1]},
{"minio-bucket", "", inputResponses[2], expectedErrResponse[2]},
{"minio-bucket", "Asia/", inputResponses[3], expectedErrResponse[3]},
{"minio-bucket", "", inputResponses[4], expectedErrResponse[4]},
{"minio-bucket", "", inputResponses[5], expectedErrResponse[5]},
}
for i, testCase := range testCases {
actualResult := httpRespToErrorResponse(testCase.inputHTTPResp, testCase.bucketName, testCase.objectName)
if !reflect.DeepEqual(testCase.expectedResult, actualResult) {
t.Errorf("Test %d: Expected result to be '%+v', but instead got '%+v'", i+1, testCase.expectedResult, actualResult)
}
}
}
// Test validates 'ErrEntityTooLarge' error response.
func TestErrEntityTooLarge(t *testing.T) {
msg := fmt.Sprintf("Your proposed upload size %d exceeds the maximum allowed object size %d for single PUT operation.", 1000000, 99999)
expectedResult := ErrorResponse{
Code: "EntityTooLarge",
Message: msg,
BucketName: "minio-bucket",
Key: "Asia/",
}
actualResult := ErrEntityTooLarge(1000000, 99999, "minio-bucket", "Asia/")
if !reflect.DeepEqual(expectedResult, actualResult) {
t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult)
}
}
// Test validates 'ErrEntityTooSmall' error response.
func TestErrEntityTooSmall(t *testing.T) {
msg := fmt.Sprintf("Your proposed upload size %d is below the minimum allowed object size '0B' for single PUT operation.", -1)
expectedResult := ErrorResponse{
Code: "EntityTooLarge",
Message: msg,
BucketName: "minio-bucket",
Key: "Asia/",
}
actualResult := ErrEntityTooSmall(-1, "minio-bucket", "Asia/")
if !reflect.DeepEqual(expectedResult, actualResult) {
t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult)
}
}
// Test validates 'ErrUnexpectedEOF' error response.
func TestErrUnexpectedEOF(t *testing.T) {
msg := fmt.Sprintf("Data read %s is not equal to the size %s of the input Reader.",
strconv.FormatInt(100, 10), strconv.FormatInt(101, 10))
expectedResult := ErrorResponse{
Code: "UnexpectedEOF",
Message: msg,
BucketName: "minio-bucket",
Key: "Asia/",
}
actualResult := ErrUnexpectedEOF(100, 101, "minio-bucket", "Asia/")
if !reflect.DeepEqual(expectedResult, actualResult) {
t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult)
}
}
// Test validates 'ErrInvalidBucketName' error response.
func TestErrInvalidBucketName(t *testing.T) {
expectedResult := ErrorResponse{
Code: "InvalidBucketName",
Message: "Invalid Bucket name",
RequestID: "minio",
}
actualResult := ErrInvalidBucketName("Invalid Bucket name")
if !reflect.DeepEqual(expectedResult, actualResult) {
t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult)
}
}
// Test validates 'ErrInvalidObjectName' error response.
func TestErrInvalidObjectName(t *testing.T) {
expectedResult := ErrorResponse{
Code: "NoSuchKey",
Message: "Invalid Object Key",
RequestID: "minio",
}
actualResult := ErrInvalidObjectName("Invalid Object Key")
if !reflect.DeepEqual(expectedResult, actualResult) {
t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult)
}
}
// Test validates 'ErrInvalidParts' error response.
func TestErrInvalidParts(t *testing.T) {
msg := fmt.Sprintf("Unexpected number of parts found Want %d, Got %d", 10, 9)
expectedResult := ErrorResponse{
Code: "InvalidParts",
Message: msg,
RequestID: "minio",
}
actualResult := ErrInvalidParts(10, 9)
if !reflect.DeepEqual(expectedResult, actualResult) {
t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult)
}
}
// Test validates 'ErrInvalidArgument' response.
func TestErrInvalidArgument(t *testing.T) {
expectedResult := ErrorResponse{
Code: "InvalidArgument",
Message: "Invalid Argument",
RequestID: "minio",
}
actualResult := ErrInvalidArgument("Invalid Argument")
if !reflect.DeepEqual(expectedResult, actualResult) {
t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult)
}
}

View File

@ -61,7 +61,7 @@ func (c Client) getBucketPolicy(bucketName string, objectPrefix string) (BucketA
} }
// processes the GetPolicy http resposne from the server. // processes the GetPolicy http response from the server.
func processBucketPolicyResponse(bucketName string, resp *http.Response) (BucketAccessPolicy, error) { func processBucketPolicyResponse(bucketName string, resp *http.Response) (BucketAccessPolicy, error) {
if resp != nil { if resp != nil {
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {

View File

@ -19,65 +19,24 @@ package minio
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"encoding/xml"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"reflect" "reflect"
"testing" "testing"
) )
type APIError struct {
Code string
Description string
HTTPStatusCode int
}
// Mocks XML error response from the server.
func generateErrorResponse(resp *http.Response, APIErr APIError, bucketName string) *http.Response {
// generate error response.
errorResponse := getAPIErrorResponse(APIErr, bucketName)
encodedErrorResponse := encodeResponse(errorResponse)
// write Header.
resp.StatusCode = APIErr.HTTPStatusCode
resp.Body = ioutil.NopCloser(bytes.NewBuffer(encodedErrorResponse))
return resp
}
// getErrorResponse gets in standard error and resource value and
// provides a encodable populated response values.
func getAPIErrorResponse(err APIError, bucketName string) ErrorResponse {
var data = ErrorResponse{}
data.Code = err.Code
data.Message = err.Description
data.BucketName = bucketName
// TODO implement this in future
return data
}
// Encodes the response headers into XML format.
func encodeResponse(response interface{}) []byte {
var bytesBuffer bytes.Buffer
bytesBuffer.WriteString(xml.Header)
encode := xml.NewEncoder(&bytesBuffer)
encode.Encode(response)
return bytesBuffer.Bytes()
}
// Mocks valid http response containing bucket policy from server. // Mocks valid http response containing bucket policy from server.
func generatePolicyResponse(resp *http.Response, policy BucketAccessPolicy) (*http.Response, error) { func generatePolicyResponse(resp *http.Response, policy BucketAccessPolicy) (*http.Response, error) {
policyBytes, e := json.Marshal(policy) policyBytes, err := json.Marshal(policy)
if e != nil { if err != nil {
return nil, e return nil, err
} }
resp.StatusCode = http.StatusOK resp.StatusCode = http.StatusOK
resp.Body = ioutil.NopCloser(bytes.NewBuffer(policyBytes)) resp.Body = ioutil.NopCloser(bytes.NewBuffer(policyBytes))
return resp, nil return resp, nil
} }
// Tests the processing of GetPolicy resposne from server. // Tests the processing of GetPolicy response from server.
func TestProcessBucketPolicyResopnse(t *testing.T) { func TestProcessBucketPolicyResopnse(t *testing.T) {
bucketAccesPolicies := []BucketAccessPolicy{ bucketAccesPolicies := []BucketAccessPolicy{
{Version: "1.0"}, {Version: "1.0"},
@ -139,6 +98,5 @@ func TestProcessBucketPolicyResopnse(t *testing.T) {
t.Errorf("Test %d: The expected BucketPolicy doesnt match the actual BucketPolicy", i+1) t.Errorf("Test %d: The expected BucketPolicy doesnt match the actual BucketPolicy", i+1)
} }
} }
} }
} }

View File

@ -77,7 +77,7 @@ func (c Client) ListBuckets() ([]BucketInfo, error) {
// //
func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo {
// Allocate new list objects channel. // Allocate new list objects channel.
objectStatCh := make(chan ObjectInfo, 1000) objectStatCh := make(chan ObjectInfo)
// Default listing is delimited at "/" // Default listing is delimited at "/"
delimiter := "/" delimiter := "/"
if recursive { if recursive {
@ -254,7 +254,7 @@ func (c Client) ListIncompleteUploads(bucketName, objectPrefix string, recursive
// listIncompleteUploads lists all incomplete uploads. // listIncompleteUploads lists all incomplete uploads.
func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive, aggregateSize bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive, aggregateSize bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo {
// Allocate channel for multipart uploads. // Allocate channel for multipart uploads.
objectMultipartStatCh := make(chan ObjectMultipartInfo, 1000) objectMultipartStatCh := make(chan ObjectMultipartInfo)
// Delimiter is set to "/" by default. // Delimiter is set to "/" by default.
delimiter := "/" delimiter := "/"
if recursive { if recursive {

View File

@ -27,71 +27,71 @@ import (
"testing" "testing"
) )
// Generates expected http request for bucket creation. // Tests validate http request formulated for creation of bucket.
// Used for asserting with the actual request generated. func TestMakeBucketRequest(t *testing.T) {
func createExpectedRequest(c *Client, bucketName string, location string, req *http.Request) (*http.Request, error) { // Generates expected http request for bucket creation.
// Used for asserting with the actual request generated.
createExpectedRequest := func(c *Client, bucketName string, location string, req *http.Request) (*http.Request, error) {
targetURL := *c.endpointURL targetURL := *c.endpointURL
targetURL.Path = "/" + bucketName + "/" targetURL.Path = "/" + bucketName + "/"
// get a new HTTP request for the method. // get a new HTTP request for the method.
req, err := http.NewRequest("PUT", targetURL.String(), nil) req, err := http.NewRequest("PUT", targetURL.String(), nil)
if err != nil {
return nil, err
}
// set UserAgent for the request.
c.setUserAgent(req)
// set sha256 sum for signature calculation only with signature version '4'.
if c.signature.isV4() {
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{})))
}
// If location is not 'us-east-1' create bucket location config.
if location != "us-east-1" && location != "" {
createBucketConfig := createBucketConfiguration{}
createBucketConfig.Location = location
var createBucketConfigBytes []byte
createBucketConfigBytes, err = xml.Marshal(createBucketConfig)
if err != nil { if err != nil {
return nil, err return nil, err
} }
createBucketConfigBuffer := bytes.NewBuffer(createBucketConfigBytes)
req.Body = ioutil.NopCloser(createBucketConfigBuffer) // set UserAgent for the request.
req.ContentLength = int64(len(createBucketConfigBytes)) c.setUserAgent(req)
// Set content-md5.
req.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(sumMD5(createBucketConfigBytes))) // set sha256 sum for signature calculation only with signature version '4'.
if c.signature.isV4() { if c.signature.isV4() {
// Set sha256. req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{})))
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256(createBucketConfigBytes)))
} }
// If location is not 'us-east-1' create bucket location config.
if location != "us-east-1" && location != "" {
createBucketConfig := createBucketConfiguration{}
createBucketConfig.Location = location
var createBucketConfigBytes []byte
createBucketConfigBytes, err = xml.Marshal(createBucketConfig)
if err != nil {
return nil, err
}
createBucketConfigBuffer := bytes.NewBuffer(createBucketConfigBytes)
req.Body = ioutil.NopCloser(createBucketConfigBuffer)
req.ContentLength = int64(len(createBucketConfigBytes))
// Set content-md5.
req.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(sumMD5(createBucketConfigBytes)))
if c.signature.isV4() {
// Set sha256.
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256(createBucketConfigBytes)))
}
}
// Sign the request.
if c.signature.isV4() {
// Signature calculated for MakeBucket request should be for 'us-east-1',
// regardless of the bucket's location constraint.
req = signV4(*req, c.accessKeyID, c.secretAccessKey, "us-east-1")
} else if c.signature.isV2() {
req = signV2(*req, c.accessKeyID, c.secretAccessKey)
}
// Return signed request.
return req, nil
} }
// Sign the request. // Get Request body.
if c.signature.isV4() { getReqBody := func(reqBody io.ReadCloser) (string, error) {
// Signature calculated for MakeBucket request should be for 'us-east-1', contents, err := ioutil.ReadAll(reqBody)
// regardless of the bucket's location constraint. if err != nil {
req = signV4(*req, c.accessKeyID, c.secretAccessKey, "us-east-1") return "", err
} else if c.signature.isV2() { }
req = signV2(*req, c.accessKeyID, c.secretAccessKey) return string(contents), nil
} }
// Return signed request.
return req, nil
}
// Get Request body.
func getReqBody(reqBody io.ReadCloser) (string, error) {
contents, err := ioutil.ReadAll(reqBody)
if err != nil {
return "", err
}
return string(contents), nil
}
// Tests validate http request formulated for creation of bucket.
func TestMakeBucketRequest(t *testing.T) {
// Info for 'Client' creation. // Info for 'Client' creation.
// Will be used as arguments for 'NewClient'. // Will be used as arguments for 'NewClient'.
type infoForClient struct { type infoForClient struct {

View File

@ -419,11 +419,20 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt
bodySeeker, isRetryable = metadata.contentBody.(io.Seeker) bodySeeker, isRetryable = metadata.contentBody.(io.Seeker)
} }
// Retry executes the following function body if request has an // Create a done channel to control 'ListObjects' go routine.
// error until maxRetries have been exhausted, retry attempts are doneCh := make(chan struct{}, 1)
// performed after waiting for a given period of time in a
// binomial fashion. // Indicate to our routine to exit cleanly upon return.
for range c.newRetryTimer(MaxRetry, time.Second, time.Second*30, MaxJitter) { defer close(doneCh)
// Blank indentifier is kept here on purpose since 'range' without
// blank identifiers is only supported since go1.4
// https://golang.org/doc/go1.4#forrange.
for _ = range c.newRetryTimer(MaxRetry, time.Second, time.Second*30, MaxJitter, doneCh) {
// Retry executes the following function body if request has an
// error until maxRetries have been exhausted, retry attempts are
// performed after waiting for a given period of time in a
// binomial fashion.
if isRetryable { if isRetryable {
// Seek back to beginning for each attempt. // Seek back to beginning for each attempt.
if _, err = bodySeeker.Seek(0, 0); err != nil { if _, err = bodySeeker.Seek(0, 0); err != nil {
@ -505,8 +514,18 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R
if method == "" { if method == "" {
method = "POST" method = "POST"
} }
// Default all requests to "us-east-1" or "cn-north-1" (china region)
location := "us-east-1"
if isAmazonChinaEndpoint(c.endpointURL) {
// For china specifically we need to set everything to
// cn-north-1 for now, there is no easier way until AWS S3
// provides a cleaner compatible API across "us-east-1" and
// China region.
location = "cn-north-1"
}
// Gather location only if bucketName is present. // Gather location only if bucketName is present.
location := "us-east-1" // Default all other requests to "us-east-1".
if metadata.bucketName != "" { if metadata.bucketName != "" {
location, err = c.getBucketLocation(metadata.bucketName) location, err = c.getBucketLocation(metadata.bucketName)
if err != nil { if err != nil {
@ -648,6 +667,5 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que
if err != nil { if err != nil {
return nil, err return nil, err
} }
return u, nil return u, nil
} }

View File

@ -72,6 +72,14 @@ func (c Client) getBucketLocation(bucketName string) (string, error) {
return location, nil return location, nil
} }
if isAmazonChinaEndpoint(c.endpointURL) {
// For china specifically we need to set everything to
// cn-north-1 for now, there is no easier way until AWS S3
// provides a cleaner compatible API across "us-east-1" and
// China region.
return "cn-north-1", nil
}
// Initialize a new request. // Initialize a new request.
req, err := c.getBucketLocationRequest(bucketName) req, err := c.getBucketLocationRequest(bucketName)
if err != nil { if err != nil {
@ -84,6 +92,16 @@ func (c Client) getBucketLocation(bucketName string) (string, error) {
if err != nil { if err != nil {
return "", err return "", err
} }
location, err := processBucketLocationResponse(resp, bucketName)
if err != nil {
return "", err
}
c.bucketLocCache.Set(bucketName, location)
return location, nil
}
// processes the getBucketLocation http response from the server.
func processBucketLocationResponse(resp *http.Response, bucketName string) (bucketLocation string, err error) {
if resp != nil { if resp != nil {
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
err = httpRespToErrorResponse(resp, bucketName, "") err = httpRespToErrorResponse(resp, bucketName, "")
@ -117,7 +135,6 @@ func (c Client) getBucketLocation(bucketName string) (string, error) {
} }
// Save the location into cache. // Save the location into cache.
c.bucketLocCache.Set(bucketName, location)
// Return. // Return.
return location, nil return location, nil

View File

@ -0,0 +1,320 @@
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2016, 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"bytes"
"encoding/hex"
"encoding/xml"
"io/ioutil"
"net/http"
"net/url"
"path"
"reflect"
"testing"
)
// Test validates `newBucketLocationCache`.
func TestNewBucketLocationCache(t *testing.T) {
expectedBucketLocationcache := &bucketLocationCache{
items: make(map[string]string),
}
actualBucketLocationCache := newBucketLocationCache()
if !reflect.DeepEqual(actualBucketLocationCache, expectedBucketLocationcache) {
t.Errorf("Unexpected return value")
}
}
// Tests validate bucketLocationCache operations.
func TestBucketLocationCacheOps(t *testing.T) {
testBucketLocationCache := newBucketLocationCache()
expectedBucketName := "minio-bucket"
expectedLocation := "us-east-1"
testBucketLocationCache.Set(expectedBucketName, expectedLocation)
actualLocation, ok := testBucketLocationCache.Get(expectedBucketName)
if !ok {
t.Errorf("Bucket location cache not set")
}
if expectedLocation != actualLocation {
t.Errorf("Bucket location cache not set to expected value")
}
testBucketLocationCache.Delete(expectedBucketName)
_, ok = testBucketLocationCache.Get(expectedBucketName)
if ok {
t.Errorf("Bucket location cache not deleted as expected")
}
}
// Tests validate http request generation for 'getBucketLocation'.
func TestGetBucketLocationRequest(t *testing.T) {
// Generates expected http request for getBucketLocation.
// Used for asserting with the actual request generated.
createExpectedRequest := func(c *Client, bucketName string, req *http.Request) (*http.Request, error) {
// Set location query.
urlValues := make(url.Values)
urlValues.Set("location", "")
// Set get bucket location always as path style.
targetURL := c.endpointURL
targetURL.Path = path.Join(bucketName, "") + "/"
targetURL.RawQuery = urlValues.Encode()
// Get a new HTTP request for the method.
req, err := http.NewRequest("GET", targetURL.String(), nil)
if err != nil {
return nil, err
}
// Set UserAgent for the request.
c.setUserAgent(req)
// Set sha256 sum for signature calculation only with signature version '4'.
if c.signature.isV4() {
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{})))
}
// Sign the request.
if c.signature.isV4() {
req = signV4(*req, c.accessKeyID, c.secretAccessKey, "us-east-1")
} else if c.signature.isV2() {
req = signV2(*req, c.accessKeyID, c.secretAccessKey)
}
return req, nil
}
// Info for 'Client' creation.
// Will be used as arguments for 'NewClient'.
type infoForClient struct {
endPoint string
accessKey string
secretKey string
enableInsecure bool
}
// dataset for 'NewClient' call.
info := []infoForClient{
// endpoint localhost.
// both access-key and secret-key are empty.
{"localhost:9000", "", "", false},
// both access-key are secret-key exists.
{"localhost:9000", "my-access-key", "my-secret-key", false},
// one of acess-key and secret-key are empty.
{"localhost:9000", "", "my-secret-key", false},
// endpoint amazon s3.
{"s3.amazonaws.com", "", "", false},
{"s3.amazonaws.com", "my-access-key", "my-secret-key", false},
{"s3.amazonaws.com", "my-acess-key", "", false},
// endpoint google cloud storage.
{"storage.googleapis.com", "", "", false},
{"storage.googleapis.com", "my-access-key", "my-secret-key", false},
{"storage.googleapis.com", "", "my-secret-key", false},
// endpoint custom domain running Minio server.
{"play.minio.io", "", "", false},
{"play.minio.io", "my-access-key", "my-secret-key", false},
{"play.minio.io", "my-acess-key", "", false},
}
testCases := []struct {
bucketName string
// data for new client creation.
info infoForClient
// error in the output.
err error
// flag indicating whether tests should pass.
shouldPass bool
}{
// Client is constructed using the info struct.
// case with empty location.
{"my-bucket", info[0], nil, true},
// case with location set to standard 'us-east-1'.
{"my-bucket", info[0], nil, true},
// case with location set to a value different from 'us-east-1'.
{"my-bucket", info[0], nil, true},
{"my-bucket", info[1], nil, true},
{"my-bucket", info[1], nil, true},
{"my-bucket", info[1], nil, true},
{"my-bucket", info[2], nil, true},
{"my-bucket", info[2], nil, true},
{"my-bucket", info[2], nil, true},
{"my-bucket", info[3], nil, true},
{"my-bucket", info[3], nil, true},
{"my-bucket", info[3], nil, true},
{"my-bucket", info[4], nil, true},
{"my-bucket", info[4], nil, true},
{"my-bucket", info[4], nil, true},
{"my-bucket", info[5], nil, true},
{"my-bucket", info[5], nil, true},
{"my-bucket", info[5], nil, true},
{"my-bucket", info[6], nil, true},
{"my-bucket", info[6], nil, true},
{"my-bucket", info[6], nil, true},
{"my-bucket", info[7], nil, true},
{"my-bucket", info[7], nil, true},
{"my-bucket", info[7], nil, true},
{"my-bucket", info[8], nil, true},
{"my-bucket", info[8], nil, true},
{"my-bucket", info[8], nil, true},
{"my-bucket", info[9], nil, true},
{"my-bucket", info[9], nil, true},
{"my-bucket", info[9], nil, true},
{"my-bucket", info[10], nil, true},
{"my-bucket", info[10], nil, true},
{"my-bucket", info[10], nil, true},
{"my-bucket", info[11], nil, true},
{"my-bucket", info[11], nil, true},
{"my-bucket", info[11], nil, true},
}
for i, testCase := range testCases {
// cannot create a newclient with empty endPoint value.
// validates and creates a new client only if the endPoint value is not empty.
client := &Client{}
var err error
if testCase.info.endPoint != "" {
client, err = New(testCase.info.endPoint, testCase.info.accessKey, testCase.info.secretKey, testCase.info.enableInsecure)
if err != nil {
t.Fatalf("Test %d: Failed to create new Client: %s", i+1, err.Error())
}
}
actualReq, err := client.getBucketLocationRequest(testCase.bucketName)
if err != nil && testCase.shouldPass {
t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
}
if err == nil && !testCase.shouldPass {
t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
}
// Failed as expected, but does it fail for the expected reason.
if err != nil && !testCase.shouldPass {
if err.Error() != testCase.err.Error() {
t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error())
}
}
// Test passes as expected, but the output values are verified for correctness here.
if err == nil && testCase.shouldPass {
expectedReq := &http.Request{}
expectedReq, err = createExpectedRequest(client, testCase.bucketName, expectedReq)
if err != nil {
t.Fatalf("Test %d: Expected request Creation failed", i+1)
}
if expectedReq.Method != actualReq.Method {
t.Errorf("Test %d: The expected Request method doesn't match with the actual one", i+1)
}
if expectedReq.URL.String() != actualReq.URL.String() {
t.Errorf("Test %d: Expected the request URL to be '%s', but instead found '%s'", i+1, expectedReq.URL.String(), actualReq.URL.String())
}
if expectedReq.ContentLength != actualReq.ContentLength {
t.Errorf("Test %d: Expected the request body Content-Length to be '%d', but found '%d' instead", i+1, expectedReq.ContentLength, actualReq.ContentLength)
}
if expectedReq.Header.Get("X-Amz-Content-Sha256") != actualReq.Header.Get("X-Amz-Content-Sha256") {
t.Errorf("Test %d: 'X-Amz-Content-Sha256' header of the expected request doesn't match with that of the actual request", i+1)
}
if expectedReq.Header.Get("User-Agent") != actualReq.Header.Get("User-Agent") {
t.Errorf("Test %d: Expected 'User-Agent' header to be \"%s\",but found \"%s\" instead", i+1, expectedReq.Header.Get("User-Agent"), actualReq.Header.Get("User-Agent"))
}
}
}
}
// generates http response with bucket location set in the body.
func generateLocationResponse(resp *http.Response, bodyContent []byte) (*http.Response, error) {
resp.StatusCode = http.StatusOK
resp.Body = ioutil.NopCloser(bytes.NewBuffer(bodyContent))
return resp, nil
}
// Tests the processing of GetPolicy response from server.
func TestProcessBucketLocationResponse(t *testing.T) {
// LocationResponse - format for location response.
type LocationResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"`
Location string `xml:",chardata"`
}
APIErrors := []APIError{
{
Code: "AccessDenied",
Description: "Access Denied",
HTTPStatusCode: http.StatusUnauthorized,
},
}
testCases := []struct {
bucketName string
inputLocation string
isAPIError bool
apiErr APIError
// expected results.
expectedResult string
err error
// flag indicating whether tests should pass.
shouldPass bool
}{
{"my-bucket", "", true, APIErrors[0], "us-east-1", nil, true},
{"my-bucket", "", false, APIError{}, "us-east-1", nil, true},
{"my-bucket", "EU", false, APIError{}, "eu-west-1", nil, true},
{"my-bucket", "eu-central-1", false, APIError{}, "eu-central-1", nil, true},
{"my-bucket", "us-east-1", false, APIError{}, "us-east-1", nil, true},
}
for i, testCase := range testCases {
inputResponse := &http.Response{}
var err error
if testCase.isAPIError {
inputResponse = generateErrorResponse(inputResponse, testCase.apiErr, testCase.bucketName)
} else {
inputResponse, err = generateLocationResponse(inputResponse, encodeResponse(LocationResponse{
Location: testCase.inputLocation,
}))
if err != nil {
t.Fatalf("Test %d: Creation of valid response failed", i+1)
}
}
actualResult, err := processBucketLocationResponse(inputResponse, "my-bucket")
if err != nil && testCase.shouldPass {
t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
}
if err == nil && !testCase.shouldPass {
t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
}
// Failed as expected, but does it fail for the expected reason.
if err != nil && !testCase.shouldPass {
if err.Error() != testCase.err.Error() {
t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error())
}
}
if err == nil && testCase.shouldPass {
if !reflect.DeepEqual(testCase.expectedResult, actualResult) {
t.Errorf("Test %d: The expected BucketPolicy doesnt match the actual BucketPolicy", i+1)
}
}
}
}

View File

@ -0,0 +1,76 @@
// +build ignore
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"fmt"
"github.com/minio/minio-go"
)
func main() {
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname
// are dummy values, please replace them with original values.
// Requests are always secure (HTTPS) by default. Set insecure=true to enable insecure (HTTP) access.
// This boolean value is the last argument for New().
// New returns an Amazon S3 compatible client object. API copatibality (v2 or v4) is automatically
// determined based on the Endpoint value.
s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", false)
if err != nil {
fmt.Println(err)
return
}
// List 'N' number of objects from a bucket-name with a matching prefix.
listObjectsN := func(bucket, prefix string, recursive bool, N int) (objsInfo []minio.ObjectInfo, err error) {
// Create a done channel to control 'ListObjects' go routine.
doneCh := make(chan struct{}, 1)
// Free the channel upon return.
defer close(doneCh)
i := 1
for object := range s3Client.ListObjects(bucket, prefix, recursive, doneCh) {
if object.Err != nil {
return nil, object.Err
}
i++
// Verify if we have printed N objects.
if i == N {
// Indicate ListObjects go-routine to exit and stop
// feeding the objectInfo channel.
doneCh <- struct{}{}
}
objsInfo = append(objsInfo, object)
}
return objsInfo, nil
}
// List recursively first 100 entries for prefix 'my-prefixname'.
recursive := true
objsInfo, err := listObjectsN("my-bucketname", "my-prefixname", recursive, 100)
if err != nil {
fmt.Println(err)
}
// Print all the entries.
fmt.Println(objsInfo)
}

View File

@ -35,7 +35,7 @@ const NoJitter = 0.0
// newRetryTimer creates a timer with exponentially increasing delays // newRetryTimer creates a timer with exponentially increasing delays
// until the maximum retry attempts are reached. // until the maximum retry attempts are reached.
func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64) <-chan int { func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int {
attemptCh := make(chan int) attemptCh := make(chan int)
// computes the exponential backoff duration according to // computes the exponential backoff duration according to
@ -63,7 +63,13 @@ func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duratio
go func() { go func() {
defer close(attemptCh) defer close(attemptCh)
for i := 0; i < maxRetry; i++ { for i := 0; i < maxRetry; i++ {
attemptCh <- i + 1 // Attempts start from 1. select {
// Attempts start from 1.
case attemptCh <- i + 1:
case <-doneCh:
// Stop the routine.
return
}
time.Sleep(exponentialBackoffWait(i)) time.Sleep(exponentialBackoffWait(i))
} }
}() }()
@ -73,13 +79,24 @@ func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duratio
// isNetErrorRetryable - is network error retryable. // isNetErrorRetryable - is network error retryable.
func isNetErrorRetryable(err error) bool { func isNetErrorRetryable(err error) bool {
switch err.(type) { switch err.(type) {
case *net.DNSError, *net.OpError, net.UnknownNetworkError: case net.Error:
return true switch err.(type) {
case *url.Error: case *net.DNSError, *net.OpError, net.UnknownNetworkError:
// For a URL error, where it replies back "connection closed"
// retry again.
if strings.Contains(err.Error(), "Connection closed by foreign host") {
return true return true
case *url.Error:
// For a URL error, where it replies back "connection closed"
// retry again.
if strings.Contains(err.Error(), "Connection closed by foreign host") {
return true
}
default:
if strings.Contains(err.Error(), "net/http: TLS handshake timeout") {
// If error is - tlsHandshakeTimeoutError, retry.
return true
} else if strings.Contains(err.Error(), "i/o timeout") {
// If error is - tcp timeoutError, retry.
return true
}
} }
} }
return false return false

View File

@ -0,0 +1,64 @@
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"bytes"
"encoding/xml"
"io/ioutil"
"net/http"
)
// Contains common used utilities for tests.
// APIError Used for mocking error response from server.
type APIError struct {
Code string
Description string
HTTPStatusCode int
}
// Mocks XML error response from the server.
func generateErrorResponse(resp *http.Response, APIErr APIError, bucketName string) *http.Response {
// generate error response.
errorResponse := getAPIErrorResponse(APIErr, bucketName)
encodedErrorResponse := encodeResponse(errorResponse)
// write Header.
resp.StatusCode = APIErr.HTTPStatusCode
resp.Body = ioutil.NopCloser(bytes.NewBuffer(encodedErrorResponse))
return resp
}
// getErrorResponse gets in standard error and resource value and
// provides a encodable populated response values.
func getAPIErrorResponse(err APIError, bucketName string) ErrorResponse {
var errResp = ErrorResponse{}
errResp.Code = err.Code
errResp.Message = err.Description
errResp.BucketName = bucketName
return errResp
}
// Encodes the response headers into XML format.
func encodeResponse(response interface{}) []byte {
var bytesBuffer bytes.Buffer
bytesBuffer.WriteString(xml.Header)
encode := xml.NewEncoder(&bytesBuffer)
encode.Encode(response)
return bytesBuffer.Bytes()
}