From 4c555bad2ed4316ee3e53835f96b601e184d08ab Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 22 Jul 2017 10:19:52 +0200 Subject: [PATCH] Set minio-go to v3.0.0 --- vendor/manifest | 4 +- .../minio/minio-go/api-get-object.go | 18 +- .../minio/minio-go/api-put-object.go | 4 +- .../src/github.com/minio/minio-go/api-stat.go | 10 +- vendor/src/github.com/minio/minio-go/api.go | 2 +- .../minio/minio-go/api_functional_v2_test.go | 1470 ++++++ .../minio/minio-go/api_functional_v4_test.go | 2410 ++++++++++ .../github.com/minio/minio-go/bucket-cache.go | 28 +- .../github.com/minio/minio-go/core_test.go | 149 +- .../src/github.com/minio/minio-go/docs/API.md | 22 +- .../minio-go/examples/s3/composeobject.go | 21 +- .../minio/minio-go/examples/s3/copyobject.go | 5 +- .../examples/s3/putobject-progress.go | 4 +- .../examples/s3/putobject-s3-accelerate.go | 2 +- .../minio/minio-go/functional_tests.go | 4058 ----------------- .../s3signer/request-signature-streaming.go | 2 +- .../minio/minio-go/test-utils_test.go | 4 +- .../github.com/minio/minio-go/transport.go | 48 - .../minio/minio-go/transport_1_5.go | 39 - .../minio/minio-go/transport_1_6.go | 40 - 20 files changed, 3933 insertions(+), 4407 deletions(-) create mode 100644 vendor/src/github.com/minio/minio-go/api_functional_v2_test.go create mode 100644 vendor/src/github.com/minio/minio-go/api_functional_v4_test.go delete mode 100644 vendor/src/github.com/minio/minio-go/functional_tests.go delete mode 100644 vendor/src/github.com/minio/minio-go/transport.go delete mode 100644 vendor/src/github.com/minio/minio-go/transport_1_5.go delete mode 100644 vendor/src/github.com/minio/minio-go/transport_1_6.go diff --git a/vendor/manifest b/vendor/manifest index 9f8f9308c..9240d1fa7 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -46,8 +46,8 @@ { "importpath": "github.com/minio/minio-go", "repository": "https://github.com/minio/minio-go", - "revision": "bd8e1d8a93f006a0207e026353bf0644ffcdd320", - "branch": "master" + "revision": "5ca66c9a35ba1cd674484be99dc97aa0973afe12", + "branch": "HEAD" }, { "importpath": "github.com/ncw/swift", diff --git a/vendor/src/github.com/minio/minio-go/api-get-object.go b/vendor/src/github.com/minio/minio-go/api-get-object.go index 9bd784ffa..1078d2f98 100644 --- a/vendor/src/github.com/minio/minio-go/api-get-object.go +++ b/vendor/src/github.com/minio/minio-go/api-get-object.go @@ -679,18 +679,12 @@ func (c Client) getObject(bucketName, objectName string, reqHeaders RequestHeade if contentType == "" { contentType = "application/octet-stream" } - - objectStat := ObjectInfo{ - ETag: md5sum, - Key: objectName, - Size: resp.ContentLength, - LastModified: date, - ContentType: contentType, - // Extract only the relevant header keys describing the object. - // following function filters out a list of standard set of keys - // which are not part of object metadata. - Metadata: extractObjMetadata(resp.Header), - } + var objectStat ObjectInfo + objectStat.ETag = md5sum + objectStat.Key = objectName + objectStat.Size = resp.ContentLength + objectStat.LastModified = date + objectStat.ContentType = contentType // do not close body here, caller will close return resp.Body, objectStat, nil diff --git a/vendor/src/github.com/minio/minio-go/api-put-object.go b/vendor/src/github.com/minio/minio-go/api-put-object.go index 1fa0a9238..2ea498789 100644 --- a/vendor/src/github.com/minio/minio-go/api-put-object.go +++ b/vendor/src/github.com/minio/minio-go/api-put-object.go @@ -194,7 +194,7 @@ func (c Client) putObjectCommon(bucketName, objectName string, reader io.Reader, } if c.overrideSignerType.IsV2() { - if size >= 0 && size < minPartSize { + if size > 0 && size < minPartSize { return c.putObjectNoChecksum(bucketName, objectName, reader, size, metadata, progress) } return c.putObjectMultipart(bucketName, objectName, reader, size, metadata, progress) @@ -203,8 +203,6 @@ func (c Client) putObjectCommon(bucketName, objectName string, reader io.Reader, // If size cannot be found on a stream, it is not possible // to upload using streaming signature, fall back to multipart. if size < 0 { - // Set regular signature calculation. - c.overrideSignerType = credentials.SignatureV4 return c.putObjectMultipart(bucketName, objectName, reader, size, metadata, progress) } diff --git a/vendor/src/github.com/minio/minio-go/api-stat.go b/vendor/src/github.com/minio/minio-go/api-stat.go index 5f06bfc9e..4b530327b 100644 --- a/vendor/src/github.com/minio/minio-go/api-stat.go +++ b/vendor/src/github.com/minio/minio-go/api-stat.go @@ -167,6 +167,11 @@ func (c Client) statObject(bucketName, objectName string, reqHeaders RequestHead contentType = "application/octet-stream" } + // Extract only the relevant header keys describing the object. + // following function filters out a list of standard set of keys + // which are not part of object metadata. + metadata := extractObjMetadata(resp.Header) + // Save object metadata info. return ObjectInfo{ ETag: md5sum, @@ -174,9 +179,6 @@ func (c Client) statObject(bucketName, objectName string, reqHeaders RequestHead Size: size, LastModified: date, ContentType: contentType, - // Extract only the relevant header keys describing the object. - // following function filters out a list of standard set of keys - // which are not part of object metadata. - Metadata: extractObjMetadata(resp.Header), + Metadata: metadata, }, nil } diff --git a/vendor/src/github.com/minio/minio-go/api.go b/vendor/src/github.com/minio/minio-go/api.go index a25acc338..6fe508aa1 100644 --- a/vendor/src/github.com/minio/minio-go/api.go +++ b/vendor/src/github.com/minio/minio-go/api.go @@ -211,7 +211,7 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re // Instantiate http client and bucket location cache. clnt.httpClient = &http.Client{ - Transport: defaultMinioTransport, + Transport: http.DefaultTransport, CheckRedirect: redirectHeaders, } diff --git a/vendor/src/github.com/minio/minio-go/api_functional_v2_test.go b/vendor/src/github.com/minio/minio-go/api_functional_v2_test.go new file mode 100644 index 000000000..e81596ecf --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/api_functional_v2_test.go @@ -0,0 +1,1470 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + "log" + "math/rand" + "net/http" + "net/url" + "os" + "reflect" + "strings" + "testing" + "time" + + "github.com/minio/minio-go/pkg/policy" +) + +// Tests bucket re-create errors. +func TestMakeBucketErrorV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + t.Skip("skipping region functional tests for non s3 runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'eu-west-1'. + if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { + t.Fatal("Error:", err, bucketName) + } + if err = c.MakeBucket(bucketName, "eu-west-1"); err == nil { + t.Fatal("Error: make bucket should should fail for", bucketName) + } + // Verify valid error response from server. + if ToErrorResponse(err).Code != "BucketAlreadyExists" && + ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { + t.Fatal("Error: Invalid error returned by server", err) + } + if err = c.RemoveBucket(bucketName); err != nil { + t.Fatal("Error:", err, bucketName) + } +} + +// Test get object reader to not throw error on being closed twice. +func TestGetObjectClosedTwiceV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K. + buf := bytes.Repeat([]byte("h"), rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + if err := r.Close(); err != nil { + t.Fatal("Error:", err) + } + if err := r.Close(); err == nil { + t.Fatal("Error: object is already closed, should return error") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests removing partially uploaded objects. +func TestRemovePartiallyUploadedV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping function tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + r := bytes.NewReader(bytes.Repeat([]byte("a"), 128*1024)) + + reader, writer := io.Pipe() + go func() { + i := 0 + for i < 25 { + _, cerr := io.CopyN(writer, r, 128*1024) + if cerr != nil { + t.Fatal("Error:", cerr, bucketName) + } + i++ + r.Seek(0, 0) + } + writer.CloseWithError(errors.New("proactively closed to be verified later")) + }() + + objectName := bucketName + "-resumable" + _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") + if err == nil { + t.Fatal("Error: PutObject should fail.") + } + if err.Error() != "proactively closed to be verified later" { + t.Fatal("Error:", err) + } + err = c.RemoveIncompleteUpload(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests FPutObject hidden contentType setting +func TestFPutObjectV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Make a temp file with 11*1024*1024 bytes of data. + file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") + if err != nil { + t.Fatal("Error:", err) + } + + r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024)) + n, err := io.CopyN(file, r, 11*1024*1024) + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + // Close the file pro-actively for windows. + err = file.Close() + if err != nil { + t.Fatal("Error:", err) + } + + // Set base object name + objectName := bucketName + "FPutObject" + + // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) + n, err = c.FPutObject(bucketName, objectName+"-standard", file.Name(), "application/octet-stream") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + // Perform FPutObject with no contentType provided (Expecting application/octet-stream) + n, err = c.FPutObject(bucketName, objectName+"-Octet", file.Name(), "") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + // Add extension to temp file name + fileName := file.Name() + err = os.Rename(file.Name(), fileName+".gtar") + if err != nil { + t.Fatal("Error:", err) + } + + // Perform FPutObject with no contentType provided (Expecting application/x-gtar) + n, err = c.FPutObject(bucketName, objectName+"-GTar", fileName+".gtar", "") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + // Check headers + rStandard, err := c.StatObject(bucketName, objectName+"-standard") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName+"-standard") + } + if rStandard.ContentType != "application/octet-stream" { + t.Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", + "application/octet-stream", rStandard.ContentType) + } + + rOctet, err := c.StatObject(bucketName, objectName+"-Octet") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName+"-Octet") + } + if rOctet.ContentType != "application/octet-stream" { + t.Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", + "application/octet-stream", rStandard.ContentType) + } + + rGTar, err := c.StatObject(bucketName, objectName+"-GTar") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName+"-GTar") + } + if rGTar.ContentType != "application/x-gtar" { + t.Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", + "application/x-gtar", rStandard.ContentType) + } + + // Remove all objects and bucket and temp file + err = c.RemoveObject(bucketName, objectName+"-standard") + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveObject(bucketName, objectName+"-Octet") + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveObject(bucketName, objectName+"-GTar") + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + + err = os.Remove(fileName + ".gtar") + if err != nil { + t.Fatal("Error:", err) + } + +} + +// Tests various bucket supported formats. +func TestMakeBucketRegionsV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + t.Skip("skipping region functional tests for non s3 runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'eu-central-1'. + if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { + t.Fatal("Error:", err, bucketName) + } + + if err = c.RemoveBucket(bucketName); err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Make a new bucket with '.' in its name, in 'us-west-2'. This + // request is internally staged into a path style instead of + // virtual host style. + if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { + t.Fatal("Error:", err, bucketName+".withperiod") + } + + // Remove the newly created bucket. + if err = c.RemoveBucket(bucketName + ".withperiod"); err != nil { + t.Fatal("Error:", err, bucketName+".withperiod") + } +} + +// Tests get object ReaderSeeker interface methods. +func TestGetObjectReadSeekFunctionalV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K. + buf := bytes.Repeat([]byte("2"), rand.Intn(1<<20)+32*1024) + + // Save the data. + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + offset := int64(2048) + n, err = r.Seek(offset, 0) + if err != nil { + t.Fatal("Error:", err, offset) + } + if n != offset { + t.Fatalf("Error: number of bytes seeked does not match, want %v, got %v\n", + offset, n) + } + n, err = r.Seek(0, 1) + if err != nil { + t.Fatal("Error:", err) + } + if n != offset { + t.Fatalf("Error: number of current seek does not match, want %v, got %v\n", + offset, n) + } + _, err = r.Seek(offset, 2) + if err == nil { + t.Fatal("Error: seek on positive offset for whence '2' should error out") + } + n, err = r.Seek(-offset, 2) + if err != nil { + t.Fatal("Error:", err) + } + if n != st.Size-offset { + t.Fatalf("Error: number of bytes seeked back does not match, want %d, got %v\n", st.Size-offset, n) + } + + var buffer1 bytes.Buffer + if _, err = io.CopyN(&buffer1, r, st.Size); err != nil { + if err != io.EOF { + t.Fatal("Error:", err) + } + } + if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) { + t.Fatal("Error: Incorrect read bytes v/s original buffer.") + } + + // Seek again and read again. + n, err = r.Seek(offset-1, 0) + if err != nil { + t.Fatal("Error:", err) + } + if n != (offset - 1) { + t.Fatalf("Error: number of bytes seeked back does not match, want %v, got %v\n", offset-1, n) + } + + var buffer2 bytes.Buffer + if _, err = io.CopyN(&buffer2, r, st.Size); err != nil { + if err != io.EOF { + t.Fatal("Error:", err) + } + } + // Verify now lesser bytes. + if !bytes.Equal(buf[2047:], buffer2.Bytes()) { + t.Fatal("Error: Incorrect read bytes v/s original buffer.") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests get object ReaderAt interface methods. +func TestGetObjectReadAtFunctionalV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := bytes.Repeat([]byte("8"), rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + offset := int64(2048) + + // Read directly + buf2 := make([]byte, 512) + buf3 := make([]byte, 512) + buf4 := make([]byte, 512) + + m, err := r.ReadAt(buf2, offset) + if err != nil { + t.Fatal("Error:", err, st.Size, len(buf2), offset) + } + if m != len(buf2) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2)) + } + if !bytes.Equal(buf2, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + m, err = r.ReadAt(buf3, offset) + if err != nil { + t.Fatal("Error:", err, st.Size, len(buf3), offset) + } + if m != len(buf3) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3)) + } + if !bytes.Equal(buf3, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + m, err = r.ReadAt(buf4, offset) + if err != nil { + t.Fatal("Error:", err, st.Size, len(buf4), offset) + } + if m != len(buf4) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4)) + } + if !bytes.Equal(buf4, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + + buf5 := make([]byte, n) + // Read the whole object. + m, err = r.ReadAt(buf5, 0) + if err != nil { + if err != io.EOF { + t.Fatal("Error:", err, len(buf5)) + } + } + if m != len(buf5) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf5)) + } + if !bytes.Equal(buf, buf5) { + t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") + } + + buf6 := make([]byte, n+1) + // Read the whole object and beyond. + _, err = r.ReadAt(buf6, 0) + if err != nil { + if err != io.EOF { + t.Fatal("Error:", err, len(buf6)) + } + } + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests copy object +func TestCopyObjectV2(t *testing.T) { + if testing.Short() { + t.Skip("Skipping functional tests for short runs") + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object + c, err := NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Make a new bucket in 'us-east-1' (destination bucket). + err = c.MakeBucket(bucketName+"-copy", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName+"-copy") + } + + // Generate data more than 32K + buf := bytes.Repeat([]byte("9"), rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match want %v, got %v", + len(buf), n) + } + + dst, err := NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) + if err != nil { + t.Fatal(err) + } + + src := NewSourceInfo(bucketName, objectName, nil) + err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatal("Error:", err) + } + + // Perform the Copy + err = c.CopyObject(dst, src) + if err != nil { + t.Fatal("Error:", err, bucketName+"-copy", objectName+"-copy") + } + + // Source object + reader, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + // Destination object + readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy") + if err != nil { + t.Fatal("Error:", err) + } + // Check the various fields of source object against destination object. + objInfo, err := reader.Stat() + if err != nil { + t.Fatal("Error:", err) + } + objInfoCopy, err := readerCopy.Stat() + if err != nil { + t.Fatal("Error:", err) + } + if objInfo.Size != objInfoCopy.Size { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", + objInfo.Size, objInfoCopy.Size) + } + + // Remove all objects and buckets + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + + err = c.RemoveObject(bucketName+"-copy", objectName+"-copy") + if err != nil { + t.Fatal("Error:", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + + err = c.RemoveBucket(bucketName + "-copy") + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests comprehensive list of all methods. +func TestFunctionalV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + c, err := NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable to debug + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate a random file name. + fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + file, err := os.Create(fileName) + if err != nil { + t.Fatal("Error:", err) + } + for i := 0; i < 3; i++ { + buf := make([]byte, rand.Intn(1<<19)) + _, err = file.Write(buf) + if err != nil { + t.Fatal("Error:", err) + } + } + file.Close() + + // Verify if bucket exits and you have access. + var exists bool + exists, err = c.BucketExists(bucketName) + if err != nil { + t.Fatal("Error:", err, bucketName) + } + if !exists { + t.Fatal("Error: could not find ", bucketName) + } + + // Make the bucket 'public read/write'. + err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadWrite) + if err != nil { + t.Fatal("Error:", err) + } + + // List all buckets. + buckets, err := c.ListBuckets() + if len(buckets) == 0 { + t.Fatal("Error: list buckets cannot be empty", buckets) + } + if err != nil { + t.Fatal("Error:", err) + } + + // Verify if previously created bucket is listed in list buckets. + bucketFound := false + for _, bucket := range buckets { + if bucket.Name == bucketName { + bucketFound = true + } + } + + // If bucket not found error out. + if !bucketFound { + t.Fatal("Error: bucket ", bucketName, "not found") + } + + objectName := bucketName + "unique" + + // Generate data + buf := bytes.Repeat([]byte("n"), rand.Intn(1<<19)) + + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "") + if err != nil { + t.Fatal("Error: ", err) + } + if n != int64(len(buf)) { + t.Fatal("Error: bad length ", n, len(buf)) + } + + n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName+"-nolength") + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Instantiate a done channel to close all listing. + doneCh := make(chan struct{}) + defer close(doneCh) + + objFound := false + isRecursive := true // Recursive is true. + for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + t.Fatal("Error: object " + objectName + " not found.") + } + + objFound = false + isRecursive = true // Recursive is true. + for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + t.Fatal("Error: object " + objectName + " not found.") + } + + incompObjNotFound := true + for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { + if objIncompl.Key != "" { + incompObjNotFound = false + break + } + } + if !incompObjNotFound { + t.Fatal("Error: unexpected dangling incomplete upload found.") + } + + newReader, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + newReadBytes, err := ioutil.ReadAll(newReader) + if err != nil { + t.Fatal("Error: ", err) + } + + if !bytes.Equal(newReadBytes, buf) { + t.Fatal("Error: bytes mismatch.") + } + + err = c.FGetObject(bucketName, objectName, fileName+"-f") + if err != nil { + t.Fatal("Error: ", err) + } + + // Generate presigned GET object url. + presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) + if err != nil { + t.Fatal("Error: ", err) + } + // Verify if presigned url works. + resp, err := http.Get(presignedGetURL.String()) + if err != nil { + t.Fatal("Error: ", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatal("Error: ", resp.Status) + } + newPresignedBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal("Error: ", err) + } + if !bytes.Equal(newPresignedBytes, buf) { + t.Fatal("Error: bytes mismatch.") + } + + // Set request parameters. + reqParams := make(url.Values) + reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") + // Generate presigned GET object url. + presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) + if err != nil { + t.Fatal("Error: ", err) + } + // Verify if presigned url works. + resp, err = http.Get(presignedGetURL.String()) + if err != nil { + t.Fatal("Error: ", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatal("Error: ", resp.Status) + } + newPresignedBytes, err = ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal("Error: ", err) + } + if !bytes.Equal(newPresignedBytes, buf) { + t.Fatal("Error: bytes mismatch for presigned GET url.") + } + // Verify content disposition. + if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { + t.Fatalf("Error: wrong Content-Disposition received %s", resp.Header.Get("Content-Disposition")) + } + + presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) + if err != nil { + t.Fatal("Error: ", err) + } + // Generate data more than 32K + buf = bytes.Repeat([]byte("1"), rand.Intn(1<<20)+32*1024) + + req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) + if err != nil { + t.Fatal("Error: ", err) + } + httpClient := &http.Client{ + // Setting a sensible time out of 30secs to wait for response + // headers. Request is pro-actively cancelled after 30secs + // with no response. + Timeout: 30 * time.Second, + Transport: http.DefaultTransport, + } + resp, err = httpClient.Do(req) + if err != nil { + t.Fatal("Error: ", err) + } + + newReader, err = c.GetObject(bucketName, objectName+"-presigned") + if err != nil { + t.Fatal("Error: ", err) + } + + newReadBytes, err = ioutil.ReadAll(newReader) + if err != nil { + t.Fatal("Error: ", err) + } + + if !bytes.Equal(newReadBytes, buf) { + t.Fatal("Error: bytes mismatch.") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-f") + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-nolength") + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-presigned") + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + err = c.RemoveBucket(bucketName) + if err == nil { + t.Fatal("Error:") + } + if err.Error() != "The specified bucket does not exist" { + t.Fatal("Error: ", err) + } + if err = os.Remove(fileName); err != nil { + t.Fatal("Error: ", err) + } + if err = os.Remove(fileName + "-f"); err != nil { + t.Fatal("Error: ", err) + } +} + +func testComposeObjectErrorCases(c *Client, t *testing.T) { + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'us-east-1' (source bucket). + err := c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Test that more than 10K source objects cannot be + // concatenated. + srcArr := [10001]SourceInfo{} + srcSlice := srcArr[:] + dst, err := NewDestinationInfo(bucketName, "object", nil, nil) + if err != nil { + t.Fatal(err) + } + + if err := c.ComposeObject(dst, srcSlice); err == nil { + t.Fatal("Error was expected.") + } else if err.Error() != "There must be as least one and upto 10000 source objects." { + t.Fatal("Got unexpected error: ", err) + } + + // Create a source with invalid offset spec and check that + // error is returned: + // 1. Create the source object. + const badSrcSize = 5 * 1024 * 1024 + buf := bytes.Repeat([]byte("1"), badSrcSize) + _, err = c.PutObject(bucketName, "badObject", bytes.NewReader(buf), "") + if err != nil { + t.Fatal("Error:", err) + } + // 2. Set invalid range spec on the object (going beyond + // object size) + badSrc := NewSourceInfo(bucketName, "badObject", nil) + err = badSrc.SetRange(1, badSrcSize) + if err != nil { + t.Fatal("Error:", err) + } + // 3. ComposeObject call should fail. + if err := c.ComposeObject(dst, []SourceInfo{badSrc}); err == nil { + t.Fatal("Error was expected.") + } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") { + t.Fatal("Got unexpected error: ", err) + } +} + +// Test expected error cases +func TestComposeObjectErrorCasesV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Instantiate new minio client object + c, err := NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + testComposeObjectErrorCases(c, t) +} + +func testComposeMultipleSources(c *Client, t *testing.T) { + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + // Make a new bucket in 'us-east-1' (source bucket). + err := c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Upload a small source object + const srcSize = 1024 * 1024 * 5 + buf := bytes.Repeat([]byte("1"), srcSize) + _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err) + } + + // We will append 10 copies of the object. + srcs := []SourceInfo{} + for i := 0; i < 10; i++ { + srcs = append(srcs, NewSourceInfo(bucketName, "srcObject", nil)) + } + // make the last part very small + err = srcs[9].SetRange(0, 0) + if err != nil { + t.Fatal("unexpected error:", err) + } + + dst, err := NewDestinationInfo(bucketName, "dstObject", nil, nil) + if err != nil { + t.Fatal(err) + } + err = c.ComposeObject(dst, srcs) + if err != nil { + t.Fatal("Error:", err) + } + + objProps, err := c.StatObject(bucketName, "dstObject") + if err != nil { + t.Fatal("Error:", err) + } + + if objProps.Size != 9*srcSize+1 { + t.Fatal("Size mismatched! Expected:", 10000*srcSize, "but got:", objProps.Size) + } +} + +// Test concatenating multiple objects objects +func TestCompose10KSourcesV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Instantiate new minio client object + c, err := NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + testComposeMultipleSources(c, t) +} + +func testEncryptedCopyObject(c *Client, t *testing.T) { + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + // Make a new bucket in 'us-east-1' (source bucket). + err := c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + key1 := NewSSEInfo([]byte("32byteslongsecretkeymustbegiven1"), "AES256") + key2 := NewSSEInfo([]byte("32byteslongsecretkeymustbegiven2"), "AES256") + + // 1. create an sse-c encrypted object to copy by uploading + const srcSize = 1024 * 1024 + buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB + metadata := make(map[string][]string) + for k, v := range key1.GetSSEHeaders() { + metadata[k] = append(metadata[k], v) + } + _, err = c.PutObjectWithSize(bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), metadata, nil) + if err != nil { + t.Fatal("PutObjectWithSize Error:", err) + } + + // 2. copy object and change encryption key + src := NewSourceInfo(bucketName, "srcObject", &key1) + dst, err := NewDestinationInfo(bucketName, "dstObject", &key2, nil) + if err != nil { + t.Fatal("Error:", err) + } + + err = c.CopyObject(dst, src) + if err != nil { + t.Fatal("CopyObject Error:", err) + } + + // 3. get copied object and check if content is equal + reqH := NewGetReqHeaders() + for k, v := range key2.GetSSEHeaders() { + reqH.Set(k, v) + } + coreClient := Core{c} + reader, _, err := coreClient.GetObject(bucketName, "dstObject", reqH) + if err != nil { + t.Fatal("GetObject Error:", err) + } + defer reader.Close() + + decBytes, err := ioutil.ReadAll(reader) + if err != nil { + log.Fatalln(err) + } + if !bytes.Equal(decBytes, buf) { + log.Fatal("downloaded object mismatched for encrypted object") + } +} + +// Test encrypted copy object +func TestEncryptedCopyObjectV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Instantiate new minio client object + c, err := NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + testEncryptedCopyObject(c, t) +} + +func testUserMetadataCopying(c *Client, t *testing.T) { + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + // Make a new bucket in 'us-east-1' (source bucket). + err := c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + fetchMeta := func(object string) (h http.Header) { + objInfo, err := c.StatObject(bucketName, object) + if err != nil { + t.Fatal("Metadata fetch error:", err) + } + h = make(http.Header) + for k, vs := range objInfo.Metadata { + if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { + for _, v := range vs { + h.Add(k, v) + } + } + } + return h + } + + // 1. create a client encrypted object to copy by uploading + const srcSize = 1024 * 1024 + buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB + metadata := make(http.Header) + metadata.Set("x-amz-meta-myheader", "myvalue") + _, err = c.PutObjectWithMetadata(bucketName, "srcObject", + bytes.NewReader(buf), metadata, nil) + if err != nil { + t.Fatal("Put Error:", err) + } + if !reflect.DeepEqual(metadata, fetchMeta("srcObject")) { + t.Fatal("Unequal metadata") + } + + // 2. create source + src := NewSourceInfo(bucketName, "srcObject", nil) + // 2.1 create destination with metadata set + dst1, err := NewDestinationInfo(bucketName, "dstObject-1", nil, map[string]string{"notmyheader": "notmyvalue"}) + if err != nil { + t.Fatal("Error:", err) + } + + // 3. Check that copying to an object with metadata set resets + // the headers on the copy. + err = c.CopyObject(dst1, src) + if err != nil { + t.Fatal("Error:", err) + } + + expectedHeaders := make(http.Header) + expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") + if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-1")) { + t.Fatal("Unequal metadata") + } + + // 4. create destination with no metadata set and same source + dst2, err := NewDestinationInfo(bucketName, "dstObject-2", nil, nil) + if err != nil { + t.Fatal("Error:", err) + + } + src = NewSourceInfo(bucketName, "srcObject", nil) + + // 5. Check that copying to an object with no metadata set, + // copies metadata. + err = c.CopyObject(dst2, src) + if err != nil { + t.Fatal("Error:", err) + } + + expectedHeaders = metadata + if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-2")) { + t.Fatal("Unequal metadata") + } + + // 6. Compose a pair of sources. + srcs := []SourceInfo{ + NewSourceInfo(bucketName, "srcObject", nil), + NewSourceInfo(bucketName, "srcObject", nil), + } + dst3, err := NewDestinationInfo(bucketName, "dstObject-3", nil, nil) + if err != nil { + t.Fatal("Error:", err) + + } + + err = c.ComposeObject(dst3, srcs) + if err != nil { + t.Fatal("Error:", err) + } + + // Check that no headers are copied in this case + if !reflect.DeepEqual(make(http.Header), fetchMeta("dstObject-3")) { + t.Fatal("Unequal metadata") + } + + // 7. Compose a pair of sources with dest user metadata set. + srcs = []SourceInfo{ + NewSourceInfo(bucketName, "srcObject", nil), + NewSourceInfo(bucketName, "srcObject", nil), + } + dst4, err := NewDestinationInfo(bucketName, "dstObject-4", nil, map[string]string{"notmyheader": "notmyvalue"}) + if err != nil { + t.Fatal("Error:", err) + + } + + err = c.ComposeObject(dst4, srcs) + if err != nil { + t.Fatal("Error:", err) + } + + // Check that no headers are copied in this case + expectedHeaders = make(http.Header) + expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") + if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-4")) { + t.Fatal("Unequal metadata") + } +} + +func TestUserMetadataCopyingV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Instantiate new minio client object + c, err := NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // c.TraceOn(os.Stderr) + testUserMetadataCopying(c, t) +} diff --git a/vendor/src/github.com/minio/minio-go/api_functional_v4_test.go b/vendor/src/github.com/minio/minio-go/api_functional_v4_test.go new file mode 100644 index 000000000..e9593ddaf --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/api_functional_v4_test.go @@ -0,0 +1,2410 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/pkg/policy" +) + +const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + b[i] = letterBytes[idx] + i-- + } + cache >>= letterIdxBits + remain-- + } + return prefix + string(b[0:30-len(prefix)]) +} + +// Tests bucket re-create errors. +func TestMakeBucketError(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + t.Skip("skipping region functional tests for non s3 runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'eu-central-1'. + if err = c.MakeBucket(bucketName, "eu-central-1"); err != nil { + t.Fatal("Error:", err, bucketName) + } + if err = c.MakeBucket(bucketName, "eu-central-1"); err == nil { + t.Fatal("Error: make bucket should should fail for", bucketName) + } + // Verify valid error response from server. + if ToErrorResponse(err).Code != "BucketAlreadyExists" && + ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { + t.Fatal("Error: Invalid error returned by server", err) + } + if err = c.RemoveBucket(bucketName); err != nil { + t.Fatal("Error:", err, bucketName) + } + if err = c.MakeBucket(bucketName+"..-1", "eu-central-1"); err == nil { + t.Fatal("Error:", err, bucketName+"..-1") + } + // Verify valid error response. + if err != nil && err.Error() != "Bucket name contains invalid characters" { + t.Fatal("Error: Invalid error returned by server", err) + } + if err = c.MakeBucket(bucketName+"AAA-1", "eu-central-1"); err == nil { + t.Fatal("Error:", err, bucketName+"..-1") + } + // Verify valid error response. + if err != nil && err.Error() != "Bucket name contains invalid characters" { + t.Fatal("Error: Invalid error returned by server", err) + } +} + +// Tests various bucket supported formats. +func TestMakeBucketRegions(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + t.Skip("skipping region functional tests for non s3 runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'eu-central-1'. + if err = c.MakeBucket(bucketName, "eu-central-1"); err != nil { + t.Fatal("Error:", err, bucketName) + } + + if err = c.RemoveBucket(bucketName); err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Make a new bucket with '.' in its name, in 'us-west-2'. This + // request is internally staged into a path style instead of + // virtual host style. + if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { + t.Fatal("Error:", err, bucketName+".withperiod") + } + + // Remove the newly created bucket. + if err = c.RemoveBucket(bucketName + ".withperiod"); err != nil { + t.Fatal("Error:", err, bucketName+".withperiod") + } +} + +// Test PutObject using a large data to trigger multipart readat +func TestPutObjectReadAt(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data using 4 parts so that all 3 'workers' are utilized and a part is leftover. + // Use different data for each part for multipart tests to ensure part order at the end. + var buf []byte + for i := 0; i < 4; i++ { + buf = append(buf, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) + } + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + // Object content type + objectContentType := "binary/octet-stream" + + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), objectContentType) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + if st.ContentType != objectContentType { + t.Fatalf("Error: Content types don't match, expected: %+v, found: %+v\n", objectContentType, st.ContentType) + } + if err := r.Close(); err != nil { + t.Fatal("Error:", err) + } + if err := r.Close(); err == nil { + t.Fatal("Error: object is already closed, should return error") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Test PutObject using a large data to trigger multipart readat +func TestPutObjectWithMetadata(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data using 2 parts + // Use different data in each part for multipart tests to ensure part order at the end. + var buf []byte + for i := 0; i < 2; i++ { + buf = append(buf, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) + } + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + + // Object custom metadata + customContentType := "custom/contenttype" + + n, err := c.PutObjectWithMetadata(bucketName, objectName, bytes.NewReader(buf), map[string][]string{ + "Content-Type": {customContentType}, + }, nil) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + if st.ContentType != customContentType { + t.Fatalf("Error: Expected and found content types do not match, want %v, got %v\n", + customContentType, st.ContentType) + } + if err := r.Close(); err != nil { + t.Fatal("Error:", err) + } + if err := r.Close(); err == nil { + t.Fatal("Error: object is already closed, should return error") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Test put object with streaming signature. +func TestPutObjectStreaming(t *testing.T) { + if testing.Short() { + t.Skip("skipping function tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), + "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Upload an object. + sizes := []int64{0, 64*1024 - 1, 64 * 1024} + objectName := "test-object" + for i, size := range sizes { + data := bytes.Repeat([]byte("a"), int(size)) + n, err := c.PutObjectStreaming(bucketName, objectName, bytes.NewReader(data)) + if err != nil { + t.Fatalf("Test %d Error: %v %s %s", i+1, err, bucketName, objectName) + } + + if n != size { + t.Errorf("Test %d Expected upload object size %d but got %d", i+1, size, n) + } + } + + // Remove the object. + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + + // Remove the bucket. + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Test listing no partially uploaded objects upon putObject error. +func TestListNoPartiallyUploadedObjects(t *testing.T) { + if testing.Short() { + t.Skip("skipping function tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + r := bytes.NewReader(bytes.Repeat([]byte("0"), minPartSize*2)) + + reader, writer := io.Pipe() + go func() { + i := 0 + for i < 25 { + _, cerr := io.CopyN(writer, r, (minPartSize*2)/25) + if cerr != nil { + t.Fatal("Error:", cerr, bucketName) + } + i++ + r.Seek(0, 0) + } + writer.CloseWithError(errors.New("proactively closed to be verified later")) + }() + + objectName := bucketName + "-resumable" + _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") + if err == nil { + t.Fatal("Error: PutObject should fail.") + } + if !strings.Contains(err.Error(), "proactively closed to be verified later") { + t.Fatal("Error:", err) + } + + doneCh := make(chan struct{}) + defer close(doneCh) + + isRecursive := true + multiPartObjectCh := c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) + + var activeUploads bool + for multiPartObject := range multiPartObjectCh { + if multiPartObject.Err != nil { + t.Fatalf("Error: Error when listing incomplete upload") + } + activeUploads = true + } + if activeUploads { + t.Errorf("There should be no active uploads in progress upon error for %s/%s", bucketName, objectName) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Test get object seeker from the end, using whence set to '2'. +func TestGetOjectSeekEnd(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := bytes.Repeat([]byte("1"), rand.Intn(1<<20)+32*1024) + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + pos, err := r.Seek(-100, 2) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if pos != st.Size-100 { + t.Fatalf("Expected %d, got %d instead", pos, st.Size-100) + } + buf2 := make([]byte, 100) + m, err := io.ReadFull(r, buf2) + if err != nil { + t.Fatal("Error: reading through io.ReadFull", err, bucketName, objectName) + } + if m != len(buf2) { + t.Fatalf("Expected %d bytes, got %d", len(buf2), m) + } + hexBuf1 := fmt.Sprintf("%02x", buf[len(buf)-100:]) + hexBuf2 := fmt.Sprintf("%02x", buf2[:m]) + if hexBuf1 != hexBuf2 { + t.Fatalf("Expected %s, got %s instead", hexBuf1, hexBuf2) + } + pos, err = r.Seek(-100, 2) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if pos != st.Size-100 { + t.Fatalf("Expected %d, got %d instead", pos, st.Size-100) + } + if err = r.Close(); err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } +} + +// Test get object reader to not throw error on being closed twice. +func TestGetObjectClosedTwice(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := bytes.Repeat([]byte("1"), rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + if err := r.Close(); err != nil { + t.Fatal("Error:", err) + } + if err := r.Close(); err == nil { + t.Fatal("Error: object is already closed, should return error") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Test removing multiple objects with Remove API +func TestRemoveMultipleObjects(t *testing.T) { + if testing.Short() { + t.Skip("skipping function tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + + if err != nil { + t.Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) + + // Multi remove of 1100 objects + nrObjects := 1100 + + objectsCh := make(chan string) + + go func() { + defer close(objectsCh) + // Upload objects and send them to objectsCh + for i := 0; i < nrObjects; i++ { + objectName := "sample" + strconv.Itoa(i) + ".txt" + _, err = c.PutObject(bucketName, objectName, r, "application/octet-stream") + if err != nil { + t.Error("Error: PutObject shouldn't fail.", err) + continue + } + objectsCh <- objectName + } + }() + + // Call RemoveObjects API + errorCh := c.RemoveObjects(bucketName, objectsCh) + + // Check if errorCh doesn't receive any error + select { + case r, more := <-errorCh: + if more { + t.Fatalf("Unexpected error, objName(%v) err(%v)", r.ObjectName, r.Err) + } + } + + // Clean the bucket created by the test + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests FPutObject of a big file to trigger multipart +func TestFPutObjectMultipart(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Make a temp file with minPartSize*4 bytes of data. + file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") + if err != nil { + t.Fatal("Error:", err) + } + + // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. + var buffer []byte + for i := 0; i < 4; i++ { + buffer = append(buffer, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) + } + + size, err := file.Write(buffer) + if err != nil { + t.Fatal("Error:", err) + } + if size != minPartSize*4 { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, size) + } + + // Close the file pro-actively for windows. + err = file.Close() + if err != nil { + t.Fatal("Error:", err) + } + + // Set base object name + objectName := bucketName + "FPutObject" + objectContentType := "testapplication/octet-stream" + + // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) + n, err := c.FPutObject(bucketName, objectName+"-standard", file.Name(), objectContentType) + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(minPartSize*4) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, n) + } + + r, err := c.GetObject(bucketName, objectName+"-standard") + if err != nil { + t.Fatalf("Unexpected error: %v\n", err) + } + objInfo, err := r.Stat() + if err != nil { + t.Fatalf("Unexpected error: %v\n", err) + } + if objInfo.Size != minPartSize*4 { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, n) + } + if objInfo.ContentType != objectContentType { + t.Fatalf("Error: Content types don't match, want %v, got %v\n", objectContentType, objInfo.ContentType) + } + + // Remove all objects and bucket and temp file + err = c.RemoveObject(bucketName, objectName+"-standard") + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests FPutObject hidden contentType setting +func TestFPutObject(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Make a temp file with minPartSize*4 bytes of data. + file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") + if err != nil { + t.Fatal("Error:", err) + } + + // Upload 4 parts worth of data to use all 3 of multiparts 'workers' and have an extra part. + // Use different data in part for multipart tests to check parts are uploaded in correct order. + var buffer []byte + for i := 0; i < 4; i++ { + buffer = append(buffer, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) + } + + // Write the data to the file. + size, err := file.Write(buffer) + if err != nil { + t.Fatal("Error:", err) + } + if size != minPartSize*4 { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, size) + } + + // Close the file pro-actively for windows. + err = file.Close() + if err != nil { + t.Fatal("Error:", err) + } + + // Set base object name + objectName := bucketName + "FPutObject" + + // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) + n, err := c.FPutObject(bucketName, objectName+"-standard", file.Name(), "application/octet-stream") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(minPartSize*4) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, n) + } + + // Perform FPutObject with no contentType provided (Expecting application/octet-stream) + n, err = c.FPutObject(bucketName, objectName+"-Octet", file.Name(), "") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(minPartSize*4) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, n) + } + + // Add extension to temp file name + fileName := file.Name() + err = os.Rename(file.Name(), fileName+".gtar") + if err != nil { + t.Fatal("Error:", err) + } + + // Perform FPutObject with no contentType provided (Expecting application/x-gtar) + n, err = c.FPutObject(bucketName, objectName+"-GTar", fileName+".gtar", "") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(minPartSize*4) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, n) + } + + // Check headers + rStandard, err := c.StatObject(bucketName, objectName+"-standard") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName+"-standard") + } + if rStandard.ContentType != "application/octet-stream" { + t.Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", + "application/octet-stream", rStandard.ContentType) + } + + rOctet, err := c.StatObject(bucketName, objectName+"-Octet") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName+"-Octet") + } + if rOctet.ContentType != "application/octet-stream" { + t.Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", + "application/octet-stream", rStandard.ContentType) + } + + rGTar, err := c.StatObject(bucketName, objectName+"-GTar") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName+"-GTar") + } + if rGTar.ContentType != "application/x-gtar" { + t.Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", + "application/x-gtar", rStandard.ContentType) + } + + // Remove all objects and bucket and temp file + err = c.RemoveObject(bucketName, objectName+"-standard") + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveObject(bucketName, objectName+"-Octet") + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveObject(bucketName, objectName+"-GTar") + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + + err = os.Remove(fileName + ".gtar") + if err != nil { + t.Fatal("Error:", err) + } + +} + +// Tests get object ReaderSeeker interface methods. +func TestGetObjectReadSeekFunctional(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := bytes.Repeat([]byte("2"), rand.Intn(1<<20)+32*1024) + bufSize := len(buf) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(bufSize) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + defer func() { + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + }() + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(bufSize) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + // This following function helps us to compare data from the reader after seek + // with the data from the original buffer + cmpData := func(r io.Reader, start, end int) { + if end-start == 0 { + return + } + buffer := bytes.NewBuffer([]byte{}) + if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { + if err != io.EOF { + t.Fatal("Error:", err) + } + } + if !bytes.Equal(buf[start:end], buffer.Bytes()) { + t.Fatal("Error: Incorrect read bytes v/s original buffer.") + } + } + + // Generic seek error for errors other than io.EOF + seekErr := errors.New("seek error") + + testCases := []struct { + offset int64 + whence int + pos int64 + err error + shouldCmp bool + start int + end int + }{ + // Start from offset 0, fetch data and compare + {0, 0, 0, nil, true, 0, 0}, + // Start from offset 2048, fetch data and compare + {2048, 0, 2048, nil, true, 2048, bufSize}, + // Start from offset larger than possible + {int64(bufSize) + 1024, 0, 0, seekErr, false, 0, 0}, + // Move to offset 0 without comparing + {0, 0, 0, nil, false, 0, 0}, + // Move one step forward and compare + {1, 1, 1, nil, true, 1, bufSize}, + // Move larger than possible + {int64(bufSize), 1, 0, seekErr, false, 0, 0}, + // Provide negative offset with CUR_SEEK + {int64(-1), 1, 0, seekErr, false, 0, 0}, + // Test with whence SEEK_END and with positive offset + {1024, 2, int64(bufSize) - 1024, io.EOF, true, 0, 0}, + // Test with whence SEEK_END and with negative offset + {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, + // Test with whence SEEK_END and with large negative offset + {-int64(bufSize) * 2, 2, 0, seekErr, true, 0, 0}, + } + + for i, testCase := range testCases { + // Perform seek operation + n, err := r.Seek(testCase.offset, testCase.whence) + // We expect an error + if testCase.err == seekErr && err == nil { + t.Fatalf("Test %d, unexpected err value: expected: %v, found: %v", i+1, testCase.err, err) + } + // We expect a specific error + if testCase.err != seekErr && testCase.err != err { + t.Fatalf("Test %d, unexpected err value: expected: %v, found: %v", i+1, testCase.err, err) + } + // If we expect an error go to the next loop + if testCase.err != nil { + continue + } + // Check the returned seek pos + if n != testCase.pos { + t.Fatalf("Test %d, error: number of bytes seeked does not match, want %v, got %v\n", i+1, + testCase.pos, n) + } + // Compare only if shouldCmp is activated + if testCase.shouldCmp { + cmpData(r, testCase.start, testCase.end) + } + } +} + +// Tests get object ReaderAt interface methods. +func TestGetObjectReadAtFunctional(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := bytes.Repeat([]byte("3"), rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + offset := int64(2048) + + // read directly + buf1 := make([]byte, 512) + buf2 := make([]byte, 512) + buf3 := make([]byte, 512) + buf4 := make([]byte, 512) + + // Test readAt before stat is called. + m, err := r.ReadAt(buf1, offset) + if err != nil { + t.Fatal("Error:", err, len(buf1), offset) + } + if m != len(buf1) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf1)) + } + if !bytes.Equal(buf1, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + m, err = r.ReadAt(buf2, offset) + if err != nil { + t.Fatal("Error:", err, st.Size, len(buf2), offset) + } + if m != len(buf2) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2)) + } + if !bytes.Equal(buf2, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + m, err = r.ReadAt(buf3, offset) + if err != nil { + t.Fatal("Error:", err, st.Size, len(buf3), offset) + } + if m != len(buf3) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3)) + } + if !bytes.Equal(buf3, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + m, err = r.ReadAt(buf4, offset) + if err != nil { + t.Fatal("Error:", err, st.Size, len(buf4), offset) + } + if m != len(buf4) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4)) + } + if !bytes.Equal(buf4, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + + buf5 := make([]byte, n) + // Read the whole object. + m, err = r.ReadAt(buf5, 0) + if err != nil { + if err != io.EOF { + t.Fatal("Error:", err, len(buf5)) + } + } + if m != len(buf5) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf5)) + } + if !bytes.Equal(buf, buf5) { + t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") + } + + buf6 := make([]byte, n+1) + // Read the whole object and beyond. + _, err = r.ReadAt(buf6, 0) + if err != nil { + if err != io.EOF { + t.Fatal("Error:", err, len(buf6)) + } + } + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Test Presigned Post Policy +func TestPresignedPostPolicy(t *testing.T) { + if testing.Short() { + t.Skip("Skipping functional tests for short runs") + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object + c, err := NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := bytes.Repeat([]byte("4"), rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match want %v, got %v", + len(buf), n) + } + + policy := NewPostPolicy() + + if err := policy.SetBucket(""); err == nil { + t.Fatalf("Error: %s", err) + } + if err := policy.SetKey(""); err == nil { + t.Fatalf("Error: %s", err) + } + if err := policy.SetKeyStartsWith(""); err == nil { + t.Fatalf("Error: %s", err) + } + if err := policy.SetExpires(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)); err == nil { + t.Fatalf("Error: %s", err) + } + if err := policy.SetContentType(""); err == nil { + t.Fatalf("Error: %s", err) + } + if err := policy.SetContentLengthRange(1024*1024, 1024); err == nil { + t.Fatalf("Error: %s", err) + } + + policy.SetBucket(bucketName) + policy.SetKey(objectName) + policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days + policy.SetContentType("image/png") + policy.SetContentLengthRange(1024, 1024*1024) + + _, _, err = c.PresignedPostPolicy(policy) + if err != nil { + t.Fatal("Error:", err) + } + + policy = NewPostPolicy() + + // Remove all objects and buckets + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests copy object +func TestCopyObject(t *testing.T) { + if testing.Short() { + t.Skip("Skipping functional tests for short runs") + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object + c, err := NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Make a new bucket in 'us-east-1' (destination bucket). + err = c.MakeBucket(bucketName+"-copy", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName+"-copy") + } + + // Generate data more than 32K + buf := bytes.Repeat([]byte("5"), rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match want %v, got %v", + len(buf), n) + } + + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + // Check the various fields of source object against destination object. + objInfo, err := r.Stat() + if err != nil { + t.Fatal("Error:", err) + } + + // Copy Source + src := NewSourceInfo(bucketName, objectName, nil) + + // Set copy conditions. + + // All invalid conditions first. + err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) + if err == nil { + t.Fatal("Error:", err) + } + err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) + if err == nil { + t.Fatal("Error:", err) + } + err = src.SetMatchETagCond("") + if err == nil { + t.Fatal("Error:", err) + } + err = src.SetMatchETagExceptCond("") + if err == nil { + t.Fatal("Error:", err) + } + + err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatal("Error:", err) + } + err = src.SetMatchETagCond(objInfo.ETag) + if err != nil { + t.Fatal("Error:", err) + } + + dst, err := NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) + if err != nil { + t.Fatal(err) + } + + // Perform the Copy + err = c.CopyObject(dst, src) + if err != nil { + t.Fatal("Error:", err, bucketName+"-copy", objectName+"-copy") + } + + // Source object + reader, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + // Destination object + readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy") + if err != nil { + t.Fatal("Error:", err) + } + // Check the various fields of source object against destination object. + objInfo, err = reader.Stat() + if err != nil { + t.Fatal("Error:", err) + } + objInfoCopy, err := readerCopy.Stat() + if err != nil { + t.Fatal("Error:", err) + } + if objInfo.Size != objInfoCopy.Size { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", + objInfo.Size, objInfoCopy.Size) + } + + // CopyObject again but with wrong conditions + src = NewSourceInfo(bucketName, objectName, nil) + err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatal("Error:", err) + } + err = src.SetMatchETagExceptCond(objInfo.ETag) + if err != nil { + t.Fatal("Error:", err) + } + + // Perform the Copy which should fail + err = c.CopyObject(dst, src) + if err == nil { + t.Fatal("Error:", err, bucketName+"-copy", objectName+"-copy should fail") + } + + // Remove all objects and buckets + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + + err = c.RemoveObject(bucketName+"-copy", objectName+"-copy") + if err != nil { + t.Fatal("Error:", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + + err = c.RemoveBucket(bucketName + "-copy") + if err != nil { + t.Fatal("Error:", err) + } +} + +// TestEncryptionPutGet tests client side encryption +func TestEncryptionPutGet(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate a symmetric key + symKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) + + // Generate an assymmetric key from predefine public and private certificates + privateKey, err := hex.DecodeString( + "30820277020100300d06092a864886f70d0101010500048202613082025d" + + "0201000281810087b42ea73243a3576dc4c0b6fa245d339582dfdbddc20c" + + "bb8ab666385034d997210c54ba79275c51162a1221c3fb1a4c7c61131ca6" + + "5563b319d83474ef5e803fbfa7e52b889e1893b02586b724250de7ac6351" + + "cc0b7c638c980acec0a07020a78eed7eaa471eca4b92071394e061346c06" + + "15ccce2f465dee2080a89e43f29b5702030100010281801dd5770c3af8b3" + + "c85cd18cacad81a11bde1acfac3eac92b00866e142301fee565365aa9af4" + + "57baebf8bb7711054d071319a51dd6869aef3848ce477a0dc5f0dbc0c336" + + "5814b24c820491ae2bb3c707229a654427e03307fec683e6b27856688f08" + + "bdaa88054c5eeeb773793ff7543ee0fb0e2ad716856f2777f809ef7e6fa4" + + "41024100ca6b1edf89e8a8f93cce4b98c76c6990a09eb0d32ad9d3d04fbf" + + "0b026fa935c44f0a1c05dd96df192143b7bda8b110ec8ace28927181fd8c" + + "d2f17330b9b63535024100aba0260afb41489451baaeba423bee39bcbd1e" + + "f63dd44ee2d466d2453e683bf46d019a8baead3a2c7fca987988eb4d565e" + + "27d6be34605953f5034e4faeec9bdb0241009db2cb00b8be8c36710aff96" + + "6d77a6dec86419baca9d9e09a2b761ea69f7d82db2ae5b9aae4246599bb2" + + "d849684d5ab40e8802cfe4a2b358ad56f2b939561d2902404e0ead9ecafd" + + "bb33f22414fa13cbcc22a86bdf9c212ce1a01af894e3f76952f36d6c904c" + + "bd6a7e0de52550c9ddf31f1e8bfe5495f79e66a25fca5c20b3af5b870241" + + "0083456232aa58a8c45e5b110494599bda8dbe6a094683a0539ddd24e19d" + + "47684263bbe285ad953d725942d670b8f290d50c0bca3d1dc9688569f1d5" + + "9945cb5c7d") + + if err != nil { + t.Fatal(err) + } + + publicKey, err := hex.DecodeString("30819f300d06092a864886f70d010101050003818d003081890281810087" + + "b42ea73243a3576dc4c0b6fa245d339582dfdbddc20cbb8ab666385034d9" + + "97210c54ba79275c51162a1221c3fb1a4c7c61131ca65563b319d83474ef" + + "5e803fbfa7e52b889e1893b02586b724250de7ac6351cc0b7c638c980ace" + + "c0a07020a78eed7eaa471eca4b92071394e061346c0615ccce2f465dee20" + + "80a89e43f29b570203010001") + if err != nil { + t.Fatal(err) + } + + // Generate an asymmetric key + asymKey, err := encrypt.NewAsymmetricKey(privateKey, publicKey) + if err != nil { + t.Fatal(err) + } + + // Object custom metadata + customContentType := "custom/contenttype" + + testCases := []struct { + buf []byte + encKey encrypt.Key + }{ + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 0)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 15)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 16)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 17)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 31)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 32)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 33)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*2)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, + + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 0)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 16)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 32)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, + } + + for i, testCase := range testCases { + // Generate a random object name + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + + // Secured object + cbcMaterials, err := encrypt.NewCBCSecureMaterials(testCase.encKey) + if err != nil { + t.Fatal(err) + } + + // Put encrypted data + _, err = c.PutEncryptedObject(bucketName, objectName, bytes.NewReader(testCase.buf), cbcMaterials, map[string][]string{"Content-Type": {customContentType}}, nil) + if err != nil { + t.Fatalf("Test %d, error: %v %v %v", i+1, err, bucketName, objectName) + } + + // Read the data back + r, err := c.GetEncryptedObject(bucketName, objectName, cbcMaterials) + if err != nil { + t.Fatalf("Test %d, error: %v %v %v", i+1, err, bucketName, objectName) + } + defer r.Close() + + // Compare the sent object with the received one + recvBuffer := bytes.NewBuffer([]byte{}) + if _, err = io.Copy(recvBuffer, r); err != nil { + t.Fatalf("Test %d, error: %v", i+1, err) + } + if recvBuffer.Len() != len(testCase.buf) { + t.Fatalf("Test %d, error: number of bytes of received object does not match, want %v, got %v\n", + i+1, len(testCase.buf), recvBuffer.Len()) + } + if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { + t.Fatalf("Test %d, error: Encrypted sent is not equal to decrypted, want `%x`, go `%x`", i+1, testCase.buf, recvBuffer.Bytes()) + } + + // Remove test object + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatalf("Test %d, error: %v", i+1, err) + } + + } + + // Remove test bucket + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + +} + +func TestBucketNotification(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + if os.Getenv("NOTIFY_BUCKET") == "" || + os.Getenv("NOTIFY_SERVICE") == "" || + os.Getenv("NOTIFY_REGION") == "" || + os.Getenv("NOTIFY_ACCOUNTID") == "" || + os.Getenv("NOTIFY_RESOURCE") == "" { + t.Skip("skipping notification test if not configured") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + c, err := New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable to debug + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + bucketName := os.Getenv("NOTIFY_BUCKET") + + topicArn := NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE")) + queueArn := NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource") + + topicConfig := NewNotificationConfig(topicArn) + topicConfig.AddEvents(ObjectCreatedAll, ObjectRemovedAll) + topicConfig.AddFilterSuffix("jpg") + + queueConfig := NewNotificationConfig(queueArn) + queueConfig.AddEvents(ObjectCreatedAll) + queueConfig.AddFilterPrefix("photos/") + + bNotification := BucketNotification{} + bNotification.AddTopic(topicConfig) + + // Add the same topicConfig again, should have no effect + // because it is duplicated + bNotification.AddTopic(topicConfig) + if len(bNotification.TopicConfigs) != 1 { + t.Fatal("Error: duplicated entry added") + } + + // Add and remove a queue config + bNotification.AddQueue(queueConfig) + bNotification.RemoveQueueByArn(queueArn) + + err = c.SetBucketNotification(bucketName, bNotification) + if err != nil { + t.Fatal("Error: ", err) + } + + bNotification, err = c.GetBucketNotification(bucketName) + if err != nil { + t.Fatal("Error: ", err) + } + + if len(bNotification.TopicConfigs) != 1 { + t.Fatal("Error: Topic config is empty") + } + + if bNotification.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" { + t.Fatal("Error: cannot get the suffix") + } + + err = c.RemoveAllBucketNotification(bucketName) + if err != nil { + t.Fatal("Error: cannot delete bucket notification") + } +} + +// Tests comprehensive list of all methods. +func TestFunctional(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + c, err := New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable to debug + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate a random file name. + fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + file, err := os.Create(fileName) + if err != nil { + t.Fatal("Error:", err) + } + for i := 0; i < 3; i++ { + buf := make([]byte, rand.Intn(1<<19)) + _, err = file.Write(buf) + if err != nil { + t.Fatal("Error:", err) + } + } + file.Close() + + // Verify if bucket exits and you have access. + var exists bool + exists, err = c.BucketExists(bucketName) + if err != nil { + t.Fatal("Error:", err, bucketName) + } + if !exists { + t.Fatal("Error: could not find ", bucketName) + } + + // Asserting the default bucket policy. + policyAccess, err := c.GetBucketPolicy(bucketName, "") + if err != nil { + t.Fatal("Error:", err) + } + if policyAccess != "none" { + t.Fatalf("Default bucket policy incorrect") + } + // Set the bucket policy to 'public readonly'. + err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadOnly) + if err != nil { + t.Fatal("Error:", err) + } + // should return policy `readonly`. + policyAccess, err = c.GetBucketPolicy(bucketName, "") + if err != nil { + t.Fatal("Error:", err) + } + if policyAccess != "readonly" { + t.Fatalf("Expected bucket policy to be readonly") + } + + // Make the bucket 'public writeonly'. + err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyWriteOnly) + if err != nil { + t.Fatal("Error:", err) + } + // should return policy `writeonly`. + policyAccess, err = c.GetBucketPolicy(bucketName, "") + if err != nil { + t.Fatal("Error:", err) + } + if policyAccess != "writeonly" { + t.Fatalf("Expected bucket policy to be writeonly") + } + // Make the bucket 'public read/write'. + err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadWrite) + if err != nil { + t.Fatal("Error:", err) + } + // should return policy `readwrite`. + policyAccess, err = c.GetBucketPolicy(bucketName, "") + if err != nil { + t.Fatal("Error:", err) + } + if policyAccess != "readwrite" { + t.Fatalf("Expected bucket policy to be readwrite") + } + // List all buckets. + buckets, err := c.ListBuckets() + if len(buckets) == 0 { + t.Fatal("Error: list buckets cannot be empty", buckets) + } + if err != nil { + t.Fatal("Error:", err) + } + + // Verify if previously created bucket is listed in list buckets. + bucketFound := false + for _, bucket := range buckets { + if bucket.Name == bucketName { + bucketFound = true + } + } + + // If bucket not found error out. + if !bucketFound { + t.Fatal("Error: bucket ", bucketName, "not found") + } + + objectName := bucketName + "unique" + + // Generate data + buf := bytes.Repeat([]byte("f"), 1<<19) + + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "") + if err != nil { + t.Fatal("Error: ", err) + } + if n != int64(len(buf)) { + t.Fatal("Error: bad length ", n, len(buf)) + } + + n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName+"-nolength") + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Instantiate a done channel to close all listing. + doneCh := make(chan struct{}) + defer close(doneCh) + + objFound := false + isRecursive := true // Recursive is true. + for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + t.Fatal("Error: object " + objectName + " not found.") + } + + objFound = false + isRecursive = true // Recursive is true. + for obj := range c.ListObjectsV2(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + t.Fatal("Error: object " + objectName + " not found.") + } + + incompObjNotFound := true + for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { + if objIncompl.Key != "" { + incompObjNotFound = false + break + } + } + if !incompObjNotFound { + t.Fatal("Error: unexpected dangling incomplete upload found.") + } + + newReader, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + newReadBytes, err := ioutil.ReadAll(newReader) + if err != nil { + t.Fatal("Error: ", err) + } + + if !bytes.Equal(newReadBytes, buf) { + t.Fatal("Error: bytes mismatch.") + } + + err = c.FGetObject(bucketName, objectName, fileName+"-f") + if err != nil { + t.Fatal("Error: ", err) + } + + // Generate presigned GET object url. + presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) + if err != nil { + t.Fatal("Error: ", err) + } + + // Verify if presigned url works. + resp, err := http.Get(presignedGetURL.String()) + if err != nil { + t.Fatal("Error: ", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatal("Error: ", resp.Status) + } + newPresignedBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal("Error: ", err) + } + if !bytes.Equal(newPresignedBytes, buf) { + t.Fatal("Error: bytes mismatch.") + } + + // Set request parameters. + reqParams := make(url.Values) + reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") + presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) + if err != nil { + t.Fatal("Error: ", err) + } + // Verify if presigned url works. + resp, err = http.Get(presignedGetURL.String()) + if err != nil { + t.Fatal("Error: ", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatal("Error: ", resp.Status) + } + newPresignedBytes, err = ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal("Error: ", err) + } + if !bytes.Equal(newPresignedBytes, buf) { + t.Fatal("Error: bytes mismatch for presigned GET URL.") + } + if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { + t.Fatalf("Error: wrong Content-Disposition received %s", resp.Header.Get("Content-Disposition")) + } + + presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) + if err != nil { + t.Fatal("Error: ", err) + } + + buf = bytes.Repeat([]byte("g"), 1<<19) + + req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) + if err != nil { + t.Fatal("Error: ", err) + } + httpClient := &http.Client{ + // Setting a sensible time out of 30secs to wait for response + // headers. Request is pro-actively cancelled after 30secs + // with no response. + Timeout: 30 * time.Second, + Transport: http.DefaultTransport, + } + resp, err = httpClient.Do(req) + if err != nil { + t.Fatal("Error: ", err) + } + + newReader, err = c.GetObject(bucketName, objectName+"-presigned") + if err != nil { + t.Fatal("Error: ", err) + } + + newReadBytes, err = ioutil.ReadAll(newReader) + if err != nil { + t.Fatal("Error: ", err) + } + + if !bytes.Equal(newReadBytes, buf) { + t.Fatal("Error: bytes mismatch.") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-f") + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-nolength") + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-presigned") + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + err = c.RemoveBucket(bucketName) + if err == nil { + t.Fatal("Error:") + } + if err.Error() != "The specified bucket does not exist" { + t.Fatal("Error: ", err) + } + if err = os.Remove(fileName); err != nil { + t.Fatal("Error: ", err) + } + if err = os.Remove(fileName + "-f"); err != nil { + t.Fatal("Error: ", err) + } +} + +// Test for validating GetObject Reader* methods functioning when the +// object is modified in the object store. +func TestGetObjectObjectModified(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Instantiate new minio client object. + c, err := NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Make a new bucket. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + defer c.RemoveBucket(bucketName) + + // Upload an object. + objectName := "myobject" + content := "helloworld" + _, err = c.PutObject(bucketName, objectName, strings.NewReader(content), "application/text") + if err != nil { + t.Fatalf("Failed to upload %s/%s: %v", bucketName, objectName, err) + } + + defer c.RemoveObject(bucketName, objectName) + + reader, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatalf("Failed to get object %s/%s: %v", bucketName, objectName, err) + } + defer reader.Close() + + // Read a few bytes of the object. + b := make([]byte, 5) + n, err := reader.ReadAt(b, 0) + if err != nil { + t.Fatalf("Failed to read object %s/%s at an offset: %v", bucketName, objectName, err) + } + + // Upload different contents to the same object while object is being read. + newContent := "goodbyeworld" + _, err = c.PutObject(bucketName, objectName, strings.NewReader(newContent), "application/text") + if err != nil { + t.Fatalf("Failed to upload %s/%s: %v", bucketName, objectName, err) + } + + // Confirm that a Stat() call in between doesn't change the Object's cached etag. + _, err = reader.Stat() + if err.Error() != s3ErrorResponseMap["PreconditionFailed"] { + t.Errorf("Expected Stat to fail with error %s but received %s", s3ErrorResponseMap["PreconditionFailed"], err.Error()) + } + + // Read again only to find object contents have been modified since last read. + _, err = reader.ReadAt(b, int64(n)) + if err.Error() != s3ErrorResponseMap["PreconditionFailed"] { + t.Errorf("Expected ReadAt to fail with error %s but received %s", s3ErrorResponseMap["PreconditionFailed"], err.Error()) + } +} + +// Test validates putObject to upload a file seeked at a given offset. +func TestPutObjectUploadSeekedObject(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Instantiate new minio client object. + c, err := NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Make a new bucket. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + defer c.RemoveBucket(bucketName) + + tempfile, err := ioutil.TempFile("", "minio-go-upload-test-") + if err != nil { + t.Fatal("Error:", err) + } + + var length = 120000 + data := bytes.Repeat([]byte("1"), length) + + if _, err = tempfile.Write(data); err != nil { + t.Fatal("Error:", err) + } + + objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) + + offset := length / 2 + if _, err := tempfile.Seek(int64(offset), 0); err != nil { + t.Fatal("Error:", err) + } + + n, err := c.PutObject(bucketName, objectName, tempfile, "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(length-offset) { + t.Fatalf("Invalid length returned, want %v, got %v", int64(length-offset), n) + } + tempfile.Close() + if err = os.Remove(tempfile.Name()); err != nil { + t.Fatal("Error:", err) + } + + length = int(n) + + obj, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + + n, err = obj.Seek(int64(offset), 0) + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(offset) { + t.Fatalf("Invalid offset returned, want %v, got %v", int64(offset), n) + } + + n, err = c.PutObject(bucketName, objectName+"getobject", obj, "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(length-offset) { + t.Fatalf("Invalid length returned, want %v, got %v", int64(length-offset), n) + } + + if err = c.RemoveObject(bucketName, objectName); err != nil { + t.Fatal("Error:", err) + } + + if err = c.RemoveObject(bucketName, objectName+"getobject"); err != nil { + t.Fatal("Error:", err) + } +} + +// Test expected error cases +func TestComposeObjectErrorCases(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Instantiate new minio client object + c, err := NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + testComposeObjectErrorCases(c, t) +} + +// Test concatenating 10K objects +func TestCompose10KSources(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Instantiate new minio client object + c, err := NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + testComposeMultipleSources(c, t) +} + +// Test encrypted copy object +func TestEncryptedCopyObject(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Instantiate new minio client object + c, err := NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // c.TraceOn(os.Stderr) + testEncryptedCopyObject(c, t) +} + +func TestUserMetadataCopying(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Instantiate new minio client object + c, err := NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // c.TraceOn(os.Stderr) + testUserMetadataCopying(c, t) +} diff --git a/vendor/src/github.com/minio/minio-go/bucket-cache.go b/vendor/src/github.com/minio/minio-go/bucket-cache.go index 748fd01ee..6d2a40f78 100644 --- a/vendor/src/github.com/minio/minio-go/bucket-cache.go +++ b/vendor/src/github.com/minio/minio-go/bucket-cache.go @@ -213,24 +213,20 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro signerType = credentials.SignatureAnonymous } - if signerType.IsAnonymous() { - return req, nil - } - - if signerType.IsV2() { - req = s3signer.SignV2(*req, accessKeyID, secretAccessKey) - return req, nil - } - // Set sha256 sum for signature calculation only with signature version '4'. - var contentSha256 string - if c.secure { - contentSha256 = unsignedPayload - } else { - contentSha256 = hex.EncodeToString(sum256([]byte{})) + switch { + case signerType.IsV4(): + var contentSha256 string + if c.secure { + contentSha256 = unsignedPayload + } else { + contentSha256 = hex.EncodeToString(sum256([]byte{})) + } + req.Header.Set("X-Amz-Content-Sha256", contentSha256) + req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") + case signerType.IsV2(): + req = s3signer.SignV2(*req, accessKeyID, secretAccessKey) } - req.Header.Set("X-Amz-Content-Sha256", contentSha256) - req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") return req, nil } diff --git a/vendor/src/github.com/minio/minio-go/core_test.go b/vendor/src/github.com/minio/minio-go/core_test.go index 032e86ec5..81e1cd5bf 100644 --- a/vendor/src/github.com/minio/minio-go/core_test.go +++ b/vendor/src/github.com/minio/minio-go/core_test.go @@ -18,15 +18,14 @@ package minio import ( "bytes" + "crypto/md5" + "io" - "log" + "math/rand" "os" "reflect" "testing" "time" - - "crypto/md5" - "math/rand" ) const ( @@ -36,33 +35,6 @@ const ( enableSecurity = "ENABLE_HTTPS" ) -// Minimum part size -const MinPartSize = 1024 * 1024 * 64 -const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" -const ( - letterIdxBits = 6 // 6 bits to represent a letter index - letterIdxMask = 1<= 0; { - if remain == 0 { - cache, remain = src.Int63(), letterIdxMax - } - if idx := int(cache & letterIdxMask); idx < len(letterBytes) { - b[i] = letterBytes[idx] - i-- - } - cache >>= letterIdxBits - remain-- - } - return prefix + string(b[0:30-len(prefix)]) -} - // Tests for Core GetObject() function. func TestGetObjectCore(t *testing.T) { if testing.Short() { @@ -237,76 +209,6 @@ func TestGetObjectCore(t *testing.T) { } } -// Tests GetObject to return Content-Encoding properly set -// and overrides any auto decoding. -func TestGetObjectContentEncoding(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio core client object. - c, err := NewCore( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity)), - ) - if err != nil { - t.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - buf := bytes.Repeat([]byte("3"), rand.Intn(1<<20)+32*1024) - m := make(map[string][]string) - m["Content-Encoding"] = []string{"gzip"} - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.Client.PutObjectWithMetadata(bucketName, objectName, bytes.NewReader(buf), m, nil) - if err != nil { - t.Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - reqHeaders := NewGetReqHeaders() - rwc, objInfo, err := c.GetObject(bucketName, objectName, reqHeaders) - if err != nil { - t.Fatalf("Error: %v", err) - } - rwc.Close() - if objInfo.Size <= 0 { - t.Fatalf("Unexpected size of the object %v, expected %v", objInfo.Size, n) - } - value, ok := objInfo.Metadata["Content-Encoding"] - if !ok { - t.Fatalf("Expected Content-Encoding metadata to be set.") - } - if value[0] != "gzip" { - t.Fatalf("Unexpected content-encoding found, want gzip, got %v", value) - } -} - // Tests get bucket policy core API. func TestGetBucketPolicy(t *testing.T) { if testing.Short() { @@ -471,48 +373,3 @@ func TestCorePutObject(t *testing.T) { t.Fatal("Error:", err) } } - -func TestCoreGetObjectMetadata(t *testing.T) { - if testing.Short() { - t.Skip("skipping functional tests for the short runs") - } - - core, err := NewCore( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableSecurity))) - if err != nil { - log.Fatalln(err) - } - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = core.MakeBucket(bucketName, "us-east-1") - if err != nil { - t.Fatal("Error:", err, bucketName) - } - - metadata := map[string][]string{ - "X-Amz-Meta-Key-1": {"Val-1"}, - } - - _, err = core.PutObject(bucketName, "my-objectname", 5, - bytes.NewReader([]byte("hello")), nil, nil, metadata) - if err != nil { - log.Fatalln(err) - } - - reader, objInfo, err := core.GetObject(bucketName, "my-objectname", - RequestHeaders{}) - if err != nil { - log.Fatalln(err) - } - defer reader.Close() - - if objInfo.Metadata.Get("X-Amz-Meta-Key-1") != "Val-1" { - log.Fatalln("Expected metadata to be available but wasn't") - } -} diff --git a/vendor/src/github.com/minio/minio-go/docs/API.md b/vendor/src/github.com/minio/minio-go/docs/API.md index 902a975d9..e0d0a11e6 100644 --- a/vendor/src/github.com/minio/minio-go/docs/API.md +++ b/vendor/src/github.com/minio/minio-go/docs/API.md @@ -531,13 +531,9 @@ __Example__ src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil) // Destination object -dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) -if err != nil { - fmt.Println(err) - return -} +dst := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) -// Copy object call +/ Copy object call err = s3Client.CopyObject(dst, src) if err != nil { fmt.Println(err) @@ -566,13 +562,9 @@ src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 23, 0, 0, 0, 0, time.UTC) src.SetRange(0, 1024*1024-1) // Destination object -dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) -if err != nil { - fmt.Println(err) - return -} +dst := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) -// Copy object call +/ Copy object call err = s3Client.CopyObject(dst, src) if err != nil { fmt.Println(err) @@ -656,7 +648,7 @@ src := NewSourceInfo("bucket", "object", decKey) ``` -### NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo, userMeta map[string]string) (DestinationInfo, error) +### NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo, userMeta map[string]string) DestinationInfo Construct a `DestinationInfo` object that can be used as the destination object for server-side copying operations like `CopyObject` and `ComposeObject`. @@ -673,11 +665,11 @@ __Example__ ``` go // No encryption parameter. -dst, err := NewDestinationInfo("bucket", "object", nil, nil) +src := NewDestinationInfo("bucket", "object", nil, nil) // With encryption parameter. encKey := NewSSEKey([]byte{1,2,3}, "") -dst, err := NewDecryptionInfo("bucket", "object", encKey, nil) +src := NewDecryptionInfo("bucket", "object", encKey, nil) ``` diff --git a/vendor/src/github.com/minio/minio-go/examples/s3/composeobject.go b/vendor/src/github.com/minio/minio-go/examples/s3/composeobject.go index 8aec6c158..555d98bc3 100644 --- a/vendor/src/github.com/minio/minio-go/examples/s3/composeobject.go +++ b/vendor/src/github.com/minio/minio-go/examples/s3/composeobject.go @@ -47,14 +47,14 @@ func main() { // Source objects to concatenate. We also specify decryption // key for each - src1 := minio.NewSourceInfo("bucket1", "object1", &decKey) - src1.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") + src1 := minio.NewSourceInfo("bucket1", "object1", decKey) + src1.SetMatchETag("31624deb84149d2f8ef9c385918b653a") - src2 := minio.NewSourceInfo("bucket2", "object2", &decKey) - src2.SetMatchETagCond("f8ef9c385918b653a31624deb84149d2") + src2 := minio.NewSourceInfo("bucket2", "object2", decKey) + src2.SetMatchETag("f8ef9c385918b653a31624deb84149d2") - src3 := minio.NewSourceInfo("bucket3", "object3", &decKey) - src3.SetMatchETagCond("5918b653a31624deb84149d2f8ef9c38") + src3 := minio.NewSourceInfo("bucket3", "object3", decKey) + src3.SetMatchETag("5918b653a31624deb84149d2f8ef9c38") // Create slice of sources. srcs := []minio.SourceInfo{src1, src2, src3} @@ -63,14 +63,11 @@ func main() { encKey := minio.NewSSEInfo([]byte{8, 9, 0}, "") // Create destination info - dst, err := minio.NewDestinationInfo("bucket", "object", &encKey, nil) - if err != nil { - log.Fatalln(err) - } - + dst := minio.NewDestinationInfo("bucket", "object", encKey) err = s3Client.ComposeObject(dst, srcs) if err != nil { - log.Fatalln(err) + log.Println(err) + return } log.Println("Composed object successfully.") diff --git a/vendor/src/github.com/minio/minio-go/examples/s3/copyobject.go b/vendor/src/github.com/minio/minio-go/examples/s3/copyobject.go index c1d92d73a..0de865555 100644 --- a/vendor/src/github.com/minio/minio-go/examples/s3/copyobject.go +++ b/vendor/src/github.com/minio/minio-go/examples/s3/copyobject.go @@ -60,10 +60,7 @@ func main() { // src.SetMatchETagExceptCond("31624deb84149d2f8ef9c385918b653a") // Destination object - dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) - if err != nil { - log.Fatalln(err) - } + dst := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil) // Initiate copy object. err = s3Client.CopyObject(dst, src) diff --git a/vendor/src/github.com/minio/minio-go/examples/s3/putobject-progress.go b/vendor/src/github.com/minio/minio-go/examples/s3/putobject-progress.go index 26e77b9e6..1179fd787 100644 --- a/vendor/src/github.com/minio/minio-go/examples/s3/putobject-progress.go +++ b/vendor/src/github.com/minio/minio-go/examples/s3/putobject-progress.go @@ -55,9 +55,7 @@ func main() { progress := pb.New64(objectInfo.Size) progress.Start() - n, err := s3Client.PutObjectWithProgress("my-bucketname", "my-objectname-progress", reader, map[string][]string{ - "Content-Type": []string{"application/octet-stream"}, - }, progress) + n, err := s3Client.PutObjectWithProgress("my-bucketname", "my-objectname-progress", reader, "application/octet-stream", progress) if err != nil { log.Fatalln(err) } diff --git a/vendor/src/github.com/minio/minio-go/examples/s3/putobject-s3-accelerate.go b/vendor/src/github.com/minio/minio-go/examples/s3/putobject-s3-accelerate.go index a26415c7a..e47976f2e 100644 --- a/vendor/src/github.com/minio/minio-go/examples/s3/putobject-s3-accelerate.go +++ b/vendor/src/github.com/minio/minio-go/examples/s3/putobject-s3-accelerate.go @@ -40,7 +40,7 @@ func main() { } // Enable S3 transfer accelerate endpoint. - s3Client.SetS3TransferAccelerate("s3-accelerate.amazonaws.com") + s3Client.S3TransferAccelerate("s3-accelerate.amazonaws.com") object, err := os.Open("my-testfile") if err != nil { diff --git a/vendor/src/github.com/minio/minio-go/functional_tests.go b/vendor/src/github.com/minio/minio-go/functional_tests.go deleted file mode 100644 index 991e2d3c0..000000000 --- a/vendor/src/github.com/minio/minio-go/functional_tests.go +++ /dev/null @@ -1,4058 +0,0 @@ -// +build ignore - -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "bytes" - "encoding/hex" - "errors" - "fmt" - "io" - "io/ioutil" - "math/rand" - "net/http" - "net/url" - "os" - "path" - "reflect" - "runtime" - "strconv" - "strings" - "time" - - minio "github.com/minio/minio-go" - log "github.com/sirupsen/logrus" - logrus "github.com/sirupsen/logrus" - - "github.com/minio/minio-go/pkg/encrypt" - "github.com/minio/minio-go/pkg/policy" -) - -// MinPartSize ... Minimum part size -const MinPartSize = 1024 * 1024 * 64 -const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" -const ( - letterIdxBits = 6 // 6 bits to represent a letter index - letterIdxMask = 1<= 0; { - if remain == 0 { - cache, remain = src.Int63(), letterIdxMax - } - if idx := int(cache & letterIdxMask); idx < len(letterBytes) { - b[i] = letterBytes[idx] - i-- - } - cache >>= letterIdxBits - remain-- - } - return prefix + string(b[0:30-len(prefix)]) -} - -func isQuickMode() bool { - return os.Getenv("MODE") == "quick" -} - -// Tests bucket re-create errors. -func testMakeBucketError() { - logger().Info() - - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - logger().Info("skipping region functional tests for non s3 runs") - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatalf("Error: %s", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, "eu-central-1"); err != nil { - logger().Fatal("Error:", err, bucketName) - } - if err = c.MakeBucket(bucketName, "eu-central-1"); err == nil { - logger().Fatal("Error: make bucket should should fail for", bucketName) - } - // Verify valid error response from server. - if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && - minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { - logger().Fatal("Error: Invalid error returned by server", err) - } - if err = c.RemoveBucket(bucketName); err != nil { - logger().Fatal("Error:", err, bucketName) - } -} - -// Tests various bucket supported formats. -func testMakeBucketRegions() { - logger().Info() - - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - logger().Info("skipping region functional tests for non s3 runs") - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, "eu-central-1"); err != nil { - logger().Fatal("Error:", err, bucketName) - } - - if err = c.RemoveBucket(bucketName); err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Make a new bucket with '.' in its name, in 'us-west-2'. This - // request is internally staged into a path style instead of - // virtual host style. - if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { - logger().Fatal("Error:", err, bucketName+".withperiod") - } - - // Remove the newly created bucket. - if err = c.RemoveBucket(bucketName + ".withperiod"); err != nil { - logger().Fatal("Error:", err, bucketName+".withperiod") - } -} - -// Test PutObject using a large data to trigger multipart readat -func testPutObjectReadAt() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Generate data using 4 parts so that all 3 'workers' are utilized and a part is leftover. - // Use different data for each part for multipart tests to ensure part order at the end. - var buf = getDataBuffer("datafile-65-MB", MinPartSize) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - // Object content type - objectContentType := "binary/octet-stream" - - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), objectContentType) - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - if st.ContentType != objectContentType { - logger().Fatalf("Error: Content types don't match, expected: %+v, found: %+v\n", objectContentType, st.ContentType) - } - if err := r.Close(); err != nil { - logger().Fatal("Error:", err) - } - if err := r.Close(); err == nil { - logger().Fatal("Error: object is already closed, should return error") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } -} - -// Test PutObject using a large data to trigger multipart readat -func testPutObjectWithMetadata() { - logger().Info() - if isQuickMode() { - logger().Info("skipping functional tests for short runs") - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Generate data using 2 parts - // Use different data in each part for multipart tests to ensure part order at the end. - var buf = getDataBuffer("datafile-65-MB", MinPartSize) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - - // Object custom metadata - customContentType := "custom/contenttype" - - n, err := c.PutObjectWithMetadata(bucketName, objectName, bytes.NewReader(buf), map[string][]string{ - "Content-Type": {customContentType}, - }, nil) - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - if st.ContentType != customContentType { - logger().Fatalf("Error: Expected and found content types do not match, want %v, got %v\n", - customContentType, st.ContentType) - } - if err := r.Close(); err != nil { - logger().Fatal("Error:", err) - } - if err := r.Close(); err == nil { - logger().Fatal("Error: object is already closed, should return error") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } -} - -// Test put object with streaming signature. -func testPutObjectStreaming() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), - "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Upload an object. - sizes := []int64{0, 64*1024 - 1, 64 * 1024} - objectName := "test-object" - for i, size := range sizes { - data := bytes.Repeat([]byte("a"), int(size)) - n, err := c.PutObjectStreaming(bucketName, objectName, bytes.NewReader(data)) - if err != nil { - logger().Fatalf("Test %d Error: %v %s %s", i+1, err, bucketName, objectName) - } - - if n != size { - log.Error(fmt.Errorf("Test %d Expected upload object size %d but got %d", i+1, size, n)) - } - } - - // Remove the object. - err = c.RemoveObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err) - } - - // Remove the bucket. - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } -} - -// Test listing partially uploaded objects. -func testListPartiallyUploaded() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - r := bytes.NewReader(bytes.Repeat([]byte("0"), MinPartSize*2)) - - reader, writer := io.Pipe() - go func() { - i := 0 - for i < 25 { - _, cerr := io.CopyN(writer, r, (MinPartSize*2)/25) - if cerr != nil { - logger().Fatal("Error:", cerr, bucketName) - } - i++ - r.Seek(0, 0) - } - writer.CloseWithError(errors.New("proactively closed to be verified later")) - }() - - objectName := bucketName + "-resumable" - _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") - if err == nil { - logger().Fatal("Error: PutObject should fail.") - } - if !strings.Contains(err.Error(), "proactively closed to be verified later") { - logger().Fatal("Error:", err) - } - - doneCh := make(chan struct{}) - defer close(doneCh) - isRecursive := true - multiPartObjectCh := c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) - for multiPartObject := range multiPartObjectCh { - if multiPartObject.Err != nil { - logger().Fatalf("Error: Error when listing incomplete upload") - } - } - - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } -} - -// Test get object seeker from the end, using whence set to '2'. -func testGetObjectSeekEnd() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - - pos, err := r.Seek(-100, 2) - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - if pos != st.Size-100 { - logger().Fatalf("Expected %d, got %d instead", pos, st.Size-100) - } - buf2 := make([]byte, 100) - m, err := io.ReadFull(r, buf2) - if err != nil { - logger().Fatal("Error: reading through io.ReadFull", err, bucketName, objectName) - } - if m != len(buf2) { - logger().Fatalf("Expected %d bytes, got %d", len(buf2), m) - } - hexBuf1 := fmt.Sprintf("%02x", buf[len(buf)-100:]) - hexBuf2 := fmt.Sprintf("%02x", buf2[:m]) - if hexBuf1 != hexBuf2 { - logger().Fatalf("Expected %s, got %s instead", hexBuf1, hexBuf2) - } - pos, err = r.Seek(-100, 2) - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - if pos != st.Size-100 { - logger().Fatalf("Expected %d, got %d instead", pos, st.Size-100) - } - if err = r.Close(); err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } -} - -// Test get object reader to not throw error on being closed twice. -func testGetObjectClosedTwice() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - if err := r.Close(); err != nil { - logger().Fatal("Error:", err) - } - if err := r.Close(); err == nil { - logger().Fatal("Error: object is already closed, should return error") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } -} - -// Test removing multiple objects with Remove API -func testRemoveMultipleObjects() { - logger().Info() - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - - if err != nil { - logger().Fatal("Error:", err) - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) - - // Multi remove of 1100 objects - nrObjects := 1100 - - objectsCh := make(chan string) - - go func() { - defer close(objectsCh) - // Upload objects and send them to objectsCh - for i := 0; i < nrObjects; i++ { - objectName := "sample" + strconv.Itoa(i) + ".txt" - _, err = c.PutObject(bucketName, objectName, r, "application/octet-stream") - if err != nil { - log.Error("Error: PutObject shouldn't fail.", err) - continue - } - objectsCh <- objectName - } - }() - - // Call RemoveObjects API - errorCh := c.RemoveObjects(bucketName, objectsCh) - - // Check if errorCh doesn't receive any error - select { - case r, more := <-errorCh: - if more { - logger().Fatalf("Unexpected error, objName(%v) err(%v)", r.ObjectName, r.Err) - } - } - - // Clean the bucket created by the test - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } -} - -// Tests removing partially uploaded objects. -func testRemovePartiallyUploaded() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - r := bytes.NewReader(bytes.Repeat([]byte("a"), 128*1024)) - - reader, writer := io.Pipe() - go func() { - i := 0 - for i < 25 { - _, cerr := io.CopyN(writer, r, 128*1024) - if cerr != nil { - logger().Fatal("Error:", cerr, bucketName) - } - i++ - r.Seek(0, 0) - } - writer.CloseWithError(errors.New("proactively closed to be verified later")) - }() - - objectName := bucketName + "-resumable" - _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") - if err == nil { - logger().Fatal("Error: PutObject should fail.") - } - if !strings.Contains(err.Error(), "proactively closed to be verified later") { - logger().Fatal("Error:", err) - } - err = c.RemoveIncompleteUpload(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } -} - -// Tests FPutObject of a big file to trigger multipart -func testFPutObjectMultipart() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. - var fileName = getFilePath("datafile-65-MB") - if os.Getenv("MINT_DATA_DIR") == "" { - // Make a temp file with minPartSize bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") - if err != nil { - logger().Fatal("Error:", err) - } - - // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. - var buffer = bytes.Repeat([]byte(string('a')), MinPartSize) - if _, err := file.Write(buffer); err != nil { - logger().Fatal("Error:", err) - } - // Close the file pro-actively for windows. - err = file.Close() - if err != nil { - logger().Fatal("Error:", err) - } - fileName = file.Name() - } - totalSize := MinPartSize * 1 - // Set base object name - objectName := bucketName + "FPutObject" - objectContentType := "testapplication/octet-stream" - - // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err := c.FPutObject(bucketName, objectName+"-standard", fileName, objectContentType) - if err != nil { - logger().Fatal("Error:", err) - } - if n != int64(totalSize) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", totalSize, n) - } - - r, err := c.GetObject(bucketName, objectName+"-standard") - if err != nil { - logger().Fatalf("Unexpected error: %v\n", err) - } - objInfo, err := r.Stat() - if err != nil { - logger().Fatalf("Unexpected error: %v\n", err) - } - if objInfo.Size != int64(totalSize) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", totalSize, n) - } - if objInfo.ContentType != objectContentType { - logger().Fatalf("Error: Content types don't match, want %v, got %v\n", objectContentType, objInfo.ContentType) - } - - // Remove all objects and bucket and temp file - err = c.RemoveObject(bucketName, objectName+"-standard") - if err != nil { - logger().Fatal("Error: ", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } -} - -// Tests FPutObject hidden contentType setting -func testFPutObject() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Upload 3 parts worth of data to use all 3 of multiparts 'workers' and have an extra part. - // Use different data in part for multipart tests to check parts are uploaded in correct order. - var fName = getFilePath("datafile-65-MB") - if os.Getenv("MINT_DATA_DIR") == "" { - // Make a temp file with minPartSize bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") - if err != nil { - logger().Fatal("Error:", err) - } - - // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. - var buffer = bytes.Repeat([]byte(string('a')), MinPartSize) - if _, err = file.Write(buffer); err != nil { - logger().Fatal("Error:", err) - } - // Close the file pro-actively for windows. - err = file.Close() - if err != nil { - logger().Fatal("Error:", err) - } - fName = file.Name() - } - var totalSize = MinPartSize * 1 - - // Set base object name - objectName := bucketName + "FPutObject" - - // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err := c.FPutObject(bucketName, objectName+"-standard", fName, "application/octet-stream") - if err != nil { - logger().Fatal("Error:", err) - } - if n != int64(totalSize) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", totalSize, n) - } - - // Perform FPutObject with no contentType provided (Expecting application/octet-stream) - n, err = c.FPutObject(bucketName, objectName+"-Octet", fName, "") - if err != nil { - logger().Fatal("Error:", err) - } - if n != int64(totalSize) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", totalSize, n) - } - srcFile, err := os.Open(fName) - if err != nil { - logger().Fatal("Error:", err) - } - defer srcFile.Close() - // Add extension to temp file name - tmpFile, err := os.Create(fName + ".gtar") - if err != nil { - logger().Fatal("Error:", err) - } - defer tmpFile.Close() - _, err = io.Copy(tmpFile, srcFile) - if err != nil { - logger().Fatal("Error:", err) - } - - // Perform FPutObject with no contentType provided (Expecting application/x-gtar) - n, err = c.FPutObject(bucketName, objectName+"-GTar", fName+".gtar", "") - if err != nil { - logger().Fatal("Error:", err) - } - if n != int64(totalSize) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", totalSize, n) - } - - // Check headers - rStandard, err := c.StatObject(bucketName, objectName+"-standard") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName+"-standard") - } - if rStandard.ContentType != "application/octet-stream" { - logger().Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", - "application/octet-stream", rStandard.ContentType) - } - - rOctet, err := c.StatObject(bucketName, objectName+"-Octet") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName+"-Octet") - } - if rOctet.ContentType != "application/octet-stream" { - logger().Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", - "application/octet-stream", rStandard.ContentType) - } - - rGTar, err := c.StatObject(bucketName, objectName+"-GTar") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName+"-GTar") - } - if rGTar.ContentType != "application/x-gtar" { - logger().Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", - "application/x-gtar", rStandard.ContentType) - } - - // Remove all objects and bucket and temp file - err = c.RemoveObject(bucketName, objectName+"-standard") - if err != nil { - logger().Fatal("Error: ", err) - } - - err = c.RemoveObject(bucketName, objectName+"-Octet") - if err != nil { - logger().Fatal("Error: ", err) - } - - err = c.RemoveObject(bucketName, objectName+"-GTar") - if err != nil { - logger().Fatal("Error: ", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } - - err = os.Remove(fName + ".gtar") - if err != nil { - logger().Fatal("Error:", err) - } - -} - -// Tests get object ReaderSeeker interface methods. -func testGetObjectReadSeekFunctional() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) - - bufSize := len(buf) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(bufSize) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - defer func() { - err = c.RemoveObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } - }() - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(bufSize) { - logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - - // This following function helps us to compare data from the reader after seek - // with the data from the original buffer - cmpData := func(r io.Reader, start, end int) { - if end-start == 0 { - return - } - buffer := bytes.NewBuffer([]byte{}) - if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { - if err != io.EOF { - logger().Fatal("Error:", err) - } - } - if !bytes.Equal(buf[start:end], buffer.Bytes()) { - logger().Fatal("Error: Incorrect read bytes v/s original buffer.") - } - } - - // Generic seek error for errors other than io.EOF - seekErr := errors.New("seek error") - - testCases := []struct { - offset int64 - whence int - pos int64 - err error - shouldCmp bool - start int - end int - }{ - // Start from offset 0, fetch data and compare - {0, 0, 0, nil, true, 0, 0}, - // Start from offset 2048, fetch data and compare - {2048, 0, 2048, nil, true, 2048, bufSize}, - // Start from offset larger than possible - {int64(bufSize) + 1024, 0, 0, seekErr, false, 0, 0}, - // Move to offset 0 without comparing - {0, 0, 0, nil, false, 0, 0}, - // Move one step forward and compare - {1, 1, 1, nil, true, 1, bufSize}, - // Move larger than possible - {int64(bufSize), 1, 0, seekErr, false, 0, 0}, - // Provide negative offset with CUR_SEEK - {int64(-1), 1, 0, seekErr, false, 0, 0}, - // Test with whence SEEK_END and with positive offset - {1024, 2, int64(bufSize) - 1024, io.EOF, true, 0, 0}, - // Test with whence SEEK_END and with negative offset - {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, - // Test with whence SEEK_END and with large negative offset - {-int64(bufSize) * 2, 2, 0, seekErr, true, 0, 0}, - } - - for i, testCase := range testCases { - // Perform seek operation - n, err := r.Seek(testCase.offset, testCase.whence) - // We expect an error - if testCase.err == seekErr && err == nil { - logger().Fatalf("Test %d, unexpected err value: expected: %v, found: %v", i+1, testCase.err, err) - } - // We expect a specific error - if testCase.err != seekErr && testCase.err != err { - logger().Fatalf("Test %d, unexpected err value: expected: %v, found: %v", i+1, testCase.err, err) - } - // If we expect an error go to the next loop - if testCase.err != nil { - continue - } - // Check the returned seek pos - if n != testCase.pos { - logger().Fatalf("Test %d, error: number of bytes seeked does not match, want %v, got %v\n", i+1, - testCase.pos, n) - } - // Compare only if shouldCmp is activated - if testCase.shouldCmp { - cmpData(r, testCase.start, testCase.end) - } - } -} - -// Tests get object ReaderAt interface methods. -func testGetObjectReadAtFunctional() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - offset := int64(2048) - - // read directly - buf1 := make([]byte, 512) - buf2 := make([]byte, 512) - buf3 := make([]byte, 512) - buf4 := make([]byte, 512) - - // Test readAt before stat is called. - m, err := r.ReadAt(buf1, offset) - if err != nil { - logger().Fatal("Error:", err, len(buf1), offset) - } - if m != len(buf1) { - logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf1)) - } - if !bytes.Equal(buf1, buf[offset:offset+512]) { - logger().Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - offset += 512 - - st, err := r.Stat() - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - - m, err = r.ReadAt(buf2, offset) - if err != nil { - logger().Fatal("Error:", err, st.Size, len(buf2), offset) - } - if m != len(buf2) { - logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2)) - } - if !bytes.Equal(buf2, buf[offset:offset+512]) { - logger().Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - offset += 512 - m, err = r.ReadAt(buf3, offset) - if err != nil { - logger().Fatal("Error:", err, st.Size, len(buf3), offset) - } - if m != len(buf3) { - logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3)) - } - if !bytes.Equal(buf3, buf[offset:offset+512]) { - logger().Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - offset += 512 - m, err = r.ReadAt(buf4, offset) - if err != nil { - logger().Fatal("Error:", err, st.Size, len(buf4), offset) - } - if m != len(buf4) { - logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4)) - } - if !bytes.Equal(buf4, buf[offset:offset+512]) { - logger().Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - - buf5 := make([]byte, n) - // Read the whole object. - m, err = r.ReadAt(buf5, 0) - if err != nil { - if err != io.EOF { - logger().Fatal("Error:", err, len(buf5)) - } - } - if m != len(buf5) { - logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf5)) - } - if !bytes.Equal(buf, buf5) { - logger().Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") - } - - buf6 := make([]byte, n+1) - // Read the whole object and beyond. - _, err = r.ReadAt(buf6, 0) - if err != nil { - if err != io.EOF { - logger().Fatal("Error:", err, len(buf6)) - } - } - err = c.RemoveObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } -} - -// Test Presigned Post Policy -func testPresignedPostPolicy() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - logger().Fatalf("Error: number of bytes does not match want %v, got %v", - len(buf), n) - } - - policy := minio.NewPostPolicy() - - if err := policy.SetBucket(""); err == nil { - logger().Fatalf("Error: %s", err) - } - if err := policy.SetKey(""); err == nil { - logger().Fatalf("Error: %s", err) - } - if err := policy.SetKeyStartsWith(""); err == nil { - logger().Fatalf("Error: %s", err) - } - if err := policy.SetExpires(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)); err == nil { - logger().Fatalf("Error: %s", err) - } - if err := policy.SetContentType(""); err == nil { - logger().Fatalf("Error: %s", err) - } - if err := policy.SetContentLengthRange(1024*1024, 1024); err == nil { - logger().Fatalf("Error: %s", err) - } - - policy.SetBucket(bucketName) - policy.SetKey(objectName) - policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days - policy.SetContentType("image/png") - policy.SetContentLengthRange(1024, 1024*1024) - - _, _, err = c.PresignedPostPolicy(policy) - if err != nil { - logger().Fatal("Error:", err) - } - - policy = minio.NewPostPolicy() - - // Remove all objects and buckets - err = c.RemoveObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } -} - -// Tests copy object -func testCopyObject() { - logger().Info() - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Make a new bucket in 'us-east-1' (destination bucket). - err = c.MakeBucket(bucketName+"-copy", "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName+"-copy") - } - - // Generate data more than 32K - buf := bytes.Repeat([]byte("5"), rand.Intn(1<<20)+32*1024) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - logger().Fatalf("Error: number of bytes does not match want %v, got %v", - len(buf), n) - } - - r, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err) - } - // Check the various fields of source object against destination object. - objInfo, err := r.Stat() - if err != nil { - logger().Fatal("Error:", err) - } - - // Copy Source - src := minio.NewSourceInfo(bucketName, objectName, nil) - - // Set copy conditions. - - // All invalid conditions first. - err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - logger().Fatal("Error:", err) - } - err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - logger().Fatal("Error:", err) - } - err = src.SetMatchETagCond("") - if err == nil { - logger().Fatal("Error:", err) - } - err = src.SetMatchETagExceptCond("") - if err == nil { - logger().Fatal("Error:", err) - } - - err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - logger().Fatal("Error:", err) - } - err = src.SetMatchETagCond(objInfo.ETag) - if err != nil { - logger().Fatal("Error:", err) - } - - dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) - if err != nil { - logger().Fatal(err) - } - - // Perform the Copy - err = c.CopyObject(dst, src) - if err != nil { - logger().Fatal("Error:", err, bucketName+"-copy", objectName+"-copy") - } - - // Source object - reader, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err) - } - // Destination object - readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy") - if err != nil { - logger().Fatal("Error:", err) - } - // Check the various fields of source object against destination object. - objInfo, err = reader.Stat() - if err != nil { - logger().Fatal("Error:", err) - } - objInfoCopy, err := readerCopy.Stat() - if err != nil { - logger().Fatal("Error:", err) - } - if objInfo.Size != objInfoCopy.Size { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", - objInfo.Size, objInfoCopy.Size) - } - - // CopyObject again but with wrong conditions - src = minio.NewSourceInfo(bucketName, objectName, nil) - err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - logger().Fatal("Error:", err) - } - err = src.SetMatchETagExceptCond(objInfo.ETag) - if err != nil { - logger().Fatal("Error:", err) - } - - // Perform the Copy which should fail - err = c.CopyObject(dst, src) - if err == nil { - logger().Fatal("Error:", err, bucketName+"-copy", objectName+"-copy should fail") - } - - // Remove all objects and buckets - err = c.RemoveObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err) - } - - err = c.RemoveObject(bucketName+"-copy", objectName+"-copy") - if err != nil { - logger().Fatal("Error:", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } - - err = c.RemoveBucket(bucketName + "-copy") - if err != nil { - logger().Fatal("Error:", err) - } -} - -// TestEncryptionPutGet tests client side encryption -func testEncryptionPutGet() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Generate a symmetric key - symKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) - - // Generate an assymmetric key from predefine public and private certificates - privateKey, err := hex.DecodeString( - "30820277020100300d06092a864886f70d0101010500048202613082025d" + - "0201000281810087b42ea73243a3576dc4c0b6fa245d339582dfdbddc20c" + - "bb8ab666385034d997210c54ba79275c51162a1221c3fb1a4c7c61131ca6" + - "5563b319d83474ef5e803fbfa7e52b889e1893b02586b724250de7ac6351" + - "cc0b7c638c980acec0a07020a78eed7eaa471eca4b92071394e061346c06" + - "15ccce2f465dee2080a89e43f29b5702030100010281801dd5770c3af8b3" + - "c85cd18cacad81a11bde1acfac3eac92b00866e142301fee565365aa9af4" + - "57baebf8bb7711054d071319a51dd6869aef3848ce477a0dc5f0dbc0c336" + - "5814b24c820491ae2bb3c707229a654427e03307fec683e6b27856688f08" + - "bdaa88054c5eeeb773793ff7543ee0fb0e2ad716856f2777f809ef7e6fa4" + - "41024100ca6b1edf89e8a8f93cce4b98c76c6990a09eb0d32ad9d3d04fbf" + - "0b026fa935c44f0a1c05dd96df192143b7bda8b110ec8ace28927181fd8c" + - "d2f17330b9b63535024100aba0260afb41489451baaeba423bee39bcbd1e" + - "f63dd44ee2d466d2453e683bf46d019a8baead3a2c7fca987988eb4d565e" + - "27d6be34605953f5034e4faeec9bdb0241009db2cb00b8be8c36710aff96" + - "6d77a6dec86419baca9d9e09a2b761ea69f7d82db2ae5b9aae4246599bb2" + - "d849684d5ab40e8802cfe4a2b358ad56f2b939561d2902404e0ead9ecafd" + - "bb33f22414fa13cbcc22a86bdf9c212ce1a01af894e3f76952f36d6c904c" + - "bd6a7e0de52550c9ddf31f1e8bfe5495f79e66a25fca5c20b3af5b870241" + - "0083456232aa58a8c45e5b110494599bda8dbe6a094683a0539ddd24e19d" + - "47684263bbe285ad953d725942d670b8f290d50c0bca3d1dc9688569f1d5" + - "9945cb5c7d") - - if err != nil { - logger().Fatal(err) - } - - publicKey, err := hex.DecodeString("30819f300d06092a864886f70d010101050003818d003081890281810087" + - "b42ea73243a3576dc4c0b6fa245d339582dfdbddc20cbb8ab666385034d9" + - "97210c54ba79275c51162a1221c3fb1a4c7c61131ca65563b319d83474ef" + - "5e803fbfa7e52b889e1893b02586b724250de7ac6351cc0b7c638c980ace" + - "c0a07020a78eed7eaa471eca4b92071394e061346c0615ccce2f465dee20" + - "80a89e43f29b570203010001") - if err != nil { - logger().Fatal(err) - } - - // Generate an asymmetric key - asymKey, err := encrypt.NewAsymmetricKey(privateKey, publicKey) - if err != nil { - logger().Fatal(err) - } - - // Object custom metadata - customContentType := "custom/contenttype" - - testCases := []struct { - buf []byte - encKey encrypt.Key - }{ - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 0)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 15)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 16)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 17)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 31)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 32)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 33)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*2)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, - - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 0)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 16)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 32)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, - } - - for i, testCase := range testCases { - // Generate a random object name - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - - // Secured object - cbcMaterials, err := encrypt.NewCBCSecureMaterials(testCase.encKey) - if err != nil { - logger().Fatal(err) - } - - // Put encrypted data - _, err = c.PutEncryptedObject(bucketName, objectName, bytes.NewReader(testCase.buf), cbcMaterials, map[string][]string{"Content-Type": {customContentType}}, nil) - if err != nil { - logger().Fatalf("Test %d, error: %v %v %v", i+1, err, bucketName, objectName) - } - - // Read the data back - r, err := c.GetEncryptedObject(bucketName, objectName, cbcMaterials) - if err != nil { - logger().Fatalf("Test %d, error: %v %v %v", i+1, err, bucketName, objectName) - } - defer r.Close() - - // Compare the sent object with the received one - recvBuffer := bytes.NewBuffer([]byte{}) - if _, err = io.Copy(recvBuffer, r); err != nil { - logger().Fatalf("Test %d, error: %v", i+1, err) - } - if recvBuffer.Len() != len(testCase.buf) { - logger().Fatalf("Test %d, error: number of bytes of received object does not match, want %v, got %v\n", - i+1, len(testCase.buf), recvBuffer.Len()) - } - if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { - logger().Fatalf("Test %d, error: Encrypted sent is not equal to decrypted, want `%x`, go `%x`", i+1, testCase.buf, recvBuffer.Bytes()) - } - - // Remove test object - err = c.RemoveObject(bucketName, objectName) - if err != nil { - logger().Fatalf("Test %d, error: %v", i+1, err) - } - - } - - // Remove test bucket - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } - -} - -func testBucketNotification() { - logger().Info() - - if os.Getenv("NOTIFY_BUCKET") == "" || - os.Getenv("NOTIFY_SERVICE") == "" || - os.Getenv("NOTIFY_REGION") == "" || - os.Getenv("NOTIFY_ACCOUNTID") == "" || - os.Getenv("NOTIFY_RESOURCE") == "" { - logger().Info("skipping notification test if not configured") - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable to debug - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - bucketName := os.Getenv("NOTIFY_BUCKET") - - topicArn := minio.NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE")) - queueArn := minio.NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource") - - topicConfig := minio.NewNotificationConfig(topicArn) - topicConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) - topicConfig.AddFilterSuffix("jpg") - - queueConfig := minio.NewNotificationConfig(queueArn) - queueConfig.AddEvents(minio.ObjectCreatedAll) - queueConfig.AddFilterPrefix("photos/") - - bNotification := minio.BucketNotification{} - bNotification.AddTopic(topicConfig) - - // Add the same topicConfig again, should have no effect - // because it is duplicated - bNotification.AddTopic(topicConfig) - if len(bNotification.TopicConfigs) != 1 { - logger().Fatal("Error: duplicated entry added") - } - - // Add and remove a queue config - bNotification.AddQueue(queueConfig) - bNotification.RemoveQueueByArn(queueArn) - - err = c.SetBucketNotification(bucketName, bNotification) - if err != nil { - logger().Fatal("Error: ", err) - } - - bNotification, err = c.GetBucketNotification(bucketName) - if err != nil { - logger().Fatal("Error: ", err) - } - - if len(bNotification.TopicConfigs) != 1 { - logger().Fatal("Error: Topic config is empty") - } - - if bNotification.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" { - logger().Fatal("Error: cannot get the suffix") - } - - err = c.RemoveAllBucketNotification(bucketName) - if err != nil { - logger().Fatal("Error: cannot delete bucket notification") - } -} - -// Tests comprehensive list of all methods. -func testFunctional() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable to debug - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Generate a random file name. - fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - file, err := os.Create(fileName) - if err != nil { - logger().Fatal("Error:", err) - } - for i := 0; i < 3; i++ { - buf := make([]byte, rand.Intn(1<<19)) - _, err = file.Write(buf) - if err != nil { - logger().Fatal("Error:", err) - } - } - file.Close() - - // Verify if bucket exits and you have access. - var exists bool - exists, err = c.BucketExists(bucketName) - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - if !exists { - logger().Fatal("Error: could not find ", bucketName) - } - - // Asserting the default bucket policy. - policyAccess, err := c.GetBucketPolicy(bucketName, "") - if err != nil { - logger().Fatal("Error:", err) - } - if policyAccess != "none" { - logger().Fatalf("Default bucket policy incorrect") - } - // Set the bucket policy to 'public readonly'. - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadOnly) - if err != nil { - logger().Fatal("Error:", err) - } - // should return policy `readonly`. - policyAccess, err = c.GetBucketPolicy(bucketName, "") - if err != nil { - logger().Fatal("Error:", err) - } - if policyAccess != "readonly" { - logger().Fatalf("Expected bucket policy to be readonly") - } - - // Make the bucket 'public writeonly'. - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyWriteOnly) - if err != nil { - logger().Fatal("Error:", err) - } - // should return policy `writeonly`. - policyAccess, err = c.GetBucketPolicy(bucketName, "") - if err != nil { - logger().Fatal("Error:", err) - } - if policyAccess != "writeonly" { - logger().Fatalf("Expected bucket policy to be writeonly") - } - // Make the bucket 'public read/write'. - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadWrite) - if err != nil { - logger().Fatal("Error:", err) - } - // should return policy `readwrite`. - policyAccess, err = c.GetBucketPolicy(bucketName, "") - if err != nil { - logger().Fatal("Error:", err) - } - if policyAccess != "readwrite" { - logger().Fatalf("Expected bucket policy to be readwrite") - } - // List all buckets. - buckets, err := c.ListBuckets() - if len(buckets) == 0 { - logger().Fatal("Error: list buckets cannot be empty", buckets) - } - if err != nil { - logger().Fatal("Error:", err) - } - - // Verify if previously created bucket is listed in list buckets. - bucketFound := false - for _, bucket := range buckets { - if bucket.Name == bucketName { - bucketFound = true - } - } - - // If bucket not found error out. - if !bucketFound { - logger().Fatal("Error: bucket ", bucketName, "not found") - } - - objectName := bucketName + "unique" - - // Generate data - buf := bytes.Repeat([]byte("f"), 1<<19) - - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "") - if err != nil { - logger().Fatal("Error: ", err) - } - if n != int64(len(buf)) { - logger().Fatal("Error: bad length ", n, len(buf)) - } - - n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName+"-nolength") - } - - if n != int64(len(buf)) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Instantiate a done channel to close all listing. - doneCh := make(chan struct{}) - defer close(doneCh) - - objFound := false - isRecursive := true // Recursive is true. - for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - logger().Fatal("Error: object " + objectName + " not found.") - } - - objFound = false - isRecursive = true // Recursive is true. - for obj := range c.ListObjectsV2(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - logger().Fatal("Error: object " + objectName + " not found.") - } - - incompObjNotFound := true - for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { - if objIncompl.Key != "" { - incompObjNotFound = false - break - } - } - if !incompObjNotFound { - logger().Fatal("Error: unexpected dangling incomplete upload found.") - } - - newReader, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error: ", err) - } - - newReadBytes, err := ioutil.ReadAll(newReader) - if err != nil { - logger().Fatal("Error: ", err) - } - - if !bytes.Equal(newReadBytes, buf) { - logger().Fatal("Error: bytes mismatch.") - } - - err = c.FGetObject(bucketName, objectName, fileName+"-f") - if err != nil { - logger().Fatal("Error: ", err) - } - - // Generate presigned GET object url. - presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) - if err != nil { - logger().Fatal("Error: ", err) - } - - // Verify if presigned url works. - resp, err := http.Get(presignedGetURL.String()) - if err != nil { - logger().Fatal("Error: ", err) - } - if resp.StatusCode != http.StatusOK { - logger().Fatal("Error: ", resp.Status) - } - newPresignedBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - logger().Fatal("Error: ", err) - } - if !bytes.Equal(newPresignedBytes, buf) { - logger().Fatal("Error: bytes mismatch.") - } - - // Set request parameters. - reqParams := make(url.Values) - reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") - presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) - if err != nil { - logger().Fatal("Error: ", err) - } - // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) - if err != nil { - logger().Fatal("Error: ", err) - } - if resp.StatusCode != http.StatusOK { - logger().Fatal("Error: ", resp.Status) - } - newPresignedBytes, err = ioutil.ReadAll(resp.Body) - if err != nil { - logger().Fatal("Error: ", err) - } - if !bytes.Equal(newPresignedBytes, buf) { - logger().Fatal("Error: bytes mismatch for presigned GET URL.") - } - if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { - logger().Fatalf("Error: wrong Content-Disposition received %s", resp.Header.Get("Content-Disposition")) - } - - presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) - if err != nil { - logger().Fatal("Error: ", err) - } - - buf = bytes.Repeat([]byte("g"), 1<<19) - - req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) - if err != nil { - logger().Fatal("Error: ", err) - } - httpClient := &http.Client{ - // Setting a sensible time out of 30secs to wait for response - // headers. Request is pro-actively cancelled after 30secs - // with no response. - Timeout: 30 * time.Second, - Transport: http.DefaultTransport, - } - resp, err = httpClient.Do(req) - if err != nil { - logger().Fatal("Error: ", err) - } - - newReader, err = c.GetObject(bucketName, objectName+"-presigned") - if err != nil { - logger().Fatal("Error: ", err) - } - - newReadBytes, err = ioutil.ReadAll(newReader) - if err != nil { - logger().Fatal("Error: ", err) - } - - if !bytes.Equal(newReadBytes, buf) { - logger().Fatal("Error: bytes mismatch.") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-f") - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-nolength") - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-presigned") - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } - err = c.RemoveBucket(bucketName) - if err == nil { - logger().Fatal("Error:") - } - if err.Error() != "The specified bucket does not exist" { - logger().Fatal("Error: ", err) - } - if err = os.Remove(fileName); err != nil { - logger().Fatal("Error: ", err) - } - if err = os.Remove(fileName + "-f"); err != nil { - logger().Fatal("Error: ", err) - } -} - -// Test for validating GetObject Reader* methods functioning when the -// object is modified in the object store. -func testGetObjectObjectModified() { - logger().Info() - - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Make a new bucket. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - defer c.RemoveBucket(bucketName) - - // Upload an object. - objectName := "myobject" - content := "helloworld" - _, err = c.PutObject(bucketName, objectName, strings.NewReader(content), "application/text") - if err != nil { - logger().Fatalf("Failed to upload %s/%s: %v", bucketName, objectName, err) - } - - defer c.RemoveObject(bucketName, objectName) - - reader, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatalf("Failed to get object %s/%s: %v", bucketName, objectName, err) - } - defer reader.Close() - - // Read a few bytes of the object. - b := make([]byte, 5) - n, err := reader.ReadAt(b, 0) - if err != nil { - logger().Fatalf("Failed to read object %s/%s at an offset: %v", bucketName, objectName, err) - } - - // Upload different contents to the same object while object is being read. - newContent := "goodbyeworld" - _, err = c.PutObject(bucketName, objectName, strings.NewReader(newContent), "application/text") - if err != nil { - logger().Fatalf("Failed to upload %s/%s: %v", bucketName, objectName, err) - } - - // Confirm that a Stat() call in between doesn't change the Object's cached etag. - _, err = reader.Stat() - if err.Error() != "At least one of the pre-conditions you specified did not hold" { - log.Error(fmt.Errorf("Expected Stat to fail with error %s but received %s", "At least one of the pre-conditions you specified did not hold", err.Error())) - } - - // Read again only to find object contents have been modified since last read. - _, err = reader.ReadAt(b, int64(n)) - if err.Error() != "At least one of the pre-conditions you specified did not hold" { - log.Error(fmt.Errorf("Expected ReadAt to fail with error %s but received %s", "At least one of the pre-conditions you specified did not hold", err.Error())) - } -} - -// Test validates putObject to upload a file seeked at a given offset. -func testPutObjectUploadSeekedObject() { - logger().Info() - - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Make a new bucket. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - defer c.RemoveBucket(bucketName) - - tempfile, err := ioutil.TempFile("", "minio-go-upload-test-") - if err != nil { - logger().Fatal("Error:", err) - } - - var data []byte - if fileName := getFilePath("datafile-100-kB"); fileName != "" { - data, _ = ioutil.ReadFile(fileName) - } else { - // Generate data more than 32K - data = bytes.Repeat([]byte("1"), 120000) - } - var length = len(data) - if _, err = tempfile.Write(data); err != nil { - logger().Fatal("Error:", err) - } - - objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) - - offset := length / 2 - if _, err := tempfile.Seek(int64(offset), 0); err != nil { - logger().Fatal("Error:", err) - } - - n, err := c.PutObject(bucketName, objectName, tempfile, "binary/octet-stream") - if err != nil { - logger().Fatal("Error:", err) - } - if n != int64(length-offset) { - logger().Fatalf("Invalid length returned, want %v, got %v", int64(length-offset), n) - } - tempfile.Close() - if err = os.Remove(tempfile.Name()); err != nil { - logger().Fatal("Error:", err) - } - - length = int(n) - - obj, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err) - } - - n, err = obj.Seek(int64(offset), 0) - if err != nil { - logger().Fatal("Error:", err) - } - if n != int64(offset) { - logger().Fatalf("Invalid offset returned, want %v, got %v", int64(offset), n) - } - - n, err = c.PutObject(bucketName, objectName+"getobject", obj, "binary/octet-stream") - if err != nil { - logger().Fatal("Error:", err) - } - if n != int64(length-offset) { - logger().Fatalf("Invalid length returned, want %v, got %v", int64(length-offset), n) - } - - if err = c.RemoveObject(bucketName, objectName); err != nil { - logger().Fatal("Error:", err) - } - - if err = c.RemoveObject(bucketName, objectName+"getobject"); err != nil { - logger().Fatal("Error:", err) - } -} - -// Tests bucket re-create errors. -func testMakeBucketErrorV2() { - logger().Info() - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - logger().Info("skipping region functional tests for non s3 runs") - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'eu-west-1'. - if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { - logger().Fatal("Error:", err, bucketName) - } - if err = c.MakeBucket(bucketName, "eu-west-1"); err == nil { - logger().Fatal("Error: make bucket should should fail for", bucketName) - } - // Verify valid error response from server. - if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && - minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { - logger().Fatal("Error: Invalid error returned by server", err) - } - if err = c.RemoveBucket(bucketName); err != nil { - logger().Fatal("Error:", err, bucketName) - } -} - -// Test get object reader to not throw error on being closed twice. -func testGetObjectClosedTwiceV2() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K. - var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - if err := r.Close(); err != nil { - logger().Fatal("Error:", err) - } - if err := r.Close(); err == nil { - logger().Fatal("Error: object is already closed, should return error") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } -} - -// Tests removing partially uploaded objects. -func testRemovePartiallyUploadedV2() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - r := bytes.NewReader(bytes.Repeat([]byte("a"), 128*1024)) - - reader, writer := io.Pipe() - go func() { - i := 0 - for i < 25 { - _, cerr := io.CopyN(writer, r, 128*1024) - if cerr != nil { - logger().Fatal("Error:", cerr, bucketName) - } - i++ - r.Seek(0, 0) - } - writer.CloseWithError(errors.New("proactively closed to be verified later")) - }() - - objectName := bucketName + "-resumable" - _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") - if err == nil { - logger().Fatal("Error: PutObject should fail.") - } - if err.Error() != "proactively closed to be verified later" { - logger().Fatal("Error:", err) - } - err = c.RemoveIncompleteUpload(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } -} - -// Tests FPutObject hidden contentType setting -func testFPutObjectV2() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Make a temp file with 11*1024*1024 bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") - if err != nil { - logger().Fatal("Error:", err) - } - - r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024)) - n, err := io.CopyN(file, r, 11*1024*1024) - if err != nil { - logger().Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - // Close the file pro-actively for windows. - err = file.Close() - if err != nil { - logger().Fatal("Error:", err) - } - - // Set base object name - objectName := bucketName + "FPutObject" - - // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err = c.FPutObject(bucketName, objectName+"-standard", file.Name(), "application/octet-stream") - if err != nil { - logger().Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - // Perform FPutObject with no contentType provided (Expecting application/octet-stream) - n, err = c.FPutObject(bucketName, objectName+"-Octet", file.Name(), "") - if err != nil { - logger().Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - // Add extension to temp file name - fileName := file.Name() - err = os.Rename(file.Name(), fileName+".gtar") - if err != nil { - logger().Fatal("Error:", err) - } - - // Perform FPutObject with no contentType provided (Expecting application/x-gtar) - n, err = c.FPutObject(bucketName, objectName+"-GTar", fileName+".gtar", "") - if err != nil { - logger().Fatal("Error:", err) - } - if n != int64(11*1024*1024) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) - } - - // Check headers - rStandard, err := c.StatObject(bucketName, objectName+"-standard") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName+"-standard") - } - if rStandard.ContentType != "application/octet-stream" { - logger().Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", - "application/octet-stream", rStandard.ContentType) - } - - rOctet, err := c.StatObject(bucketName, objectName+"-Octet") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName+"-Octet") - } - if rOctet.ContentType != "application/octet-stream" { - logger().Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", - "application/octet-stream", rStandard.ContentType) - } - - rGTar, err := c.StatObject(bucketName, objectName+"-GTar") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName+"-GTar") - } - if rGTar.ContentType != "application/x-gtar" { - logger().Fatalf("Error: Content-Type headers mismatched, want %v, got %v\n", - "application/x-gtar", rStandard.ContentType) - } - - // Remove all objects and bucket and temp file - err = c.RemoveObject(bucketName, objectName+"-standard") - if err != nil { - logger().Fatal("Error: ", err) - } - - err = c.RemoveObject(bucketName, objectName+"-Octet") - if err != nil { - logger().Fatal("Error: ", err) - } - - err = c.RemoveObject(bucketName, objectName+"-GTar") - if err != nil { - logger().Fatal("Error: ", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } - - err = os.Remove(fileName + ".gtar") - if err != nil { - logger().Fatal("Error:", err) - } - -} - -// Tests various bucket supported formats. -func testMakeBucketRegionsV2() { - logger().Info() - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - logger().Info("skipping region functional tests for non s3 runs") - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { - logger().Fatal("Error:", err, bucketName) - } - - if err = c.RemoveBucket(bucketName); err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Make a new bucket with '.' in its name, in 'us-west-2'. This - // request is internally staged into a path style instead of - // virtual host style. - if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { - logger().Fatal("Error:", err, bucketName+".withperiod") - } - - // Remove the newly created bucket. - if err = c.RemoveBucket(bucketName + ".withperiod"); err != nil { - logger().Fatal("Error:", err, bucketName+".withperiod") - } -} - -// Tests get object ReaderSeeker interface methods. -func testGetObjectReadSeekFunctionalV2() { - logger().Info() - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K. - var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) - - // Save the data. - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - - offset := int64(2048) - n, err = r.Seek(offset, 0) - if err != nil { - logger().Fatal("Error:", err, offset) - } - if n != offset { - logger().Fatalf("Error: number of bytes seeked does not match, want %v, got %v\n", - offset, n) - } - n, err = r.Seek(0, 1) - if err != nil { - logger().Fatal("Error:", err) - } - if n != offset { - logger().Fatalf("Error: number of current seek does not match, want %v, got %v\n", - offset, n) - } - _, err = r.Seek(offset, 2) - if err == nil { - logger().Fatal("Error: seek on positive offset for whence '2' should error out") - } - n, err = r.Seek(-offset, 2) - if err != nil { - logger().Fatal("Error:", err) - } - if n != st.Size-offset { - logger().Fatalf("Error: number of bytes seeked back does not match, want %d, got %v\n", st.Size-offset, n) - } - - var buffer1 bytes.Buffer - if _, err = io.CopyN(&buffer1, r, st.Size); err != nil { - if err != io.EOF { - logger().Fatal("Error:", err) - } - } - if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) { - logger().Fatal("Error: Incorrect read bytes v/s original buffer.") - } - - // Seek again and read again. - n, err = r.Seek(offset-1, 0) - if err != nil { - logger().Fatal("Error:", err) - } - if n != (offset - 1) { - logger().Fatalf("Error: number of bytes seeked back does not match, want %v, got %v\n", offset-1, n) - } - - var buffer2 bytes.Buffer - if _, err = io.CopyN(&buffer2, r, st.Size); err != nil { - if err != io.EOF { - logger().Fatal("Error:", err) - } - } - // Verify now lesser bytes. - if !bytes.Equal(buf[2047:], buffer2.Bytes()) { - logger().Fatal("Error: Incorrect read bytes v/s original buffer.") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } -} - -// Tests get object ReaderAt interface methods. -func testGetObjectReadAtFunctionalV2() { - logger().Info() - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Generate data more than 32K - var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - st, err := r.Stat() - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - if st.Size != int64(len(buf)) { - logger().Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", - len(buf), st.Size) - } - - offset := int64(2048) - - // Read directly - buf2 := make([]byte, 512) - buf3 := make([]byte, 512) - buf4 := make([]byte, 512) - - m, err := r.ReadAt(buf2, offset) - if err != nil { - logger().Fatal("Error:", err, st.Size, len(buf2), offset) - } - if m != len(buf2) { - logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2)) - } - if !bytes.Equal(buf2, buf[offset:offset+512]) { - logger().Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - offset += 512 - m, err = r.ReadAt(buf3, offset) - if err != nil { - logger().Fatal("Error:", err, st.Size, len(buf3), offset) - } - if m != len(buf3) { - logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3)) - } - if !bytes.Equal(buf3, buf[offset:offset+512]) { - logger().Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - offset += 512 - m, err = r.ReadAt(buf4, offset) - if err != nil { - logger().Fatal("Error:", err, st.Size, len(buf4), offset) - } - if m != len(buf4) { - logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4)) - } - if !bytes.Equal(buf4, buf[offset:offset+512]) { - logger().Fatal("Error: Incorrect read between two ReadAt from same offset.") - } - - buf5 := make([]byte, n) - // Read the whole object. - m, err = r.ReadAt(buf5, 0) - if err != nil { - if err != io.EOF { - logger().Fatal("Error:", err, len(buf5)) - } - } - if m != len(buf5) { - logger().Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf5)) - } - if !bytes.Equal(buf, buf5) { - logger().Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") - } - - buf6 := make([]byte, n+1) - // Read the whole object and beyond. - _, err = r.ReadAt(buf6, 0) - if err != nil { - if err != io.EOF { - logger().Fatal("Error:", err, len(buf6)) - } - } - err = c.RemoveObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } -} - -// Tests copy object -func testCopyObjectV2() { - logger().Info() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Make a new bucket in 'us-east-1' (destination bucket). - err = c.MakeBucket(bucketName+"-copy", "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName+"-copy") - } - - // Generate data more than 32K - var buf = getDataBuffer("datafile-33-kB", rand.Intn(1<<20)+32*1024) - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName) - } - - if n != int64(len(buf)) { - logger().Fatalf("Error: number of bytes does not match want %v, got %v", - len(buf), n) - } - - r, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err) - } - // Check the various fields of source object against destination object. - objInfo, err := r.Stat() - if err != nil { - logger().Fatal("Error:", err) - } - - // Copy Source - src := minio.NewSourceInfo(bucketName, objectName, nil) - - // Set copy conditions. - - // All invalid conditions first. - err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - logger().Fatal("Error:", err) - } - err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - logger().Fatal("Error:", err) - } - err = src.SetMatchETagCond("") - if err == nil { - logger().Fatal("Error:", err) - } - err = src.SetMatchETagExceptCond("") - if err == nil { - logger().Fatal("Error:", err) - } - - err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - logger().Fatal("Error:", err) - } - err = src.SetMatchETagCond(objInfo.ETag) - if err != nil { - logger().Fatal("Error:", err) - } - - dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) - if err != nil { - logger().Fatal(err) - } - - // Perform the Copy - err = c.CopyObject(dst, src) - if err != nil { - logger().Fatal("Error:", err, bucketName+"-copy", objectName+"-copy") - } - - // Source object - reader, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err) - } - // Destination object - readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy") - if err != nil { - logger().Fatal("Error:", err) - } - // Check the various fields of source object against destination object. - objInfo, err = reader.Stat() - if err != nil { - logger().Fatal("Error:", err) - } - objInfoCopy, err := readerCopy.Stat() - if err != nil { - logger().Fatal("Error:", err) - } - if objInfo.Size != objInfoCopy.Size { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", - objInfo.Size, objInfoCopy.Size) - } - - // CopyObject again but with wrong conditions - src = minio.NewSourceInfo(bucketName, objectName, nil) - err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - logger().Fatal("Error:", err) - } - err = src.SetMatchETagExceptCond(objInfo.ETag) - if err != nil { - logger().Fatal("Error:", err) - } - - // Perform the Copy which should fail - err = c.CopyObject(dst, src) - if err == nil { - logger().Fatal("Error:", err, bucketName+"-copy", objectName+"-copy should fail") - } - - // Remove all objects and buckets - err = c.RemoveObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error:", err) - } - - err = c.RemoveObject(bucketName+"-copy", objectName+"-copy") - if err != nil { - logger().Fatal("Error:", err) - } - - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } - - err = c.RemoveBucket(bucketName + "-copy") - if err != nil { - logger().Fatal("Error:", err) - } -} - -func testComposeObjectErrorCasesWrapper(c *minio.Client) { - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Test that more than 10K source objects cannot be - // concatenated. - srcArr := [10001]minio.SourceInfo{} - srcSlice := srcArr[:] - dst, err := minio.NewDestinationInfo(bucketName, "object", nil, nil) - if err != nil { - logger().Fatal(err) - } - - if err := c.ComposeObject(dst, srcSlice); err == nil { - logger().Fatal("Error was expected.") - } else if err.Error() != "There must be as least one and upto 10000 source objects." { - logger().Fatal("Got unexpected error: ", err) - } - - // Create a source with invalid offset spec and check that - // error is returned: - // 1. Create the source object. - const badSrcSize = 5 * 1024 * 1024 - buf := bytes.Repeat([]byte("1"), badSrcSize) - _, err = c.PutObject(bucketName, "badObject", bytes.NewReader(buf), "") - if err != nil { - logger().Fatal("Error:", err) - } - // 2. Set invalid range spec on the object (going beyond - // object size) - badSrc := minio.NewSourceInfo(bucketName, "badObject", nil) - err = badSrc.SetRange(1, badSrcSize) - if err != nil { - logger().Fatal("Error:", err) - } - // 3. ComposeObject call should fail. - if err := c.ComposeObject(dst, []minio.SourceInfo{badSrc}); err == nil { - logger().Fatal("Error was expected.") - } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") { - logger().Fatal("Got unexpected error: ", err) - } -} - -// Test expected error cases -func testComposeObjectErrorCasesV2() { - logger().Info() - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - testComposeObjectErrorCasesWrapper(c) -} - -func testComposeMultipleSources(c *minio.Client) { - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Upload a small source object - const srcSize = 1024 * 1024 * 5 - buf := bytes.Repeat([]byte("1"), srcSize) - _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - logger().Fatal("Error:", err) - } - - // We will append 10 copies of the object. - srcs := []minio.SourceInfo{} - for i := 0; i < 10; i++ { - srcs = append(srcs, minio.NewSourceInfo(bucketName, "srcObject", nil)) - } - // make the last part very small - err = srcs[9].SetRange(0, 0) - if err != nil { - logger().Fatal("unexpected error:", err) - } - - dst, err := minio.NewDestinationInfo(bucketName, "dstObject", nil, nil) - if err != nil { - logger().Fatal(err) - } - err = c.ComposeObject(dst, srcs) - if err != nil { - logger().Fatal("Error:", err) - } - - objProps, err := c.StatObject(bucketName, "dstObject") - if err != nil { - logger().Fatal("Error:", err) - } - - if objProps.Size != 9*srcSize+1 { - logger().Fatal("Size mismatched! Expected:", 10000*srcSize, "but got:", objProps.Size) - } -} - -// Test concatenating multiple objects objects -func testCompose10KSourcesV2() { - logger().Info() - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - testComposeMultipleSources(c) -} -func testEncryptedCopyObjectWrapper(c *minio.Client) { - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - key1 := minio.NewSSEInfo([]byte("32byteslongsecretkeymustbegiven1"), "AES256") - key2 := minio.NewSSEInfo([]byte("32byteslongsecretkeymustbegiven2"), "AES256") - - // 1. create an sse-c encrypted object to copy by uploading - const srcSize = 1024 * 1024 - buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB - metadata := make(map[string][]string) - for k, v := range key1.GetSSEHeaders() { - metadata[k] = append(metadata[k], v) - } - _, err = c.PutObjectWithSize(bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), metadata, nil) - if err != nil { - logger().Fatal("PutObjectWithSize Error:", err) - } - - // 2. copy object and change encryption key - src := minio.NewSourceInfo(bucketName, "srcObject", &key1) - dst, err := minio.NewDestinationInfo(bucketName, "dstObject", &key2, nil) - if err != nil { - logger().Fatal("Error:", err) - } - - err = c.CopyObject(dst, src) - if err != nil { - logger().Fatal("CopyObject Error:", err) - } - - // 3. get copied object and check if content is equal - reqH := minio.NewGetReqHeaders() - for k, v := range key2.GetSSEHeaders() { - reqH.Set(k, v) - } - coreClient := minio.Core{c} - reader, _, err := coreClient.GetObject(bucketName, "dstObject", reqH) - if err != nil { - logger().Fatal("GetObject Error:", err) - } - defer reader.Close() - - decBytes, err := ioutil.ReadAll(reader) - if err != nil { - logger().Fatalln(err) - } - if !bytes.Equal(decBytes, buf) { - logger().Fatal("downloaded object mismatched for encrypted object") - } -} - -// Test encrypted copy object -func testEncryptedCopyObject() { - logger().Info() - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // c.TraceOn(os.Stderr) - testEncryptedCopyObjectWrapper(c) -} - -// Test encrypted copy object -func testEncryptedCopyObjectV2() { - logger().Info() - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - testEncryptedCopyObjectWrapper(c) -} -func testUserMetadataCopying() { - logger().Info() - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // c.TraceOn(os.Stderr) - testUserMetadataCopyingWrapper(c) -} -func testUserMetadataCopyingWrapper(c *minio.Client) { - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - fetchMeta := func(object string) (h http.Header) { - objInfo, err := c.StatObject(bucketName, object) - if err != nil { - logger().Fatal("Metadata fetch error:", err) - } - h = make(http.Header) - for k, vs := range objInfo.Metadata { - if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { - for _, v := range vs { - h.Add(k, v) - } - } - } - return h - } - - // 1. create a client encrypted object to copy by uploading - const srcSize = 1024 * 1024 - buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB - metadata := make(http.Header) - metadata.Set("x-amz-meta-myheader", "myvalue") - _, err = c.PutObjectWithMetadata(bucketName, "srcObject", - bytes.NewReader(buf), metadata, nil) - if err != nil { - logger().Fatal("Put Error:", err) - } - if !reflect.DeepEqual(metadata, fetchMeta("srcObject")) { - logger().Fatal("Unequal metadata") - } - - // 2. create source - src := minio.NewSourceInfo(bucketName, "srcObject", nil) - // 2.1 create destination with metadata set - dst1, err := minio.NewDestinationInfo(bucketName, "dstObject-1", nil, map[string]string{"notmyheader": "notmyvalue"}) - if err != nil { - logger().Fatal("Error:", err) - } - - // 3. Check that copying to an object with metadata set resets - // the headers on the copy. - err = c.CopyObject(dst1, src) - if err != nil { - logger().Fatal("Error:", err) - } - - expectedHeaders := make(http.Header) - expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") - if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-1")) { - logger().Fatal("Unequal metadata") - } - - // 4. create destination with no metadata set and same source - dst2, err := minio.NewDestinationInfo(bucketName, "dstObject-2", nil, nil) - if err != nil { - logger().Fatal("Error:", err) - - } - src = minio.NewSourceInfo(bucketName, "srcObject", nil) - - // 5. Check that copying to an object with no metadata set, - // copies metadata. - err = c.CopyObject(dst2, src) - if err != nil { - logger().Fatal("Error:", err) - } - - expectedHeaders = metadata - if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-2")) { - logger().Fatal("Unequal metadata") - } - - // 6. Compose a pair of sources. - srcs := []minio.SourceInfo{ - minio.NewSourceInfo(bucketName, "srcObject", nil), - minio.NewSourceInfo(bucketName, "srcObject", nil), - } - dst3, err := minio.NewDestinationInfo(bucketName, "dstObject-3", nil, nil) - if err != nil { - logger().Fatal("Error:", err) - - } - - err = c.ComposeObject(dst3, srcs) - if err != nil { - logger().Fatal("Error:", err) - } - - // Check that no headers are copied in this case - if !reflect.DeepEqual(make(http.Header), fetchMeta("dstObject-3")) { - logger().Fatal("Unequal metadata") - } - - // 7. Compose a pair of sources with dest user metadata set. - srcs = []minio.SourceInfo{ - minio.NewSourceInfo(bucketName, "srcObject", nil), - minio.NewSourceInfo(bucketName, "srcObject", nil), - } - dst4, err := minio.NewDestinationInfo(bucketName, "dstObject-4", nil, map[string]string{"notmyheader": "notmyvalue"}) - if err != nil { - logger().Fatal("Error:", err) - - } - - err = c.ComposeObject(dst4, srcs) - if err != nil { - logger().Fatal("Error:", err) - } - - // Check that no headers are copied in this case - expectedHeaders = make(http.Header) - expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") - if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-4")) { - logger().Fatal("Unequal metadata") - } -} - -func testUserMetadataCopyingV2() { - logger().Info() - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // c.TraceOn(os.Stderr) - testUserMetadataCopyingWrapper(c) -} - -// Test put object with 0 byte object. -func testPutObject0ByteV2() { - logTrace() - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - log.Fatal("Error:", err) - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), - "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - log.Fatal("Error:", err, bucketName) - } - - objectName := bucketName + "unique" - - // Upload an object. - n, err := c.PutObjectWithSize(bucketName, objectName, bytes.NewReader([]byte("")), 0, nil, nil) - if err != nil { - log.Fatalf("Error: %v %s %s", err, bucketName, objectName) - } - if n != 0 { - log.Error(fmt.Errorf("Expected upload object size 0 but got %d", n)) - } - - // Remove the object. - err = c.RemoveObject(bucketName, objectName) - if err != nil { - log.Fatal("Error:", err) - } - - // Remove the bucket. - err = c.RemoveBucket(bucketName) - if err != nil { - log.Fatal("Error:", err) - } -} - -// Test expected error cases -func testComposeObjectErrorCases() { - logger().Info() - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - testComposeObjectErrorCasesWrapper(c) -} - -// Test concatenating 10K objects -func testCompose10KSources() { - logger().Info() - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - testComposeMultipleSources(c) -} - -// Tests comprehensive list of all methods. -func testFunctionalV2() { - logger().Info() - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logger().Fatal("Error:", err) - } - - // Enable to debug - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - - // Generate a random file name. - fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - file, err := os.Create(fileName) - if err != nil { - logger().Fatal("Error:", err) - } - for i := 0; i < 3; i++ { - buf := make([]byte, rand.Intn(1<<19)) - _, err = file.Write(buf) - if err != nil { - logger().Fatal("Error:", err) - } - } - file.Close() - - // Verify if bucket exits and you have access. - var exists bool - exists, err = c.BucketExists(bucketName) - if err != nil { - logger().Fatal("Error:", err, bucketName) - } - if !exists { - logger().Fatal("Error: could not find ", bucketName) - } - - // Make the bucket 'public read/write'. - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadWrite) - if err != nil { - logger().Fatal("Error:", err) - } - - // List all buckets. - buckets, err := c.ListBuckets() - if len(buckets) == 0 { - logger().Fatal("Error: list buckets cannot be empty", buckets) - } - if err != nil { - logger().Fatal("Error:", err) - } - - // Verify if previously created bucket is listed in list buckets. - bucketFound := false - for _, bucket := range buckets { - if bucket.Name == bucketName { - bucketFound = true - } - } - - // If bucket not found error out. - if !bucketFound { - logger().Fatal("Error: bucket ", bucketName, "not found") - } - - objectName := bucketName + "unique" - - // Generate data - buf := bytes.Repeat([]byte("n"), rand.Intn(1<<19)) - - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "") - if err != nil { - logger().Fatal("Error: ", err) - } - if n != int64(len(buf)) { - logger().Fatal("Error: bad length ", n, len(buf)) - } - - n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream") - if err != nil { - logger().Fatal("Error:", err, bucketName, objectName+"-nolength") - } - - if n != int64(len(buf)) { - logger().Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) - } - - // Instantiate a done channel to close all listing. - doneCh := make(chan struct{}) - defer close(doneCh) - - objFound := false - isRecursive := true // Recursive is true. - for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - logger().Fatal("Error: object " + objectName + " not found.") - } - - objFound = false - isRecursive = true // Recursive is true. - for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - logger().Fatal("Error: object " + objectName + " not found.") - } - - incompObjNotFound := true - for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { - if objIncompl.Key != "" { - incompObjNotFound = false - break - } - } - if !incompObjNotFound { - logger().Fatal("Error: unexpected dangling incomplete upload found.") - } - - newReader, err := c.GetObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error: ", err) - } - - newReadBytes, err := ioutil.ReadAll(newReader) - if err != nil { - logger().Fatal("Error: ", err) - } - - if !bytes.Equal(newReadBytes, buf) { - logger().Fatal("Error: bytes mismatch.") - } - - err = c.FGetObject(bucketName, objectName, fileName+"-f") - if err != nil { - logger().Fatal("Error: ", err) - } - - // Generate presigned GET object url. - presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) - if err != nil { - logger().Fatal("Error: ", err) - } - // Verify if presigned url works. - resp, err := http.Get(presignedGetURL.String()) - if err != nil { - logger().Fatal("Error: ", err) - } - if resp.StatusCode != http.StatusOK { - logger().Fatal("Error: ", resp.Status) - } - newPresignedBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - logger().Fatal("Error: ", err) - } - if !bytes.Equal(newPresignedBytes, buf) { - logger().Fatal("Error: bytes mismatch.") - } - - // Set request parameters. - reqParams := make(url.Values) - reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") - // Generate presigned GET object url. - presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) - if err != nil { - logger().Fatal("Error: ", err) - } - // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) - if err != nil { - logger().Fatal("Error: ", err) - } - if resp.StatusCode != http.StatusOK { - logger().Fatal("Error: ", resp.Status) - } - newPresignedBytes, err = ioutil.ReadAll(resp.Body) - if err != nil { - logger().Fatal("Error: ", err) - } - if !bytes.Equal(newPresignedBytes, buf) { - logger().Fatal("Error: bytes mismatch for presigned GET url.") - } - // Verify content disposition. - if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { - logger().Fatalf("Error: wrong Content-Disposition received %s", resp.Header.Get("Content-Disposition")) - } - - presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) - if err != nil { - logger().Fatal("Error: ", err) - } - // Generate data more than 32K - buf = bytes.Repeat([]byte("1"), rand.Intn(1<<20)+32*1024) - - req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) - if err != nil { - logger().Fatal("Error: ", err) - } - httpClient := &http.Client{ - // Setting a sensible time out of 30secs to wait for response - // headers. Request is pro-actively cancelled after 30secs - // with no response. - Timeout: 30 * time.Second, - Transport: http.DefaultTransport, - } - resp, err = httpClient.Do(req) - if err != nil { - logger().Fatal("Error: ", err) - } - - newReader, err = c.GetObject(bucketName, objectName+"-presigned") - if err != nil { - logger().Fatal("Error: ", err) - } - - newReadBytes, err = ioutil.ReadAll(newReader) - if err != nil { - logger().Fatal("Error: ", err) - } - - if !bytes.Equal(newReadBytes, buf) { - logger().Fatal("Error: bytes mismatch.") - } - - err = c.RemoveObject(bucketName, objectName) - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-f") - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-nolength") - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveObject(bucketName, objectName+"-presigned") - if err != nil { - logger().Fatal("Error: ", err) - } - err = c.RemoveBucket(bucketName) - if err != nil { - logger().Fatal("Error:", err) - } - err = c.RemoveBucket(bucketName) - if err == nil { - logger().Fatal("Error:") - } - if err.Error() != "The specified bucket does not exist" { - logger().Fatal("Error: ", err) - } - if err = os.Remove(fileName); err != nil { - logger().Fatal("Error: ", err) - } - if err = os.Remove(fileName + "-f"); err != nil { - logger().Fatal("Error: ", err) - } -} - -// Convert string to bool and always return false if any error -func mustParseBool(str string) bool { - b, err := strconv.ParseBool(str) - if err != nil { - return false - } - return b -} - -func logger() *logrus.Entry { - if pc, file, line, ok := runtime.Caller(1); ok { - fName := runtime.FuncForPC(pc).Name() - return log.WithFields(log.Fields{"file": path.Base(file), "function:": fName, "line#": line}) - - } - return log.WithFields(nil) -} - -func main() { - logger().Info("Running functional tests for minio-go sdk....") - if !isQuickMode() { - testMakeBucketErrorV2() - testGetObjectClosedTwiceV2() - testRemovePartiallyUploadedV2() - testFPutObjectV2() - testMakeBucketRegionsV2() - testGetObjectReadSeekFunctionalV2() - testGetObjectReadAtFunctionalV2() - testCopyObjectV2() - testFunctionalV2() - testComposeObjectErrorCasesV2() - testCompose10KSourcesV2() - testEncryptedCopyObjectV2() - testUserMetadataCopyingV2() - testPutObject0ByteV2() - testMakeBucketError() - testMakeBucketRegions() - testPutObjectWithMetadata() - testPutObjectReadAt() - testPutObjectStreaming() - testListPartiallyUploaded() - testGetObjectSeekEnd() - testGetObjectClosedTwice() - testRemoveMultipleObjects() - testRemovePartiallyUploaded() - testFPutObjectMultipart() - testFPutObject() - testGetObjectReadSeekFunctional() - testGetObjectReadAtFunctional() - testPresignedPostPolicy() - testCopyObject() - testEncryptionPutGet() - testComposeObjectErrorCases() - testCompose10KSources() - testUserMetadataCopying() - testEncryptedCopyObject() - testBucketNotification() - testFunctional() - testGetObjectObjectModified() - testPutObjectUploadSeekedObject() - } else { - logger().Info("Running short functional tests") - testFunctional() - testFunctionalV2() - } - - logger().Info("Functional tests complete for minio-go sdk") -} diff --git a/vendor/src/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go b/vendor/src/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go index d831436cd..22059bb1d 100644 --- a/vendor/src/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go +++ b/vendor/src/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go @@ -99,7 +99,7 @@ func prepareStreamingRequest(req *http.Request, sessionToken string, dataLen int if sessionToken != "" { req.Header.Set("X-Amz-Security-Token", sessionToken) } - req.Header.Add("Content-Encoding", streamingEncoding) + req.Header.Set("Content-Encoding", streamingEncoding) req.Header.Set("X-Amz-Date", timestamp.Format(iso8601DateFormat)) // Set content length with streaming signature for each chunk included. diff --git a/vendor/src/github.com/minio/minio-go/test-utils_test.go b/vendor/src/github.com/minio/minio-go/test-utils_test.go index b109dfaf7..4134af996 100644 --- a/vendor/src/github.com/minio/minio-go/test-utils_test.go +++ b/vendor/src/github.com/minio/minio-go/test-utils_test.go @@ -64,11 +64,11 @@ func encodeResponse(response interface{}) []byte { return bytesBuffer.Bytes() } -// Convert string to bool and always return false if any error +// Convert string to bool and always return true if any error func mustParseBool(str string) bool { b, err := strconv.ParseBool(str) if err != nil { - return false + return true } return b } diff --git a/vendor/src/github.com/minio/minio-go/transport.go b/vendor/src/github.com/minio/minio-go/transport.go deleted file mode 100644 index d286bd7ae..000000000 --- a/vendor/src/github.com/minio/minio-go/transport.go +++ /dev/null @@ -1,48 +0,0 @@ -// +build go1.7 go1.8 - -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * (C) 2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "net" - "net/http" - "time" -) - -// This default transport is similar to http.DefaultTransport -// but with additional DisableCompression: -var defaultMinioTransport http.RoundTripper = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - // Set this value so that the underlying transport round-tripper - // doesn't try to auto decode the body of objects with - // content-encoding set to `gzip`. - // - // Refer: - // https://golang.org/src/net/http/transport.go?h=roundTrip#L1843 - DisableCompression: true, -} diff --git a/vendor/src/github.com/minio/minio-go/transport_1_5.go b/vendor/src/github.com/minio/minio-go/transport_1_5.go deleted file mode 100644 index 468daafd3..000000000 --- a/vendor/src/github.com/minio/minio-go/transport_1_5.go +++ /dev/null @@ -1,39 +0,0 @@ -// +build go1.5,!go1.6,!go1.7,!go1.8 - -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * (C) 2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "net/http" - "time" -) - -// This default transport is similar to http.DefaultTransport -// but with additional DisableCompression: -var defaultMinioTransport http.RoundTripper = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSHandshakeTimeout: 10 * time.Second, - // Set this value so that the underlying transport round-tripper - // doesn't try to auto decode the body of objects with - // content-encoding set to `gzip`. - // - // Refer: - // https://golang.org/src/net/http/transport.go?h=roundTrip#L1843 - DisableCompression: true, -} diff --git a/vendor/src/github.com/minio/minio-go/transport_1_6.go b/vendor/src/github.com/minio/minio-go/transport_1_6.go deleted file mode 100644 index 77e7d76fc..000000000 --- a/vendor/src/github.com/minio/minio-go/transport_1_6.go +++ /dev/null @@ -1,40 +0,0 @@ -// +build go1.6,!go1.7,!go1.8 - -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * (C) 2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "net/http" - "time" -) - -// This default transport is similar to http.DefaultTransport -// but with additional DisableCompression: -var defaultMinioTransport http.RoundTripper = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - // Set this value so that the underlying transport round-tripper - // doesn't try to auto decode the body of objects with - // content-encoding set to `gzip`. - // - // Refer: - // https://golang.org/src/net/http/transport.go?h=roundTrip#L1843 - DisableCompression: true, -}