Merge pull request #1124 from restic/use-minio-300

Set minio-go to v3.0.0

vendor/manifest

@@ -46,8 +46,8 @@
 		{
 			"importpath": "github.com/minio/minio-go",
 			"repository": "https://github.com/minio/minio-go",
-			"revision": "bd8e1d8a93f006a0207e026353bf0644ffcdd320",
+			"revision": "5ca66c9a35ba1cd674484be99dc97aa0973afe12",
-			"branch": "master"
+			"branch": "HEAD"
 		},
 		{
 			"importpath": "github.com/ncw/swift",

vendor/src/github.com/minio/minio-go/api-get-object.go

@@ -679,18 +679,12 @@ func (c Client) getObject(bucketName, objectName string, reqHeaders RequestHeade
 	if contentType == "" {
 		contentType = "application/octet-stream"
 	}
-
+	var objectStat ObjectInfo
-	objectStat := ObjectInfo{
+	objectStat.ETag = md5sum
-		ETag:         md5sum,
+	objectStat.Key = objectName
-		Key:          objectName,
+	objectStat.Size = resp.ContentLength
-		Size:         resp.ContentLength,
+	objectStat.LastModified = date
-		LastModified: date,
+	objectStat.ContentType = contentType
-		ContentType:  contentType,
-		// Extract only the relevant header keys describing the object.
-		// following function filters out a list of standard set of keys
-		// which are not part of object metadata.
-		Metadata: extractObjMetadata(resp.Header),
-	}
 
 	// do not close body here, caller will close
 	return resp.Body, objectStat, nil

vendor/src/github.com/minio/minio-go/api-put-object.go

@@ -194,7 +194,7 @@ func (c Client) putObjectCommon(bucketName, objectName string, reader io.Reader,
 	}
 
 	if c.overrideSignerType.IsV2() {
-		if size >= 0 && size < minPartSize {
+		if size > 0 && size < minPartSize {
 			return c.putObjectNoChecksum(bucketName, objectName, reader, size, metadata, progress)
 		}
 		return c.putObjectMultipart(bucketName, objectName, reader, size, metadata, progress)
@@ -203,8 +203,6 @@ func (c Client) putObjectCommon(bucketName, objectName string, reader io.Reader,
 	// If size cannot be found on a stream, it is not possible
 	// to upload using streaming signature, fall back to multipart.
 	if size < 0 {
-		// Set regular signature calculation.
-		c.overrideSignerType = credentials.SignatureV4
 		return c.putObjectMultipart(bucketName, objectName, reader, size, metadata, progress)
 	}
 

vendor/src/github.com/minio/minio-go/api-stat.go

@@ -167,6 +167,11 @@ func (c Client) statObject(bucketName, objectName string, reqHeaders RequestHead
 		contentType = "application/octet-stream"
 	}
 
+	// Extract only the relevant header keys describing the object.
+	// following function filters out a list of standard set of keys
+	// which are not part of object metadata.
+	metadata := extractObjMetadata(resp.Header)
+
 	// Save object metadata info.
 	return ObjectInfo{
 		ETag:         md5sum,
@@ -174,9 +179,6 @@ func (c Client) statObject(bucketName, objectName string, reqHeaders RequestHead
 		Size:         size,
 		LastModified: date,
 		ContentType:  contentType,
-		// Extract only the relevant header keys describing the object.
+		Metadata:     metadata,
-		// following function filters out a list of standard set of keys
-		// which are not part of object metadata.
-		Metadata: extractObjMetadata(resp.Header),
 	}, nil
 }

vendor/src/github.com/minio/minio-go/api.go

@@ -211,7 +211,7 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re
 
 	// Instantiate http client and bucket location cache.
 	clnt.httpClient = &http.Client{
-		Transport:     defaultMinioTransport,
+		Transport:     http.DefaultTransport,
 		CheckRedirect: redirectHeaders,
 	}
 

vendor/src/github.com/minio/minio-go/bucket-cache.go

@@ -213,24 +213,20 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro
 		signerType = credentials.SignatureAnonymous
 	}
 
-	if signerType.IsAnonymous() {
-		return req, nil
-	}
-
-	if signerType.IsV2() {
-		req = s3signer.SignV2(*req, accessKeyID, secretAccessKey)
-		return req, nil
-	}
-
 	// Set sha256 sum for signature calculation only with signature version '4'.
-	var contentSha256 string
+	switch {
-	if c.secure {
+	case signerType.IsV4():
-		contentSha256 = unsignedPayload
+		var contentSha256 string
-	} else {
+		if c.secure {
-		contentSha256 = hex.EncodeToString(sum256([]byte{}))
+			contentSha256 = unsignedPayload
+		} else {
+			contentSha256 = hex.EncodeToString(sum256([]byte{}))
+		}
+		req.Header.Set("X-Amz-Content-Sha256", contentSha256)
+		req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
+	case signerType.IsV2():
+		req = s3signer.SignV2(*req, accessKeyID, secretAccessKey)
 	}
 
-	req.Header.Set("X-Amz-Content-Sha256", contentSha256)
-	req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
 	return req, nil
 }

vendor/src/github.com/minio/minio-go/core_test.go

@@ -18,15 +18,14 @@ package minio
 
 import (
 	"bytes"
+	"crypto/md5"
+
 	"io"
-	"log"
+	"math/rand"
 	"os"
 	"reflect"
 	"testing"
 	"time"
-
-	"crypto/md5"
-	"math/rand"
 )
 
 const (
@@ -36,33 +35,6 @@ const (
 	enableSecurity = "ENABLE_HTTPS"
 )
 
-// Minimum part size
-const MinPartSize = 1024 * 1024 * 64
-const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
-const (
-	letterIdxBits = 6                    // 6 bits to represent a letter index
-	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
-	letterIdxMax  = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
-)
-
-// randString generates random names and prepends them with a known prefix.
-func randString(n int, src rand.Source, prefix string) string {
-	b := make([]byte, n)
-	// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
-	for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
-		if remain == 0 {
-			cache, remain = src.Int63(), letterIdxMax
-		}
-		if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
-			b[i] = letterBytes[idx]
-			i--
-		}
-		cache >>= letterIdxBits
-		remain--
-	}
-	return prefix + string(b[0:30-len(prefix)])
-}
-
 // Tests for Core GetObject() function.
 func TestGetObjectCore(t *testing.T) {
 	if testing.Short() {
@@ -237,76 +209,6 @@ func TestGetObjectCore(t *testing.T) {
 	}
 }
 
-// Tests GetObject to return Content-Encoding properly set
-// and overrides any auto decoding.
-func TestGetObjectContentEncoding(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping functional tests for the short runs")
-	}
-
-	// Seed random based on current time.
-	rand.Seed(time.Now().Unix())
-
-	// Instantiate new minio core client object.
-	c, err := NewCore(
-		os.Getenv(serverEndpoint),
-		os.Getenv(accessKey),
-		os.Getenv(secretKey),
-		mustParseBool(os.Getenv(enableSecurity)),
-	)
-	if err != nil {
-		t.Fatal("Error:", err)
-	}
-
-	// Enable tracing, write to stderr.
-	// c.TraceOn(os.Stderr)
-
-	// Set user agent.
-	c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
-
-	// Generate a new random bucket name.
-	bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
-
-	// Make a new bucket.
-	err = c.MakeBucket(bucketName, "us-east-1")
-	if err != nil {
-		t.Fatal("Error:", err, bucketName)
-	}
-
-	// Generate data more than 32K
-	buf := bytes.Repeat([]byte("3"), rand.Intn(1<<20)+32*1024)
-	m := make(map[string][]string)
-	m["Content-Encoding"] = []string{"gzip"}
-
-	// Save the data
-	objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
-	n, err := c.Client.PutObjectWithMetadata(bucketName, objectName, bytes.NewReader(buf), m, nil)
-	if err != nil {
-		t.Fatal("Error:", err, bucketName, objectName)
-	}
-
-	if n != int64(len(buf)) {
-		t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n)
-	}
-
-	reqHeaders := NewGetReqHeaders()
-	rwc, objInfo, err := c.GetObject(bucketName, objectName, reqHeaders)
-	if err != nil {
-		t.Fatalf("Error: %v", err)
-	}
-	rwc.Close()
-	if objInfo.Size <= 0 {
-		t.Fatalf("Unexpected size of the object %v, expected %v", objInfo.Size, n)
-	}
-	value, ok := objInfo.Metadata["Content-Encoding"]
-	if !ok {
-		t.Fatalf("Expected Content-Encoding metadata to be set.")
-	}
-	if value[0] != "gzip" {
-		t.Fatalf("Unexpected content-encoding found, want gzip, got %v", value)
-	}
-}
-
 // Tests get bucket policy core API.
 func TestGetBucketPolicy(t *testing.T) {
 	if testing.Short() {
@@ -471,48 +373,3 @@ func TestCorePutObject(t *testing.T) {
 		t.Fatal("Error:", err)
 	}
 }
-
-func TestCoreGetObjectMetadata(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping functional tests for the short runs")
-	}
-
-	core, err := NewCore(
-		os.Getenv(serverEndpoint),
-		os.Getenv(accessKey),
-		os.Getenv(secretKey),
-		mustParseBool(os.Getenv(enableSecurity)))
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	// Generate a new random bucket name.
-	bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
-
-	// Make a new bucket.
-	err = core.MakeBucket(bucketName, "us-east-1")
-	if err != nil {
-		t.Fatal("Error:", err, bucketName)
-	}
-
-	metadata := map[string][]string{
-		"X-Amz-Meta-Key-1": {"Val-1"},
-	}
-
-	_, err = core.PutObject(bucketName, "my-objectname", 5,
-		bytes.NewReader([]byte("hello")), nil, nil, metadata)
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	reader, objInfo, err := core.GetObject(bucketName, "my-objectname",
-		RequestHeaders{})
-	if err != nil {
-		log.Fatalln(err)
-	}
-	defer reader.Close()
-
-	if objInfo.Metadata.Get("X-Amz-Meta-Key-1") != "Val-1" {
-		log.Fatalln("Expected metadata to be available but wasn't")
-	}
-}

vendor/src/github.com/minio/minio-go/docs/API.md

@@ -531,13 +531,9 @@ __Example__
 src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil)
 
 // Destination object
-dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil)
+dst := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil)
-if err != nil {
-    fmt.Println(err)
-    return
-}
 
-// Copy object call
+/ Copy object call
 err = s3Client.CopyObject(dst, src)
 if err != nil {
     fmt.Println(err)
@@ -566,13 +562,9 @@ src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 23, 0, 0, 0, 0, time.UTC)
 src.SetRange(0, 1024*1024-1)
 
 // Destination object
-dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil)
+dst := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil)
-if err != nil {
-    fmt.Println(err)
-    return
-}
 
-// Copy object call
+/ Copy object call
 err = s3Client.CopyObject(dst, src)
 if err != nil {
     fmt.Println(err)
@@ -656,7 +648,7 @@ src := NewSourceInfo("bucket", "object", decKey)
 ```
 
 <a name="NewDestinationInfo"></a>
-### NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo, userMeta map[string]string) (DestinationInfo, error)
+### NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo, userMeta map[string]string) DestinationInfo
 
 Construct a `DestinationInfo` object that can be used as the destination object for server-side copying operations like `CopyObject` and `ComposeObject`.
 
@@ -673,11 +665,11 @@ __Example__
 
 ``` go
 // No encryption parameter.
-dst, err := NewDestinationInfo("bucket", "object", nil, nil)
+src := NewDestinationInfo("bucket", "object", nil, nil)
 
 // With encryption parameter.
 encKey := NewSSEKey([]byte{1,2,3}, "")
-dst, err := NewDecryptionInfo("bucket", "object", encKey, nil)
+src := NewDecryptionInfo("bucket", "object", encKey, nil)
 ```
 
 

vendor/src/github.com/minio/minio-go/examples/s3/composeobject.go

@@ -47,14 +47,14 @@ func main() {
 
 	// Source objects to concatenate. We also specify decryption
 	// key for each
-	src1 := minio.NewSourceInfo("bucket1", "object1", &decKey)
+	src1 := minio.NewSourceInfo("bucket1", "object1", decKey)
-	src1.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a")
+	src1.SetMatchETag("31624deb84149d2f8ef9c385918b653a")
 
-	src2 := minio.NewSourceInfo("bucket2", "object2", &decKey)
+	src2 := minio.NewSourceInfo("bucket2", "object2", decKey)
-	src2.SetMatchETagCond("f8ef9c385918b653a31624deb84149d2")
+	src2.SetMatchETag("f8ef9c385918b653a31624deb84149d2")
 
-	src3 := minio.NewSourceInfo("bucket3", "object3", &decKey)
+	src3 := minio.NewSourceInfo("bucket3", "object3", decKey)
-	src3.SetMatchETagCond("5918b653a31624deb84149d2f8ef9c38")
+	src3.SetMatchETag("5918b653a31624deb84149d2f8ef9c38")
 
 	// Create slice of sources.
 	srcs := []minio.SourceInfo{src1, src2, src3}
@@ -63,14 +63,11 @@ func main() {
 	encKey := minio.NewSSEInfo([]byte{8, 9, 0}, "")
 
 	// Create destination info
-	dst, err := minio.NewDestinationInfo("bucket", "object", &encKey, nil)
+	dst := minio.NewDestinationInfo("bucket", "object", encKey)
-	if err != nil {
-		log.Fatalln(err)
-	}
-
 	err = s3Client.ComposeObject(dst, srcs)
 	if err != nil {
-		log.Fatalln(err)
+		log.Println(err)
+		return
 	}
 
 	log.Println("Composed object successfully.")

vendor/src/github.com/minio/minio-go/examples/s3/copyobject.go

@@ -60,10 +60,7 @@ func main() {
 	// src.SetMatchETagExceptCond("31624deb84149d2f8ef9c385918b653a")
 
 	// Destination object
-	dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil)
+	dst := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil)
-	if err != nil {
-		log.Fatalln(err)
-	}
 
 	// Initiate copy object.
 	err = s3Client.CopyObject(dst, src)

vendor/src/github.com/minio/minio-go/examples/s3/putobject-progress.go

@@ -55,9 +55,7 @@ func main() {
 	progress := pb.New64(objectInfo.Size)
 	progress.Start()
 
-	n, err := s3Client.PutObjectWithProgress("my-bucketname", "my-objectname-progress", reader, map[string][]string{
+	n, err := s3Client.PutObjectWithProgress("my-bucketname", "my-objectname-progress", reader, "application/octet-stream", progress)
-		"Content-Type": []string{"application/octet-stream"},
-	}, progress)
 	if err != nil {
 		log.Fatalln(err)
 	}

vendor/src/github.com/minio/minio-go/examples/s3/putobject-s3-accelerate.go

@@ -40,7 +40,7 @@ func main() {
 	}
 
 	// Enable S3 transfer accelerate endpoint.
-	s3Client.SetS3TransferAccelerate("s3-accelerate.amazonaws.com")
+	s3Client.S3TransferAccelerate("s3-accelerate.amazonaws.com")
 
 	object, err := os.Open("my-testfile")
 	if err != nil {

vendor/src/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go

@@ -99,7 +99,7 @@ func prepareStreamingRequest(req *http.Request, sessionToken string, dataLen int
 	if sessionToken != "" {
 		req.Header.Set("X-Amz-Security-Token", sessionToken)
 	}
-	req.Header.Add("Content-Encoding", streamingEncoding)
+	req.Header.Set("Content-Encoding", streamingEncoding)
 	req.Header.Set("X-Amz-Date", timestamp.Format(iso8601DateFormat))
 
 	// Set content length with streaming signature for each chunk included.

vendor/src/github.com/minio/minio-go/test-utils_test.go

@@ -64,11 +64,11 @@ func encodeResponse(response interface{}) []byte {
 	return bytesBuffer.Bytes()
 }
 
-// Convert string to bool and always return false if any error
+// Convert string to bool and always return true if any error
 func mustParseBool(str string) bool {
 	b, err := strconv.ParseBool(str)
 	if err != nil {
-		return false
+		return true
 	}
 	return b
 }

vendor/src/github.com/minio/minio-go/transport.go

@@ -1,48 +0,0 @@
-// +build go1.7 go1.8
-
-/*
- * Minio Go Library for Amazon S3 Compatible Cloud Storage
- * (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package minio
-
-import (
-	"net"
-	"net/http"
-	"time"
-)
-
-// This default transport is similar to http.DefaultTransport
-// but with additional DisableCompression:
-var defaultMinioTransport http.RoundTripper = &http.Transport{
-	Proxy: http.ProxyFromEnvironment,
-	DialContext: (&net.Dialer{
-		Timeout:   30 * time.Second,
-		KeepAlive: 30 * time.Second,
-		DualStack: true,
-	}).DialContext,
-	MaxIdleConns:          100,
-	IdleConnTimeout:       90 * time.Second,
-	TLSHandshakeTimeout:   10 * time.Second,
-	ExpectContinueTimeout: 1 * time.Second,
-	// Set this value so that the underlying transport round-tripper
-	// doesn't try to auto decode the body of objects with
-	// content-encoding set to `gzip`.
-	//
-	// Refer:
-	//    https://golang.org/src/net/http/transport.go?h=roundTrip#L1843
-	DisableCompression: true,
-}

vendor/src/github.com/minio/minio-go/transport_1_5.go

@@ -1,39 +0,0 @@
-// +build go1.5,!go1.6,!go1.7,!go1.8
-
-/*
- * Minio Go Library for Amazon S3 Compatible Cloud Storage
- * (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package minio
-
-import (
-	"net/http"
-	"time"
-)
-
-// This default transport is similar to http.DefaultTransport
-// but with additional DisableCompression:
-var defaultMinioTransport http.RoundTripper = &http.Transport{
-	Proxy:               http.ProxyFromEnvironment,
-	TLSHandshakeTimeout: 10 * time.Second,
-	// Set this value so that the underlying transport round-tripper
-	// doesn't try to auto decode the body of objects with
-	// content-encoding set to `gzip`.
-	//
-	// Refer:
-	//    https://golang.org/src/net/http/transport.go?h=roundTrip#L1843
-	DisableCompression: true,
-}

vendor/src/github.com/minio/minio-go/transport_1_6.go

@@ -1,40 +0,0 @@
-// +build go1.6,!go1.7,!go1.8
-
-/*
- * Minio Go Library for Amazon S3 Compatible Cloud Storage
- * (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package minio
-
-import (
-	"net/http"
-	"time"
-)
-
-// This default transport is similar to http.DefaultTransport
-// but with additional DisableCompression:
-var defaultMinioTransport http.RoundTripper = &http.Transport{
-	Proxy:                 http.ProxyFromEnvironment,
-	TLSHandshakeTimeout:   10 * time.Second,
-	ExpectContinueTimeout: 1 * time.Second,
-	// Set this value so that the underlying transport round-tripper
-	// doesn't try to auto decode the body of objects with
-	// content-encoding set to `gzip`.
-	//
-	// Refer:
-	//    https://golang.org/src/net/http/transport.go?h=roundTrip#L1843
-	DisableCompression: true,
-}