octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-02 03:42:31 +00:00
Code Issues Releases Wiki Activity

Merge pull request #1124 from restic/use-minio-300

Browse Source 
Set minio-go to v3.0.0
This commit is contained in:
Alexander Neumann 2017-07-22 11:01:57 +02:00
commit 7fb1352aa1
20 changed files with 3933 additions and 4407 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
vendor/manifest vendored
View File

@ -46,8 +46,8 @@
{
"importpath": "github.com/minio/minio-go",
"repository": "https://github.com/minio/minio-go",
"revision": "bd8e1d8a93f006a0207e026353bf0644ffcdd320",
"branch": "master"
"revision": "5ca66c9a35ba1cd674484be99dc97aa0973afe12",
"branch": "HEAD"
},
{
"importpath": "github.com/ncw/swift",

18
vendor/src/github.com/minio/minio-go/api-get-object.go vendored
View File

@ -679,18 +679,12 @@ func (c Client) getObject(bucketName, objectName string, reqHeaders RequestHeade
if contentType == "" {
contentType = "application/octet-stream"
}
objectStat := ObjectInfo{
ETag: md5sum,
Key: objectName,
Size: resp.ContentLength,
LastModified: date,
ContentType: contentType,
// Extract only the relevant header keys describing the object.
// following function filters out a list of standard set of keys
// which are not part of object metadata.
Metadata: extractObjMetadata(resp.Header),
}
var objectStat ObjectInfo
objectStat.ETag = md5sum
objectStat.Key = objectName
objectStat.Size = resp.ContentLength
objectStat.LastModified = date
objectStat.ContentType = contentType
// do not close body here, caller will close
return resp.Body, objectStat, nil

4
vendor/src/github.com/minio/minio-go/api-put-object.go vendored
View File

@ -194,7 +194,7 @@ func (c Client) putObjectCommon(bucketName, objectName string, reader io.Reader,
}
if c.overrideSignerType.IsV2() {
if size >= 0 && size < minPartSize {
if size > 0 && size < minPartSize {
return c.putObjectNoChecksum(bucketName, objectName, reader, size, metadata, progress)
}
return c.putObjectMultipart(bucketName, objectName, reader, size, metadata, progress)
@ -203,8 +203,6 @@ func (c Client) putObjectCommon(bucketName, objectName string, reader io.Reader,
// If size cannot be found on a stream, it is not possible
// to upload using streaming signature, fall back to multipart.
if size < 0 {
// Set regular signature calculation.
c.overrideSignerType = credentials.SignatureV4
return c.putObjectMultipart(bucketName, objectName, reader, size, metadata, progress)
}

10
vendor/src/github.com/minio/minio-go/api-stat.go vendored
View File

@ -167,6 +167,11 @@ func (c Client) statObject(bucketName, objectName string, reqHeaders RequestHead
contentType = "application/octet-stream"
}
// Extract only the relevant header keys describing the object.
// following function filters out a list of standard set of keys
// which are not part of object metadata.
metadata := extractObjMetadata(resp.Header)
// Save object metadata info.
return ObjectInfo{
ETag: md5sum,
@ -174,9 +179,6 @@ func (c Client) statObject(bucketName, objectName string, reqHeaders RequestHead
Size: size,
LastModified: date,
ContentType: contentType,
// Extract only the relevant header keys describing the object.
// following function filters out a list of standard set of keys
// which are not part of object metadata.
Metadata: extractObjMetadata(resp.Header),
Metadata: metadata,
}, nil
}

2
vendor/src/github.com/minio/minio-go/api.go vendored
View File

@ -211,7 +211,7 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re
// Instantiate http client and bucket location cache.
clnt.httpClient = &http.Client{
Transport: defaultMinioTransport,
Transport: http.DefaultTransport,
CheckRedirect: redirectHeaders,
}

1470
vendor/src/github.com/minio/minio-go/api_functional_v2_test.go vendored Normal file
View File

File diff suppressed because it is too large Load Diff

2410
vendor/src/github.com/minio/minio-go/api_functional_v4_test.go vendored Normal file
View File

File diff suppressed because it is too large Load Diff

28
vendor/src/github.com/minio/minio-go/bucket-cache.go vendored
View File

@ -213,24 +213,20 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro
signerType = credentials.SignatureAnonymous
}
if signerType.IsAnonymous() {
return req, nil
}
if signerType.IsV2() {
req = s3signer.SignV2(*req, accessKeyID, secretAccessKey)
return req, nil
}
// Set sha256 sum for signature calculation only with signature version '4'.
var contentSha256 string
if c.secure {
contentSha256 = unsignedPayload
} else {
contentSha256 = hex.EncodeToString(sum256([]byte{}))
switch {
case signerType.IsV4():
var contentSha256 string
if c.secure {
contentSha256 = unsignedPayload
} else {
contentSha256 = hex.EncodeToString(sum256([]byte{}))
}
req.Header.Set("X-Amz-Content-Sha256", contentSha256)
req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
case signerType.IsV2():
req = s3signer.SignV2(*req, accessKeyID, secretAccessKey)
}
req.Header.Set("X-Amz-Content-Sha256", contentSha256)
req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
return req, nil
}

149
vendor/src/github.com/minio/minio-go/core_test.go vendored
View File

@ -18,15 +18,14 @@ package minio
import (
"bytes"
"crypto/md5"
"io"
"log"
"math/rand"
"os"
"reflect"
"testing"
"time"
"crypto/md5"
"math/rand"
)
const (
@ -36,33 +35,6 @@ const (
enableSecurity = "ENABLE_HTTPS"
)
// Minimum part size
const MinPartSize = 1024 * 1024 * 64
const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
const (
letterIdxBits = 6 // 6 bits to represent a letter index
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
)
// randString generates random names and prepends them with a known prefix.
func randString(n int, src rand.Source, prefix string) string {
b := make([]byte, n)
// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
if remain == 0 {
cache, remain = src.Int63(), letterIdxMax
}
if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
b[i] = letterBytes[idx]
i--
}
cache >>= letterIdxBits
remain--
}
return prefix + string(b[0:30-len(prefix)])
}
// Tests for Core GetObject() function.
func TestGetObjectCore(t *testing.T) {
if testing.Short() {
@ -237,76 +209,6 @@ func TestGetObjectCore(t *testing.T) {
}
}
// Tests GetObject to return Content-Encoding properly set
// and overrides any auto decoding.
func TestGetObjectContentEncoding(t *testing.T) {
if testing.Short() {
t.Skip("skipping functional tests for the short runs")
}
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Instantiate new minio core client object.
c, err := NewCore(
os.Getenv(serverEndpoint),
os.Getenv(accessKey),
os.Getenv(secretKey),
mustParseBool(os.Getenv(enableSecurity)),
)
if err != nil {
t.Fatal("Error:", err)
}
// Enable tracing, write to stderr.
// c.TraceOn(os.Stderr)
// Set user agent.
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
// Generate a new random bucket name.
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
// Make a new bucket.
err = c.MakeBucket(bucketName, "us-east-1")
if err != nil {
t.Fatal("Error:", err, bucketName)
}
// Generate data more than 32K
buf := bytes.Repeat([]byte("3"), rand.Intn(1<<20)+32*1024)
m := make(map[string][]string)
m["Content-Encoding"] = []string{"gzip"}
// Save the data
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
n, err := c.Client.PutObjectWithMetadata(bucketName, objectName, bytes.NewReader(buf), m, nil)
if err != nil {
t.Fatal("Error:", err, bucketName, objectName)
}
if n != int64(len(buf)) {
t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n)
}
reqHeaders := NewGetReqHeaders()
rwc, objInfo, err := c.GetObject(bucketName, objectName, reqHeaders)
if err != nil {
t.Fatalf("Error: %v", err)
}
rwc.Close()
if objInfo.Size <= 0 {
t.Fatalf("Unexpected size of the object %v, expected %v", objInfo.Size, n)
}
value, ok := objInfo.Metadata["Content-Encoding"]
if !ok {
t.Fatalf("Expected Content-Encoding metadata to be set.")
}
if value[0] != "gzip" {
t.Fatalf("Unexpected content-encoding found, want gzip, got %v", value)
}
}
// Tests get bucket policy core API.
func TestGetBucketPolicy(t *testing.T) {
if testing.Short() {
@ -471,48 +373,3 @@ func TestCorePutObject(t *testing.T) {
t.Fatal("Error:", err)
}
}
func TestCoreGetObjectMetadata(t *testing.T) {
if testing.Short() {
t.Skip("skipping functional tests for the short runs")
}
core, err := NewCore(
os.Getenv(serverEndpoint),
os.Getenv(accessKey),
os.Getenv(secretKey),
mustParseBool(os.Getenv(enableSecurity)))
if err != nil {
log.Fatalln(err)
}
// Generate a new random bucket name.
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test")
// Make a new bucket.
err = core.MakeBucket(bucketName, "us-east-1")
if err != nil {
t.Fatal("Error:", err, bucketName)
}
metadata := map[string][]string{
"X-Amz-Meta-Key-1": {"Val-1"},
}
_, err = core.PutObject(bucketName, "my-objectname", 5,
bytes.NewReader([]byte("hello")), nil, nil, metadata)
if err != nil {
log.Fatalln(err)
}
reader, objInfo, err := core.GetObject(bucketName, "my-objectname",
RequestHeaders{})
if err != nil {
log.Fatalln(err)
}
defer reader.Close()
if objInfo.Metadata.Get("X-Amz-Meta-Key-1") != "Val-1" {
log.Fatalln("Expected metadata to be available but wasn't")
}
}

22
vendor/src/github.com/minio/minio-go/docs/API.md vendored
View File

@ -531,13 +531,9 @@ __Example__
src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil)
// Destination object
dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil)
if err != nil {
fmt.Println(err)
return
}
dst := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil)
// Copy object call
/ Copy object call
err = s3Client.CopyObject(dst, src)
if err != nil {
fmt.Println(err)
@ -566,13 +562,9 @@ src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 23, 0, 0, 0, 0, time.UTC)
src.SetRange(0, 1024*1024-1)
// Destination object
dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil)
if err != nil {
fmt.Println(err)
return
}
dst := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil)
// Copy object call
/ Copy object call
err = s3Client.CopyObject(dst, src)
if err != nil {
fmt.Println(err)
@ -656,7 +648,7 @@ src := NewSourceInfo("bucket", "object", decKey)
```
<a name="NewDestinationInfo"></a>
### NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo, userMeta map[string]string) (DestinationInfo, error)
### NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo, userMeta map[string]string) DestinationInfo
Construct a `DestinationInfo` object that can be used as the destination object for server-side copying operations like `CopyObject` and `ComposeObject`.
@ -673,11 +665,11 @@ __Example__
``` go
// No encryption parameter.
dst, err := NewDestinationInfo("bucket", "object", nil, nil)
src := NewDestinationInfo("bucket", "object", nil, nil)
// With encryption parameter.
encKey := NewSSEKey([]byte{1,2,3}, "")
dst, err := NewDecryptionInfo("bucket", "object", encKey, nil)
src := NewDecryptionInfo("bucket", "object", encKey, nil)
```

21
vendor/src/github.com/minio/minio-go/examples/s3/composeobject.go vendored
View File

@ -47,14 +47,14 @@ func main() {
// Source objects to concatenate. We also specify decryption
// key for each
src1 := minio.NewSourceInfo("bucket1", "object1", &decKey)
src1.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a")
src1 := minio.NewSourceInfo("bucket1", "object1", decKey)
src1.SetMatchETag("31624deb84149d2f8ef9c385918b653a")
src2 := minio.NewSourceInfo("bucket2", "object2", &decKey)
src2.SetMatchETagCond("f8ef9c385918b653a31624deb84149d2")
src2 := minio.NewSourceInfo("bucket2", "object2", decKey)
src2.SetMatchETag("f8ef9c385918b653a31624deb84149d2")
src3 := minio.NewSourceInfo("bucket3", "object3", &decKey)
src3.SetMatchETagCond("5918b653a31624deb84149d2f8ef9c38")
src3 := minio.NewSourceInfo("bucket3", "object3", decKey)
src3.SetMatchETag("5918b653a31624deb84149d2f8ef9c38")
// Create slice of sources.
srcs := []minio.SourceInfo{src1, src2, src3}
@ -63,14 +63,11 @@ func main() {
encKey := minio.NewSSEInfo([]byte{8, 9, 0}, "")
// Create destination info
dst, err := minio.NewDestinationInfo("bucket", "object", &encKey, nil)
if err != nil {
log.Fatalln(err)
}
dst := minio.NewDestinationInfo("bucket", "object", encKey)
err = s3Client.ComposeObject(dst, srcs)
if err != nil {
log.Fatalln(err)
log.Println(err)
return
}
log.Println("Composed object successfully.")

5
vendor/src/github.com/minio/minio-go/examples/s3/copyobject.go vendored
View File

@ -60,10 +60,7 @@ func main() {
// src.SetMatchETagExceptCond("31624deb84149d2f8ef9c385918b653a")
// Destination object
dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil)
if err != nil {
log.Fatalln(err)
}
dst := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil)
// Initiate copy object.
err = s3Client.CopyObject(dst, src)

4
vendor/src/github.com/minio/minio-go/examples/s3/putobject-progress.go vendored
View File

@ -55,9 +55,7 @@ func main() {
progress := pb.New64(objectInfo.Size)
progress.Start()
n, err := s3Client.PutObjectWithProgress("my-bucketname", "my-objectname-progress", reader, map[string][]string{
"Content-Type": []string{"application/octet-stream"},
}, progress)
n, err := s3Client.PutObjectWithProgress("my-bucketname", "my-objectname-progress", reader, "application/octet-stream", progress)
if err != nil {
log.Fatalln(err)
}

2
vendor/src/github.com/minio/minio-go/examples/s3/putobject-s3-accelerate.go vendored
View File

@ -40,7 +40,7 @@ func main() {
}
// Enable S3 transfer accelerate endpoint.
s3Client.SetS3TransferAccelerate("s3-accelerate.amazonaws.com")
s3Client.S3TransferAccelerate("s3-accelerate.amazonaws.com")
object, err := os.Open("my-testfile")
if err != nil {

4058
vendor/src/github.com/minio/minio-go/functional_tests.go vendored
View File

File diff suppressed because it is too large Load Diff

2
vendor/src/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go vendored
View File

@ -99,7 +99,7 @@ func prepareStreamingRequest(req *http.Request, sessionToken string, dataLen int
if sessionToken != "" {
req.Header.Set("X-Amz-Security-Token", sessionToken)
}
req.Header.Add("Content-Encoding", streamingEncoding)
req.Header.Set("Content-Encoding", streamingEncoding)
req.Header.Set("X-Amz-Date", timestamp.Format(iso8601DateFormat))
// Set content length with streaming signature for each chunk included.

4
vendor/src/github.com/minio/minio-go/test-utils_test.go vendored
View File

@ -64,11 +64,11 @@ func encodeResponse(response interface{}) []byte {
return bytesBuffer.Bytes()
}
// Convert string to bool and always return false if any error
// Convert string to bool and always return true if any error
func mustParseBool(str string) bool {
b, err := strconv.ParseBool(str)
if err != nil {
return false
return true
}
return b
}

48
vendor/src/github.com/minio/minio-go/transport.go vendored
View File

@ -1,48 +0,0 @@
// +build go1.7 go1.8
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage
* (C) 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"net"
"net/http"
"time"
)
// This default transport is similar to http.DefaultTransport
// but with additional DisableCompression:
var defaultMinioTransport http.RoundTripper = &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
// Set this value so that the underlying transport round-tripper
// doesn't try to auto decode the body of objects with
// content-encoding set to `gzip`.
//
// Refer:
// https://golang.org/src/net/http/transport.go?h=roundTrip#L1843
DisableCompression: true,
}

39
vendor/src/github.com/minio/minio-go/transport_1_5.go vendored
View File

@ -1,39 +0,0 @@
// +build go1.5,!go1.6,!go1.7,!go1.8
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage
* (C) 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"net/http"
"time"
)
// This default transport is similar to http.DefaultTransport
// but with additional DisableCompression:
var defaultMinioTransport http.RoundTripper = &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSHandshakeTimeout: 10 * time.Second,
// Set this value so that the underlying transport round-tripper
// doesn't try to auto decode the body of objects with
// content-encoding set to `gzip`.
//
// Refer:
// https://golang.org/src/net/http/transport.go?h=roundTrip#L1843
DisableCompression: true,
}

40
vendor/src/github.com/minio/minio-go/transport_1_6.go vendored
View File

@ -1,40 +0,0 @@
// +build go1.6,!go1.7,!go1.8
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage
* (C) 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"net/http"
"time"
)
// This default transport is similar to http.DefaultTransport
// but with additional DisableCompression:
var defaultMinioTransport http.RoundTripper = &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
// Set this value so that the underlying transport round-tripper
// doesn't try to auto decode the body of objects with
// content-encoding set to `gzip`.
//
// Refer:
// https://golang.org/src/net/http/transport.go?h=roundTrip#L1843
DisableCompression: true,
}