From 10cd672a9245ffddb8e1a1d8839c8beee5a0d799 Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Wed, 26 Aug 2015 06:25:05 -0500 Subject: [PATCH 1/3] Use new version of s3 library, Fixes #276 --- Godeps/Godeps.json | 24 +- .../src/github.com/mitchellh/goamz/aws/aws.go | 445 --------- .../mitchellh/goamz/aws/aws_test.go | 203 ---- .../github.com/mitchellh/goamz/aws/client.go | 125 --- .../mitchellh/goamz/aws/client_test.go | 121 --- .../src/github.com/mitchellh/goamz/s3/s3.go | 893 ------------------ .../src/github.com/mitchellh/goamz/s3/sign.go | 126 --- .../mitchellh/goamz/s3/sign_test.go | 194 ---- .../src/github.com/vaughan0/go-ini/LICENSE | 14 - .../src/github.com/vaughan0/go-ini/README.md | 70 -- .../src/github.com/vaughan0/go-ini/ini.go | 123 --- .../vaughan0/go-ini/ini_linux_test.go | 43 - .../github.com/vaughan0/go-ini/ini_test.go | 89 -- .../src/github.com/vaughan0/go-ini/test.ini | 2 - .../golang.org/x/net/context/context_test.go | 2 +- .../x/net/context/ctxhttp/cancelreq.go | 18 + .../x/net/context/ctxhttp/cancelreq_go14.go | 23 + .../x/net/context/ctxhttp/ctxhttp.go | 79 ++ .../x/net/context/ctxhttp/ctxhttp_test.go | 72 ++ .../goamz => gopkg.in/amz.v3}/aws/attempt.go | 0 .../amz.v3}/aws/attempt_test.go | 6 +- .../_workspace/src/gopkg.in/amz.v3/aws/aws.go | 268 ++++++ .../src/gopkg.in/amz.v3/aws/aws_test.go | 84 ++ .../src/gopkg.in/amz.v3/aws/sign.go | 447 +++++++++ .../src/gopkg.in/amz.v3/aws/sign_test.go | 285 ++++++ .../amz.v3}/s3/export_test.go | 12 +- .../goamz => gopkg.in/amz.v3}/s3/multi.go | 357 ++++--- .../amz.v3}/s3/multi_test.go | 61 +- .../amz.v3}/s3/responses_test.go | 43 - .../_workspace/src/gopkg.in/amz.v3/s3/s3.go | 566 +++++++++++ .../goamz => gopkg.in/amz.v3}/s3/s3_test.go | 282 ++---- .../goamz => gopkg.in/amz.v3}/s3/s3i_test.go | 180 ++-- .../goamz => gopkg.in/amz.v3}/s3/s3t_test.go | 12 +- .../amz.v3}/s3/s3test/server.go | 41 +- backend/s3/s3.go | 19 +- backend/s3_test.go | 6 +- 36 files changed, 2323 insertions(+), 3012 deletions(-) delete mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws.go delete mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws_test.go delete mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client.go delete mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client_test.go delete mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3.go delete mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign.go delete mode 100644 Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign_test.go delete mode 100644 Godeps/_workspace/src/github.com/vaughan0/go-ini/LICENSE delete mode 100644 Godeps/_workspace/src/github.com/vaughan0/go-ini/README.md delete mode 100644 Godeps/_workspace/src/github.com/vaughan0/go-ini/ini.go delete mode 100644 Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_linux_test.go delete mode 100644 Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_test.go delete mode 100644 Godeps/_workspace/src/github.com/vaughan0/go-ini/test.ini create mode 100644 Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/cancelreq.go create mode 100644 Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/cancelreq_go14.go create mode 100644 Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/ctxhttp.go create mode 100644 Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/ctxhttp_test.go rename Godeps/_workspace/src/{github.com/mitchellh/goamz => gopkg.in/amz.v3}/aws/attempt.go (100%) rename Godeps/_workspace/src/{github.com/mitchellh/goamz => gopkg.in/amz.v3}/aws/attempt_test.go (95%) create mode 100644 Godeps/_workspace/src/gopkg.in/amz.v3/aws/aws.go create mode 100644 Godeps/_workspace/src/gopkg.in/amz.v3/aws/aws_test.go create mode 100644 Godeps/_workspace/src/gopkg.in/amz.v3/aws/sign.go create mode 100644 Godeps/_workspace/src/gopkg.in/amz.v3/aws/sign_test.go rename Godeps/_workspace/src/{github.com/mitchellh/goamz => gopkg.in/amz.v3}/s3/export_test.go (62%) rename Godeps/_workspace/src/{github.com/mitchellh/goamz => gopkg.in/amz.v3}/s3/multi.go (58%) rename Godeps/_workspace/src/{github.com/mitchellh/goamz => gopkg.in/amz.v3}/s3/multi_test.go (92%) rename Godeps/_workspace/src/{github.com/mitchellh/goamz => gopkg.in/amz.v3}/s3/responses_test.go (83%) create mode 100644 Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3.go rename Godeps/_workspace/src/{github.com/mitchellh/goamz => gopkg.in/amz.v3}/s3/s3_test.go (61%) rename Godeps/_workspace/src/{github.com/mitchellh/goamz => gopkg.in/amz.v3}/s3/s3i_test.go (85%) rename Godeps/_workspace/src/{github.com/mitchellh/goamz => gopkg.in/amz.v3}/s3/s3t_test.go (86%) rename Godeps/_workspace/src/{github.com/mitchellh/goamz => gopkg.in/amz.v3}/s3/s3test/server.go (94%) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 8e0f27b54..ddd8e96b4 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -22,14 +22,6 @@ "ImportPath": "github.com/kr/fs", "Rev": "2788f0dbd16903de03cb8186e5c7d97b69ad387b" }, - { - "ImportPath": "github.com/mitchellh/goamz/aws", - "Rev": "caaaea8b30ee15616494ee68abd5d8ebbbef05cf" - }, - { - "ImportPath": "github.com/mitchellh/goamz/s3", - "Rev": "caaaea8b30ee15616494ee68abd5d8ebbbef05cf" - }, { "ImportPath": "github.com/pkg/sftp", "Rev": "518aed2757a65cfa64d4b1b2baf08410f8b7a6bc" @@ -38,10 +30,6 @@ "ImportPath": "github.com/restic/chunker", "Rev": "e795b80f4c927ebcf2687ce18bcf1a39fee740b1" }, - { - "ImportPath": "github.com/vaughan0/go-ini", - "Rev": "a98ad7ee00ec53921f08832bc06ecf7fd600e6a1" - }, { "ImportPath": "golang.org/x/crypto/pbkdf2", "Rev": "cc04154d65fb9296747569b107cfd05380b1ea3e" @@ -57,6 +45,18 @@ { "ImportPath": "golang.org/x/crypto/ssh", "Rev": "cc04154d65fb9296747569b107cfd05380b1ea3e" + }, + { + "ImportPath": "golang.org/x/net/context", + "Rev": "7654728e381988afd88e58cabfd6363a5ea91810" + }, + { + "ImportPath": "gopkg.in/amz.v3/aws", + "Rev": "bff3a097c4108da57bb8cbe3aad2990d74d23676" + }, + { + "ImportPath": "gopkg.in/amz.v3/s3", + "Rev": "bff3a097c4108da57bb8cbe3aad2990d74d23676" } ] } diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws.go deleted file mode 100644 index cfc42c03a..000000000 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws.go +++ /dev/null @@ -1,445 +0,0 @@ -// -// goamz - Go packages to interact with the Amazon Web Services. -// -// https://wiki.ubuntu.com/goamz -// -// Copyright (c) 2011 Canonical Ltd. -// -// Written by Gustavo Niemeyer -// -package aws - -import ( - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "os" - - "github.com/vaughan0/go-ini" -) - -// Region defines the URLs where AWS services may be accessed. -// -// See http://goo.gl/d8BP1 for more details. -type Region struct { - Name string // the canonical name of this region. - EC2Endpoint string - S3Endpoint string - S3BucketEndpoint string // Not needed by AWS S3. Use ${bucket} for bucket name. - S3LocationConstraint bool // true if this region requires a LocationConstraint declaration. - S3LowercaseBucket bool // true if the region requires bucket names to be lower case. - SDBEndpoint string - SNSEndpoint string - SQSEndpoint string - IAMEndpoint string - ELBEndpoint string - AutoScalingEndpoint string - RdsEndpoint string - Route53Endpoint string -} - -var USGovWest = Region{ - "us-gov-west-1", - "https://ec2.us-gov-west-1.amazonaws.com", - "https://s3-fips-us-gov-west-1.amazonaws.com", - "", - true, - true, - "", - "https://sns.us-gov-west-1.amazonaws.com", - "https://sqs.us-gov-west-1.amazonaws.com", - "https://iam.us-gov.amazonaws.com", - "https://elasticloadbalancing.us-gov-west-1.amazonaws.com", - "https://autoscaling.us-gov-west-1.amazonaws.com", - "https://rds.us-gov-west-1.amazonaws.com", - "https://route53.amazonaws.com", -} - -var USEast = Region{ - "us-east-1", - "https://ec2.us-east-1.amazonaws.com", - "https://s3.amazonaws.com", - "", - false, - false, - "https://sdb.amazonaws.com", - "https://sns.us-east-1.amazonaws.com", - "https://sqs.us-east-1.amazonaws.com", - "https://iam.amazonaws.com", - "https://elasticloadbalancing.us-east-1.amazonaws.com", - "https://autoscaling.us-east-1.amazonaws.com", - "https://rds.us-east-1.amazonaws.com", - "https://route53.amazonaws.com", -} - -var USWest = Region{ - "us-west-1", - "https://ec2.us-west-1.amazonaws.com", - "https://s3-us-west-1.amazonaws.com", - "", - true, - true, - "https://sdb.us-west-1.amazonaws.com", - "https://sns.us-west-1.amazonaws.com", - "https://sqs.us-west-1.amazonaws.com", - "https://iam.amazonaws.com", - "https://elasticloadbalancing.us-west-1.amazonaws.com", - "https://autoscaling.us-west-1.amazonaws.com", - "https://rds.us-west-1.amazonaws.com", - "https://route53.amazonaws.com", -} - -var USWest2 = Region{ - "us-west-2", - "https://ec2.us-west-2.amazonaws.com", - "https://s3-us-west-2.amazonaws.com", - "", - true, - true, - "https://sdb.us-west-2.amazonaws.com", - "https://sns.us-west-2.amazonaws.com", - "https://sqs.us-west-2.amazonaws.com", - "https://iam.amazonaws.com", - "https://elasticloadbalancing.us-west-2.amazonaws.com", - "https://autoscaling.us-west-2.amazonaws.com", - "https://rds.us-west-2.amazonaws.com", - "https://route53.amazonaws.com", -} - -var EUWest = Region{ - "eu-west-1", - "https://ec2.eu-west-1.amazonaws.com", - "https://s3-eu-west-1.amazonaws.com", - "", - true, - true, - "https://sdb.eu-west-1.amazonaws.com", - "https://sns.eu-west-1.amazonaws.com", - "https://sqs.eu-west-1.amazonaws.com", - "https://iam.amazonaws.com", - "https://elasticloadbalancing.eu-west-1.amazonaws.com", - "https://autoscaling.eu-west-1.amazonaws.com", - "https://rds.eu-west-1.amazonaws.com", - "https://route53.amazonaws.com", -} - -var EUCentral = Region{ - "eu-central-1", - "https://ec2.eu-central-1.amazonaws.com", - "https://s3-eu-central-1.amazonaws.com", - "", - true, - true, - "", - "https://sns.eu-central-1.amazonaws.com", - "https://sqs.eu-central-1.amazonaws.com", - "https://iam.amazonaws.com", - "https://elasticloadbalancing.eu-central-1.amazonaws.com", - "https://autoscaling.eu-central-1.amazonaws.com", - "https://rds.eu-central-1.amazonaws.com", - "https://route53.amazonaws.com", -} - -var APSoutheast = Region{ - "ap-southeast-1", - "https://ec2.ap-southeast-1.amazonaws.com", - "https://s3-ap-southeast-1.amazonaws.com", - "", - true, - true, - "https://sdb.ap-southeast-1.amazonaws.com", - "https://sns.ap-southeast-1.amazonaws.com", - "https://sqs.ap-southeast-1.amazonaws.com", - "https://iam.amazonaws.com", - "https://elasticloadbalancing.ap-southeast-1.amazonaws.com", - "https://autoscaling.ap-southeast-1.amazonaws.com", - "https://rds.ap-southeast-1.amazonaws.com", - "https://route53.amazonaws.com", -} - -var APSoutheast2 = Region{ - "ap-southeast-2", - "https://ec2.ap-southeast-2.amazonaws.com", - "https://s3-ap-southeast-2.amazonaws.com", - "", - true, - true, - "https://sdb.ap-southeast-2.amazonaws.com", - "https://sns.ap-southeast-2.amazonaws.com", - "https://sqs.ap-southeast-2.amazonaws.com", - "https://iam.amazonaws.com", - "https://elasticloadbalancing.ap-southeast-2.amazonaws.com", - "https://autoscaling.ap-southeast-2.amazonaws.com", - "https://rds.ap-southeast-2.amazonaws.com", - "https://route53.amazonaws.com", -} - -var APNortheast = Region{ - "ap-northeast-1", - "https://ec2.ap-northeast-1.amazonaws.com", - "https://s3-ap-northeast-1.amazonaws.com", - "", - true, - true, - "https://sdb.ap-northeast-1.amazonaws.com", - "https://sns.ap-northeast-1.amazonaws.com", - "https://sqs.ap-northeast-1.amazonaws.com", - "https://iam.amazonaws.com", - "https://elasticloadbalancing.ap-northeast-1.amazonaws.com", - "https://autoscaling.ap-northeast-1.amazonaws.com", - "https://rds.ap-northeast-1.amazonaws.com", - "https://route53.amazonaws.com", -} - -var SAEast = Region{ - "sa-east-1", - "https://ec2.sa-east-1.amazonaws.com", - "https://s3-sa-east-1.amazonaws.com", - "", - true, - true, - "https://sdb.sa-east-1.amazonaws.com", - "https://sns.sa-east-1.amazonaws.com", - "https://sqs.sa-east-1.amazonaws.com", - "https://iam.amazonaws.com", - "https://elasticloadbalancing.sa-east-1.amazonaws.com", - "https://autoscaling.sa-east-1.amazonaws.com", - "https://rds.sa-east-1.amazonaws.com", - "https://route53.amazonaws.com", -} - -var CNNorth = Region{ - "cn-north-1", - "https://ec2.cn-north-1.amazonaws.com.cn", - "https://s3.cn-north-1.amazonaws.com.cn", - "", - true, - true, - "", - "https://sns.cn-north-1.amazonaws.com.cn", - "https://sqs.cn-north-1.amazonaws.com.cn", - "https://iam.cn-north-1.amazonaws.com.cn", - "https://elasticloadbalancing.cn-north-1.amazonaws.com.cn", - "https://autoscaling.cn-north-1.amazonaws.com.cn", - "https://rds.cn-north-1.amazonaws.com.cn", - "https://route53.amazonaws.com", -} - -var Regions = map[string]Region{ - APNortheast.Name: APNortheast, - APSoutheast.Name: APSoutheast, - APSoutheast2.Name: APSoutheast2, - EUWest.Name: EUWest, - EUCentral.Name: EUCentral, - USEast.Name: USEast, - USWest.Name: USWest, - USWest2.Name: USWest2, - SAEast.Name: SAEast, - USGovWest.Name: USGovWest, - CNNorth.Name: CNNorth, -} - -type Auth struct { - AccessKey, SecretKey, Token string -} - -var unreserved = make([]bool, 128) -var hex = "0123456789ABCDEF" - -func init() { - // RFC3986 - u := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234567890-_.~" - for _, c := range u { - unreserved[c] = true - } -} - -type credentials struct { - Code string - LastUpdated string - Type string - AccessKeyId string - SecretAccessKey string - Token string - Expiration string -} - -// GetMetaData retrieves instance metadata about the current machine. -// -// See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AESDG-chapter-instancedata.html for more details. -func GetMetaData(path string) (contents []byte, err error) { - url := "http://169.254.169.254/latest/meta-data/" + path - - resp, err := RetryingClient.Get(url) - if err != nil { - return - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - err = fmt.Errorf("Code %d returned for url %s", resp.StatusCode, url) - return - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return - } - return []byte(body), err -} - -func getInstanceCredentials() (cred credentials, err error) { - credentialPath := "iam/security-credentials/" - - // Get the instance role - role, err := GetMetaData(credentialPath) - if err != nil { - return - } - - // Get the instance role credentials - credentialJSON, err := GetMetaData(credentialPath + string(role)) - if err != nil { - return - } - - err = json.Unmarshal([]byte(credentialJSON), &cred) - return -} - -// GetAuth creates an Auth based on either passed in credentials, -// environment information or instance based role credentials. -func GetAuth(accessKey string, secretKey string) (auth Auth, err error) { - // First try passed in credentials - if accessKey != "" && secretKey != "" { - return Auth{accessKey, secretKey, ""}, nil - } - - // Next try to get auth from the environment - auth, err = SharedAuth() - if err == nil { - // Found auth, return - return - } - - // Next try to get auth from the environment - auth, err = EnvAuth() - if err == nil { - // Found auth, return - return - } - - // Next try getting auth from the instance role - cred, err := getInstanceCredentials() - if err == nil { - // Found auth, return - auth.AccessKey = cred.AccessKeyId - auth.SecretKey = cred.SecretAccessKey - auth.Token = cred.Token - return - } - err = errors.New("No valid AWS authentication found") - return -} - -// SharedAuth creates an Auth based on shared credentials stored in -// $HOME/.aws/credentials. The AWS_PROFILE environment variables is used to -// select the profile. -func SharedAuth() (auth Auth, err error) { - var profileName = os.Getenv("AWS_PROFILE") - - if profileName == "" { - profileName = "default" - } - - var credentialsFile = os.Getenv("AWS_CREDENTIAL_FILE") - if credentialsFile == "" { - var homeDir = os.Getenv("HOME") - if homeDir == "" { - err = errors.New("Could not get HOME") - return - } - credentialsFile = homeDir + "/.aws/credentials" - } - - file, err := ini.LoadFile(credentialsFile) - if err != nil { - err = errors.New("Couldn't parse AWS credentials file") - return - } - - var profile = file[profileName] - if profile == nil { - err = errors.New("Couldn't find profile in AWS credentials file") - return - } - - auth.AccessKey = profile["aws_access_key_id"] - auth.SecretKey = profile["aws_secret_access_key"] - - if auth.AccessKey == "" { - err = errors.New("AWS_ACCESS_KEY_ID not found in environment in credentials file") - } - if auth.SecretKey == "" { - err = errors.New("AWS_SECRET_ACCESS_KEY not found in credentials file") - } - return -} - -// EnvAuth creates an Auth based on environment information. -// The AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment -// For accounts that require a security token, it is read from AWS_SECURITY_TOKEN -// variables are used. -func EnvAuth() (auth Auth, err error) { - auth.AccessKey = os.Getenv("AWS_ACCESS_KEY_ID") - if auth.AccessKey == "" { - auth.AccessKey = os.Getenv("AWS_ACCESS_KEY") - } - - auth.SecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY") - if auth.SecretKey == "" { - auth.SecretKey = os.Getenv("AWS_SECRET_KEY") - } - - auth.Token = os.Getenv("AWS_SECURITY_TOKEN") - - if auth.AccessKey == "" { - err = errors.New("AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY not found in environment") - } - if auth.SecretKey == "" { - err = errors.New("AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY not found in environment") - } - return -} - -// Encode takes a string and URI-encodes it in a way suitable -// to be used in AWS signatures. -func Encode(s string) string { - encode := false - for i := 0; i != len(s); i++ { - c := s[i] - if c > 127 || !unreserved[c] { - encode = true - break - } - } - if !encode { - return s - } - e := make([]byte, len(s)*3) - ei := 0 - for i := 0; i != len(s); i++ { - c := s[i] - if c > 127 || !unreserved[c] { - e[ei] = '%' - e[ei+1] = hex[c>>4] - e[ei+2] = hex[c&0xF] - ei += 3 - } else { - e[ei] = c - ei += 1 - } - } - return string(e[:ei]) -} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws_test.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws_test.go deleted file mode 100644 index 78cbbaf03..000000000 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/aws_test.go +++ /dev/null @@ -1,203 +0,0 @@ -package aws_test - -import ( - "github.com/mitchellh/goamz/aws" - . "github.com/motain/gocheck" - "io/ioutil" - "os" - "strings" - "testing" -) - -func Test(t *testing.T) { - TestingT(t) -} - -var _ = Suite(&S{}) - -type S struct { - environ []string -} - -func (s *S) SetUpSuite(c *C) { - s.environ = os.Environ() -} - -func (s *S) TearDownTest(c *C) { - os.Clearenv() - for _, kv := range s.environ { - l := strings.SplitN(kv, "=", 2) - os.Setenv(l[0], l[1]) - } -} - -func (s *S) TestSharedAuthNoHome(c *C) { - os.Clearenv() - os.Setenv("AWS_PROFILE", "foo") - _, err := aws.SharedAuth() - c.Assert(err, ErrorMatches, "Could not get HOME") -} - -func (s *S) TestSharedAuthNoCredentialsFile(c *C) { - os.Clearenv() - os.Setenv("AWS_PROFILE", "foo") - os.Setenv("HOME", "/tmp") - _, err := aws.SharedAuth() - c.Assert(err, ErrorMatches, "Couldn't parse AWS credentials file") -} - -func (s *S) TestSharedAuthNoProfileInFile(c *C) { - os.Clearenv() - os.Setenv("AWS_PROFILE", "foo") - - d, err := ioutil.TempDir("", "") - if err != nil { - panic(err) - } - defer os.RemoveAll(d) - - err = os.Mkdir(d+"/.aws", 0755) - if err != nil { - panic(err) - } - - ioutil.WriteFile(d+"/.aws/credentials", []byte("[bar]\n"), 0644) - os.Setenv("HOME", d) - - _, err = aws.SharedAuth() - c.Assert(err, ErrorMatches, "Couldn't find profile in AWS credentials file") -} - -func (s *S) TestSharedAuthNoKeysInProfile(c *C) { - os.Clearenv() - os.Setenv("AWS_PROFILE", "bar") - - d, err := ioutil.TempDir("", "") - if err != nil { - panic(err) - } - defer os.RemoveAll(d) - - err = os.Mkdir(d+"/.aws", 0755) - if err != nil { - panic(err) - } - - ioutil.WriteFile(d+"/.aws/credentials", []byte("[bar]\nawsaccesskeyid = AK.."), 0644) - os.Setenv("HOME", d) - - _, err = aws.SharedAuth() - c.Assert(err, ErrorMatches, "AWS_SECRET_ACCESS_KEY not found in credentials file") -} - -func (s *S) TestSharedAuthDefaultCredentials(c *C) { - os.Clearenv() - - d, err := ioutil.TempDir("", "") - if err != nil { - panic(err) - } - defer os.RemoveAll(d) - - err = os.Mkdir(d+"/.aws", 0755) - if err != nil { - panic(err) - } - - ioutil.WriteFile(d+"/.aws/credentials", []byte("[default]\naws_access_key_id = access\naws_secret_access_key = secret\n"), 0644) - os.Setenv("HOME", d) - - auth, err := aws.SharedAuth() - c.Assert(err, IsNil) - c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) -} - -func (s *S) TestSharedAuth(c *C) { - os.Clearenv() - os.Setenv("AWS_PROFILE", "bar") - - d, err := ioutil.TempDir("", "") - if err != nil { - panic(err) - } - defer os.RemoveAll(d) - - err = os.Mkdir(d+"/.aws", 0755) - if err != nil { - panic(err) - } - - ioutil.WriteFile(d+"/.aws/credentials", []byte("[bar]\naws_access_key_id = access\naws_secret_access_key = secret\n"), 0644) - os.Setenv("HOME", d) - - auth, err := aws.SharedAuth() - c.Assert(err, IsNil) - c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) -} - -func (s *S) TestEnvAuthNoSecret(c *C) { - os.Clearenv() - _, err := aws.EnvAuth() - c.Assert(err, ErrorMatches, "AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY not found in environment") -} - -func (s *S) TestEnvAuthNoAccess(c *C) { - os.Clearenv() - os.Setenv("AWS_SECRET_ACCESS_KEY", "foo") - _, err := aws.EnvAuth() - c.Assert(err, ErrorMatches, "AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY not found in environment") -} - -func (s *S) TestEnvAuth(c *C) { - os.Clearenv() - os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") - os.Setenv("AWS_ACCESS_KEY_ID", "access") - auth, err := aws.EnvAuth() - c.Assert(err, IsNil) - c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) -} - -func (s *S) TestEnvAuthWithToken(c *C) { - os.Clearenv() - os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") - os.Setenv("AWS_ACCESS_KEY_ID", "access") - os.Setenv("AWS_SECURITY_TOKEN", "token") - auth, err := aws.EnvAuth() - c.Assert(err, IsNil) - c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access", Token: "token"}) -} - -func (s *S) TestEnvAuthAlt(c *C) { - os.Clearenv() - os.Setenv("AWS_SECRET_KEY", "secret") - os.Setenv("AWS_ACCESS_KEY", "access") - auth, err := aws.EnvAuth() - c.Assert(err, IsNil) - c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) -} - -func (s *S) TestGetAuthStatic(c *C) { - auth, err := aws.GetAuth("access", "secret") - c.Assert(err, IsNil) - c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) -} - -func (s *S) TestGetAuthEnv(c *C) { - os.Clearenv() - os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") - os.Setenv("AWS_ACCESS_KEY_ID", "access") - auth, err := aws.GetAuth("", "") - c.Assert(err, IsNil) - c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) -} - -func (s *S) TestEncode(c *C) { - c.Assert(aws.Encode("foo"), Equals, "foo") - c.Assert(aws.Encode("/"), Equals, "%2F") -} - -func (s *S) TestRegionsAreNamed(c *C) { - for n, r := range aws.Regions { - c.Assert(n, Equals, r.Name) - } -} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client.go deleted file mode 100644 index ee53238f7..000000000 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client.go +++ /dev/null @@ -1,125 +0,0 @@ -package aws - -import ( - "math" - "net" - "net/http" - "time" -) - -type RetryableFunc func(*http.Request, *http.Response, error) bool -type WaitFunc func(try int) -type DeadlineFunc func() time.Time - -type ResilientTransport struct { - // Timeout is the maximum amount of time a dial will wait for - // a connect to complete. - // - // The default is no timeout. - // - // With or without a timeout, the operating system may impose - // its own earlier timeout. For instance, TCP timeouts are - // often around 3 minutes. - DialTimeout time.Duration - - // MaxTries, if non-zero, specifies the number of times we will retry on - // failure. Retries are only attempted for temporary network errors or known - // safe failures. - MaxTries int - Deadline DeadlineFunc - ShouldRetry RetryableFunc - Wait WaitFunc - transport *http.Transport -} - -// Convenience method for creating an http client -func NewClient(rt *ResilientTransport) *http.Client { - rt.transport = &http.Transport{ - Dial: func(netw, addr string) (net.Conn, error) { - c, err := net.DialTimeout(netw, addr, rt.DialTimeout) - if err != nil { - return nil, err - } - c.SetDeadline(rt.Deadline()) - return c, nil - }, - DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - } - // TODO: Would be nice is ResilientTransport allowed clients to initialize - // with http.Transport attributes. - return &http.Client{ - Transport: rt, - } -} - -var retryingTransport = &ResilientTransport{ - Deadline: func() time.Time { - return time.Now().Add(5 * time.Second) - }, - DialTimeout: 10 * time.Second, - MaxTries: 3, - ShouldRetry: awsRetry, - Wait: ExpBackoff, -} - -// Exported default client -var RetryingClient = NewClient(retryingTransport) - -func (t *ResilientTransport) RoundTrip(req *http.Request) (*http.Response, error) { - return t.tries(req) -} - -// Retry a request a maximum of t.MaxTries times. -// We'll only retry if the proper criteria are met. -// If a wait function is specified, wait that amount of time -// In between requests. -func (t *ResilientTransport) tries(req *http.Request) (res *http.Response, err error) { - for try := 0; try < t.MaxTries; try += 1 { - res, err = t.transport.RoundTrip(req) - - if !t.ShouldRetry(req, res, err) { - break - } - if res != nil { - res.Body.Close() - } - if t.Wait != nil { - t.Wait(try) - } - } - - return -} - -func ExpBackoff(try int) { - time.Sleep(100 * time.Millisecond * - time.Duration(math.Exp2(float64(try)))) -} - -func LinearBackoff(try int) { - time.Sleep(time.Duration(try*100) * time.Millisecond) -} - -// Decide if we should retry a request. -// In general, the criteria for retrying a request is described here -// http://docs.aws.amazon.com/general/latest/gr/api-retries.html -func awsRetry(req *http.Request, res *http.Response, err error) bool { - retry := false - - // Retry if there's a temporary network error. - if neterr, ok := err.(net.Error); ok { - if neterr.Temporary() { - retry = true - } - } - - // Retry if we get a 5xx series error. - if res != nil { - if res.StatusCode >= 500 && res.StatusCode < 600 { - retry = true - } - } - - return retry -} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client_test.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client_test.go deleted file mode 100644 index 2f6b39cf3..000000000 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/client_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package aws_test - -import ( - "fmt" - "github.com/mitchellh/goamz/aws" - "io/ioutil" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" -) - -// Retrieve the response from handler using aws.RetryingClient -func serveAndGet(handler http.HandlerFunc) (body string, err error) { - ts := httptest.NewServer(handler) - defer ts.Close() - resp, err := aws.RetryingClient.Get(ts.URL) - if err != nil { - return - } - if resp.StatusCode != 200 { - return "", fmt.Errorf("Bad status code: %d", resp.StatusCode) - } - greeting, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - return - } - return strings.TrimSpace(string(greeting)), nil -} - -func TestClient_expected(t *testing.T) { - body := "foo bar" - - resp, err := serveAndGet(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, body) - }) - if err != nil { - t.Fatal(err) - } - if resp != body { - t.Fatal("Body not as expected.") - } -} - -func TestClient_delay(t *testing.T) { - body := "baz" - wait := 4 - resp, err := serveAndGet(func(w http.ResponseWriter, r *http.Request) { - if wait < 0 { - // If we dipped to zero delay and still failed. - t.Fatal("Never succeeded.") - } - wait -= 1 - time.Sleep(time.Second * time.Duration(wait)) - fmt.Fprintln(w, body) - }) - if err != nil { - t.Fatal(err) - } - if resp != body { - t.Fatal("Body not as expected.", resp) - } -} - -func TestClient_no4xxRetry(t *testing.T) { - tries := 0 - - // Fail once before succeeding. - _, err := serveAndGet(func(w http.ResponseWriter, r *http.Request) { - tries += 1 - http.Error(w, "error", 404) - }) - - if err == nil { - t.Fatal("should have error") - } - - if tries != 1 { - t.Fatalf("should only try once: %d", tries) - } -} - -func TestClient_retries(t *testing.T) { - body := "biz" - failed := false - // Fail once before succeeding. - resp, err := serveAndGet(func(w http.ResponseWriter, r *http.Request) { - if !failed { - http.Error(w, "error", 500) - failed = true - } else { - fmt.Fprintln(w, body) - } - }) - if failed != true { - t.Error("We didn't retry!") - } - if err != nil { - t.Fatal(err) - } - if resp != body { - t.Fatal("Body not as expected.") - } -} - -func TestClient_fails(t *testing.T) { - tries := 0 - // Fail 3 times and return the last error. - _, err := serveAndGet(func(w http.ResponseWriter, r *http.Request) { - tries += 1 - http.Error(w, "error", 500) - }) - if err == nil { - t.Fatal(err) - } - if tries != 3 { - t.Fatal("Didn't retry enough") - } -} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3.go deleted file mode 100644 index dbeec6efa..000000000 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3.go +++ /dev/null @@ -1,893 +0,0 @@ -// -// goamz - Go packages to interact with the Amazon Web Services. -// -// https://wiki.ubuntu.com/goamz -// -// Copyright (c) 2011 Canonical Ltd. -// -// Written by Gustavo Niemeyer -// - -package s3 - -import ( - "bytes" - "crypto/md5" - "encoding/base64" - "encoding/xml" - "fmt" - "github.com/mitchellh/goamz/aws" - "io" - "io/ioutil" - "log" - "net" - "net/http" - "net/http/httputil" - "net/url" - "strconv" - "strings" - "time" -) - -const debug = false - -// The S3 type encapsulates operations with an S3 region. -type S3 struct { - aws.Auth - aws.Region - HTTPClient func() *http.Client - - private byte // Reserve the right of using private data. -} - -// The Bucket type encapsulates operations with an S3 bucket. -type Bucket struct { - *S3 - Name string -} - -// The Owner type represents the owner of the object in an S3 bucket. -type Owner struct { - ID string - DisplayName string -} - -var attempts = aws.AttemptStrategy{ - Min: 5, - Total: 5 * time.Second, - Delay: 200 * time.Millisecond, -} - -// New creates a new S3. -func New(auth aws.Auth, region aws.Region) *S3 { - return &S3{ - Auth: auth, - Region: region, - HTTPClient: func() *http.Client { - return http.DefaultClient - }, - private: 0} -} - -// Bucket returns a Bucket with the given name. -func (s3 *S3) Bucket(name string) *Bucket { - if s3.Region.S3BucketEndpoint != "" || s3.Region.S3LowercaseBucket { - name = strings.ToLower(name) - } - return &Bucket{s3, name} -} - -var createBucketConfiguration = ` - %s -` - -// locationConstraint returns an io.Reader specifying a LocationConstraint if -// required for the region. -// -// See http://goo.gl/bh9Kq for details. -func (s3 *S3) locationConstraint() io.Reader { - constraint := "" - if s3.Region.S3LocationConstraint { - constraint = fmt.Sprintf(createBucketConfiguration, s3.Region.Name) - } - return strings.NewReader(constraint) -} - -type ACL string - -const ( - Private = ACL("private") - PublicRead = ACL("public-read") - PublicReadWrite = ACL("public-read-write") - AuthenticatedRead = ACL("authenticated-read") - BucketOwnerRead = ACL("bucket-owner-read") - BucketOwnerFull = ACL("bucket-owner-full-control") -) - -// The ListBucketsResp type holds the results of a List buckets operation. -type ListBucketsResp struct { - Buckets []Bucket `xml:">Bucket"` -} - -// ListBuckets lists all buckets -// -// See: http://goo.gl/NqlyMN -func (s3 *S3) ListBuckets() (result *ListBucketsResp, err error) { - req := &request{ - path: "/", - } - result = &ListBucketsResp{} - for attempt := attempts.Start(); attempt.Next(); { - err = s3.query(req, result) - if !shouldRetry(err) { - break - } - } - if err != nil { - return nil, err - } - // set S3 instance on buckets - for i := range result.Buckets { - result.Buckets[i].S3 = s3 - } - return result, nil -} - -// PutBucket creates a new bucket. -// -// See http://goo.gl/ndjnR for details. -func (b *Bucket) PutBucket(perm ACL) error { - headers := map[string][]string{ - "x-amz-acl": {string(perm)}, - } - req := &request{ - method: "PUT", - bucket: b.Name, - path: "/", - headers: headers, - payload: b.locationConstraint(), - } - return b.S3.query(req, nil) -} - -// DelBucket removes an existing S3 bucket. All objects in the bucket must -// be removed before the bucket itself can be removed. -// -// See http://goo.gl/GoBrY for details. -func (b *Bucket) DelBucket() (err error) { - req := &request{ - method: "DELETE", - bucket: b.Name, - path: "/", - } - for attempt := attempts.Start(); attempt.Next(); { - err = b.S3.query(req, nil) - if !shouldRetry(err) { - break - } - } - return err -} - -// Get retrieves an object from an S3 bucket. -// -// See http://goo.gl/isCO7 for details. -func (b *Bucket) Get(path string) (data []byte, err error) { - body, err := b.GetReader(path) - if err != nil { - return nil, err - } - data, err = ioutil.ReadAll(body) - body.Close() - return data, err -} - -// GetReader retrieves an object from an S3 bucket. -// It is the caller's responsibility to call Close on rc when -// finished reading. -func (b *Bucket) GetReader(path string) (rc io.ReadCloser, err error) { - resp, err := b.GetResponse(path) - if resp != nil { - return resp.Body, err - } - return nil, err -} - -// GetResponse retrieves an object from an S3 bucket returning the http response -// It is the caller's responsibility to call Close on rc when -// finished reading. -func (b *Bucket) GetResponse(path string) (*http.Response, error) { - return b.getResponseParams(path, nil) -} - -// GetTorrent retrieves an Torrent object from an S3 bucket an io.ReadCloser. -// It is the caller's responsibility to call Close on rc when finished reading. -func (b *Bucket) GetTorrentReader(path string) (io.ReadCloser, error) { - resp, err := b.getResponseParams(path, url.Values{"torrent": {""}}) - if err != nil { - return nil, err - } - return resp.Body, nil -} - -// GetTorrent retrieves an Torrent object from an S3, returning -// the torrent as a []byte. -func (b *Bucket) GetTorrent(path string) ([]byte, error) { - body, err := b.GetTorrentReader(path) - if err != nil { - return nil, err - } - defer body.Close() - - return ioutil.ReadAll(body) -} - -func (b *Bucket) getResponseParams(path string, params url.Values) (*http.Response, error) { - req := &request{ - bucket: b.Name, - path: path, - params: params, - } - err := b.S3.prepare(req) - if err != nil { - return nil, err - } - for attempt := attempts.Start(); attempt.Next(); { - resp, err := b.S3.run(req, nil) - if shouldRetry(err) && attempt.HasNext() { - continue - } - if err != nil { - return nil, err - } - return resp, nil - } - panic("unreachable") -} - -func (b *Bucket) Head(path string) (*http.Response, error) { - req := &request{ - method: "HEAD", - bucket: b.Name, - path: path, - } - err := b.S3.prepare(req) - if err != nil { - return nil, err - } - for attempt := attempts.Start(); attempt.Next(); { - resp, err := b.S3.run(req, nil) - if shouldRetry(err) && attempt.HasNext() { - continue - } - if err != nil { - return nil, err - } - return resp, nil - } - panic("unreachable") -} - -// Put inserts an object into the S3 bucket. -// -// See http://goo.gl/FEBPD for details. -func (b *Bucket) Put(path string, data []byte, contType string, perm ACL) error { - body := bytes.NewBuffer(data) - return b.PutReader(path, body, int64(len(data)), contType, perm) -} - -/* -PutHeader - like Put, inserts an object into the S3 bucket. -Instead of Content-Type string, pass in custom headers to override defaults. -*/ -func (b *Bucket) PutHeader(path string, data []byte, customHeaders map[string][]string, perm ACL) error { - body := bytes.NewBuffer(data) - return b.PutReaderHeader(path, body, int64(len(data)), customHeaders, perm) -} - -// PutReader inserts an object into the S3 bucket by consuming data -// from r until EOF. -func (b *Bucket) PutReader(path string, r io.Reader, length int64, contType string, perm ACL) error { - headers := map[string][]string{ - "Content-Length": {strconv.FormatInt(length, 10)}, - "Content-Type": {contType}, - "x-amz-acl": {string(perm)}, - } - req := &request{ - method: "PUT", - bucket: b.Name, - path: path, - headers: headers, - payload: r, - } - return b.S3.query(req, nil) -} - -/* -PutReaderHeader - like PutReader, inserts an object into S3 from a reader. -Instead of Content-Type string, pass in custom headers to override defaults. -*/ -func (b *Bucket) PutReaderHeader(path string, r io.Reader, length int64, customHeaders map[string][]string, perm ACL) error { - // Default headers - headers := map[string][]string{ - "Content-Length": {strconv.FormatInt(length, 10)}, - "Content-Type": {"application/text"}, - "x-amz-acl": {string(perm)}, - } - - // Override with custom headers - for key, value := range customHeaders { - headers[key] = value - } - - req := &request{ - method: "PUT", - bucket: b.Name, - path: path, - headers: headers, - payload: r, - } - return b.S3.query(req, nil) -} - -/* -Copy - copy objects inside bucket -*/ -func (b *Bucket) Copy(oldPath, newPath string, perm ACL) error { - if !strings.HasPrefix(oldPath, "/") { - oldPath = "/" + oldPath - } - - req := &request{ - method: "PUT", - bucket: b.Name, - path: newPath, - headers: map[string][]string{ - "x-amz-copy-source": {amazonEscape("/" + b.Name + oldPath)}, - "x-amz-acl": {string(perm)}, - }, - } - - err := b.S3.prepare(req) - if err != nil { - return err - } - - for attempt := attempts.Start(); attempt.Next(); { - _, err = b.S3.run(req, nil) - if shouldRetry(err) && attempt.HasNext() { - continue - } - if err != nil { - return err - } - return nil - } - panic("unreachable") -} - -// Del removes an object from the S3 bucket. -// -// See http://goo.gl/APeTt for details. -func (b *Bucket) Del(path string) error { - req := &request{ - method: "DELETE", - bucket: b.Name, - path: path, - } - return b.S3.query(req, nil) -} - -type Object struct { - Key string -} - -type MultiObjectDeleteBody struct { - XMLName xml.Name `xml:"Delete"` - Quiet bool - Object []Object -} - -func base64md5(data []byte) string { - h := md5.New() - h.Write(data) - return base64.StdEncoding.EncodeToString(h.Sum(nil)) -} - -// MultiDel removes multiple objects from the S3 bucket efficiently. -// A maximum of 1000 keys at once may be specified. -// -// See http://goo.gl/WvA5sj for details. -func (b *Bucket) MultiDel(paths []string) error { - // create XML payload - v := MultiObjectDeleteBody{} - v.Object = make([]Object, len(paths)) - for i, path := range paths { - v.Object[i] = Object{path} - } - data, _ := xml.Marshal(v) - - // Content-MD5 is required - md5hash := base64md5(data) - req := &request{ - method: "POST", - bucket: b.Name, - path: "/", - params: url.Values{"delete": {""}}, - headers: http.Header{"Content-MD5": {md5hash}}, - payload: bytes.NewReader(data), - } - - return b.S3.query(req, nil) -} - -// The ListResp type holds the results of a List bucket operation. -type ListResp struct { - Name string - Prefix string - Delimiter string - Marker string - NextMarker string - MaxKeys int - // IsTruncated is true if the results have been truncated because - // there are more keys and prefixes than can fit in MaxKeys. - // N.B. this is the opposite sense to that documented (incorrectly) in - // http://goo.gl/YjQTc - IsTruncated bool - Contents []Key - CommonPrefixes []string `xml:">Prefix"` -} - -// The Key type represents an item stored in an S3 bucket. -type Key struct { - Key string - LastModified string - Size int64 - // ETag gives the hex-encoded MD5 sum of the contents, - // surrounded with double-quotes. - ETag string - StorageClass string - Owner Owner -} - -// List returns information about objects in an S3 bucket. -// -// The prefix parameter limits the response to keys that begin with the -// specified prefix. -// -// The delim parameter causes the response to group all of the keys that -// share a common prefix up to the next delimiter in a single entry within -// the CommonPrefixes field. You can use delimiters to separate a bucket -// into different groupings of keys, similar to how folders would work. -// -// The marker parameter specifies the key to start with when listing objects -// in a bucket. Amazon S3 lists objects in alphabetical order and -// will return keys alphabetically greater than the marker. -// -// The max parameter specifies how many keys + common prefixes to return in -// the response. The default is 1000. -// -// For example, given these keys in a bucket: -// -// index.html -// index2.html -// photos/2006/January/sample.jpg -// photos/2006/February/sample2.jpg -// photos/2006/February/sample3.jpg -// photos/2006/February/sample4.jpg -// -// Listing this bucket with delimiter set to "/" would yield the -// following result: -// -// &ListResp{ -// Name: "sample-bucket", -// MaxKeys: 1000, -// Delimiter: "/", -// Contents: []Key{ -// {Key: "index.html", "index2.html"}, -// }, -// CommonPrefixes: []string{ -// "photos/", -// }, -// } -// -// Listing the same bucket with delimiter set to "/" and prefix set to -// "photos/2006/" would yield the following result: -// -// &ListResp{ -// Name: "sample-bucket", -// MaxKeys: 1000, -// Delimiter: "/", -// Prefix: "photos/2006/", -// CommonPrefixes: []string{ -// "photos/2006/February/", -// "photos/2006/January/", -// }, -// } -// -// See http://goo.gl/YjQTc for details. -func (b *Bucket) List(prefix, delim, marker string, max int) (result *ListResp, err error) { - params := map[string][]string{ - "prefix": {prefix}, - "delimiter": {delim}, - "marker": {marker}, - } - if max != 0 { - params["max-keys"] = []string{strconv.FormatInt(int64(max), 10)} - } - req := &request{ - bucket: b.Name, - params: params, - } - result = &ListResp{} - for attempt := attempts.Start(); attempt.Next(); { - err = b.S3.query(req, result) - if !shouldRetry(err) { - break - } - } - if err != nil { - return nil, err - } - return result, nil -} - -// Returns a mapping of all key names in this bucket to Key objects -func (b *Bucket) GetBucketContents() (*map[string]Key, error) { - bucket_contents := map[string]Key{} - prefix := "" - path_separator := "" - marker := "" - for { - contents, err := b.List(prefix, path_separator, marker, 1000) - if err != nil { - return &bucket_contents, err - } - last_key := "" - for _, key := range contents.Contents { - bucket_contents[key.Key] = key - last_key = key.Key - } - if contents.IsTruncated { - marker = contents.NextMarker - if marker == "" { - // From the s3 docs: If response does not include the - // NextMarker and it is truncated, you can use the value of the - // last Key in the response as the marker in the subsequent - // request to get the next set of object keys. - marker = last_key - } - } else { - break - } - } - - return &bucket_contents, nil -} - -// Get metadata from the key without returning the key content -func (b *Bucket) GetKey(path string) (*Key, error) { - req := &request{ - bucket: b.Name, - path: path, - method: "HEAD", - } - err := b.S3.prepare(req) - if err != nil { - return nil, err - } - key := &Key{} - for attempt := attempts.Start(); attempt.Next(); { - resp, err := b.S3.run(req, nil) - if shouldRetry(err) && attempt.HasNext() { - continue - } - if err != nil { - return nil, err - } - key.Key = path - key.LastModified = resp.Header.Get("Last-Modified") - key.ETag = resp.Header.Get("ETag") - contentLength := resp.Header.Get("Content-Length") - size, err := strconv.ParseInt(contentLength, 10, 64) - if err != nil { - return key, fmt.Errorf("bad s3 content-length %v: %v", - contentLength, err) - } - key.Size = size - return key, nil - } - panic("unreachable") -} - -// URL returns a non-signed URL that allows retriving the -// object at path. It only works if the object is publicly -// readable (see SignedURL). -func (b *Bucket) URL(path string) string { - req := &request{ - bucket: b.Name, - path: path, - } - err := b.S3.prepare(req) - if err != nil { - panic(err) - } - u, err := req.url(true) - if err != nil { - panic(err) - } - u.RawQuery = "" - return u.String() -} - -// SignedURL returns a signed URL that allows anyone holding the URL -// to retrieve the object at path. The signature is valid until expires. -func (b *Bucket) SignedURL(path string, expires time.Time) string { - req := &request{ - bucket: b.Name, - path: path, - params: url.Values{"Expires": {strconv.FormatInt(expires.Unix(), 10)}}, - } - err := b.S3.prepare(req) - if err != nil { - panic(err) - } - u, err := req.url(true) - if err != nil { - panic(err) - } - return u.String() -} - -type request struct { - method string - bucket string - path string - signpath string - params url.Values - headers http.Header - baseurl string - payload io.Reader - prepared bool -} - -// amazonShouldEscape returns true if byte should be escaped -func amazonShouldEscape(c byte) bool { - return !((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || - (c >= '0' && c <= '9') || c == '_' || c == '-' || c == '~' || c == '.' || c == '/' || c == ':') -} - -// amazonEscape does uri escaping exactly as Amazon does -func amazonEscape(s string) string { - hexCount := 0 - - for i := 0; i < len(s); i++ { - if amazonShouldEscape(s[i]) { - hexCount++ - } - } - - if hexCount == 0 { - return s - } - - t := make([]byte, len(s)+2*hexCount) - j := 0 - for i := 0; i < len(s); i++ { - if c := s[i]; amazonShouldEscape(c) { - t[j] = '%' - t[j+1] = "0123456789ABCDEF"[c>>4] - t[j+2] = "0123456789ABCDEF"[c&15] - j += 3 - } else { - t[j] = s[i] - j++ - } - } - return string(t) -} - -// url returns url to resource, either full (with host/scheme) or -// partial for HTTP request -func (req *request) url(full bool) (*url.URL, error) { - u, err := url.Parse(req.baseurl) - if err != nil { - return nil, fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err) - } - - u.Opaque = amazonEscape(req.path) - if full { - u.Opaque = "//" + u.Host + u.Opaque - } - u.RawQuery = req.params.Encode() - - return u, nil -} - -// query prepares and runs the req request. -// If resp is not nil, the XML data contained in the response -// body will be unmarshalled on it. -func (s3 *S3) query(req *request, resp interface{}) error { - err := s3.prepare(req) - if err == nil { - var httpResponse *http.Response - httpResponse, err = s3.run(req, resp) - if resp == nil && httpResponse != nil { - httpResponse.Body.Close() - } - } - return err -} - -// prepare sets up req to be delivered to S3. -func (s3 *S3) prepare(req *request) error { - if !req.prepared { - req.prepared = true - if req.method == "" { - req.method = "GET" - } - // Copy so they can be mutated without affecting on retries. - params := make(url.Values) - headers := make(http.Header) - for k, v := range req.params { - params[k] = v - } - for k, v := range req.headers { - headers[k] = v - } - req.params = params - req.headers = headers - if !strings.HasPrefix(req.path, "/") { - req.path = "/" + req.path - } - req.signpath = req.path - - if req.bucket != "" { - req.baseurl = s3.Region.S3BucketEndpoint - if req.baseurl == "" { - // Use the path method to address the bucket. - req.baseurl = s3.Region.S3Endpoint - req.path = "/" + req.bucket + req.path - } else { - // Just in case, prevent injection. - if strings.IndexAny(req.bucket, "/:@") >= 0 { - return fmt.Errorf("bad S3 bucket: %q", req.bucket) - } - req.baseurl = strings.Replace(req.baseurl, "${bucket}", req.bucket, -1) - } - req.signpath = "/" + req.bucket + req.signpath - } else { - req.baseurl = s3.Region.S3Endpoint - } - } - - // Always sign again as it's not clear how far the - // server has handled a previous attempt. - u, err := url.Parse(req.baseurl) - if err != nil { - return fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err) - } - req.headers["Host"] = []string{u.Host} - req.headers["Date"] = []string{time.Now().In(time.UTC).Format(time.RFC1123)} - sign(s3.Auth, req.method, amazonEscape(req.signpath), req.params, req.headers) - return nil -} - -// run sends req and returns the http response from the server. -// If resp is not nil, the XML data contained in the response -// body will be unmarshalled on it. -func (s3 *S3) run(req *request, resp interface{}) (*http.Response, error) { - if debug { - log.Printf("Running S3 request: %#v", req) - } - - u, err := req.url(false) - if err != nil { - return nil, err - } - - hreq := http.Request{ - URL: u, - Method: req.method, - ProtoMajor: 1, - ProtoMinor: 1, - Close: true, - Header: req.headers, - } - - if v, ok := req.headers["Content-Length"]; ok { - hreq.ContentLength, _ = strconv.ParseInt(v[0], 10, 64) - delete(req.headers, "Content-Length") - } - if req.payload != nil { - hreq.Body = ioutil.NopCloser(req.payload) - } - - hresp, err := s3.HTTPClient().Do(&hreq) - if err != nil { - return nil, err - } - if debug { - dump, _ := httputil.DumpResponse(hresp, true) - log.Printf("} -> %s\n", dump) - } - if hresp.StatusCode != 200 && hresp.StatusCode != 204 { - defer hresp.Body.Close() - return nil, buildError(hresp) - } - if resp != nil { - err = xml.NewDecoder(hresp.Body).Decode(resp) - hresp.Body.Close() - } - return hresp, err -} - -// Error represents an error in an operation with S3. -type Error struct { - StatusCode int // HTTP status code (200, 403, ...) - Code string // EC2 error code ("UnsupportedOperation", ...) - Message string // The human-oriented error message - BucketName string - RequestId string - HostId string -} - -func (e *Error) Error() string { - return e.Message -} - -func buildError(r *http.Response) error { - if debug { - log.Printf("got error (status code %v)", r.StatusCode) - data, err := ioutil.ReadAll(r.Body) - if err != nil { - log.Printf("\tread error: %v", err) - } else { - log.Printf("\tdata:\n%s\n\n", data) - } - r.Body = ioutil.NopCloser(bytes.NewBuffer(data)) - } - - err := Error{} - // TODO return error if Unmarshal fails? - xml.NewDecoder(r.Body).Decode(&err) - r.Body.Close() - err.StatusCode = r.StatusCode - if err.Message == "" { - err.Message = r.Status - } - if debug { - log.Printf("err: %#v\n", err) - } - return &err -} - -func shouldRetry(err error) bool { - if err == nil { - return false - } - switch err { - case io.ErrUnexpectedEOF, io.EOF: - return true - } - switch e := err.(type) { - case *net.DNSError: - return true - case *net.OpError: - switch e.Op { - case "read", "write": - return true - } - case *Error: - switch e.Code { - case "InternalError", "NoSuchUpload", "NoSuchBucket": - return true - } - } - return false -} - -func hasCode(err error, code string) bool { - s3err, ok := err.(*Error) - return ok && s3err.Code == code -} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign.go deleted file mode 100644 index 262503dc3..000000000 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign.go +++ /dev/null @@ -1,126 +0,0 @@ -package s3 - -import ( - "crypto/hmac" - "crypto/sha1" - "encoding/base64" - "log" - "sort" - "strings" - - "github.com/mitchellh/goamz/aws" -) - -var b64 = base64.StdEncoding - -// ---------------------------------------------------------------------------- -// S3 signing (http://goo.gl/G1LrK) - -var s3ParamsToSign = map[string]bool{ - "acl": true, - "delete": true, - "location": true, - "logging": true, - "notification": true, - "partNumber": true, - "policy": true, - "requestPayment": true, - "torrent": true, - "uploadId": true, - "uploads": true, - "versionId": true, - "versioning": true, - "versions": true, - "response-content-type": true, - "response-content-language": true, - "response-expires": true, - "response-cache-control": true, - "response-content-disposition": true, - "response-content-encoding": true, -} - -func sign(auth aws.Auth, method, canonicalPath string, params, headers map[string][]string) { - var md5, ctype, date, xamz string - var xamzDate bool - var sarray []string - - // add security token - if auth.Token != "" { - headers["x-amz-security-token"] = []string{auth.Token} - } - - if auth.SecretKey == "" { - // no auth secret; skip signing, e.g. for public read-only buckets. - return - } - - for k, v := range headers { - k = strings.ToLower(k) - switch k { - case "content-md5": - md5 = v[0] - case "content-type": - ctype = v[0] - case "date": - if !xamzDate { - date = v[0] - } - default: - if strings.HasPrefix(k, "x-amz-") { - vall := strings.Join(v, ",") - sarray = append(sarray, k+":"+vall) - if k == "x-amz-date" { - xamzDate = true - date = "" - } - } - } - } - if len(sarray) > 0 { - sort.StringSlice(sarray).Sort() - xamz = strings.Join(sarray, "\n") + "\n" - } - - expires := false - if v, ok := params["Expires"]; ok { - // Query string request authentication alternative. - expires = true - date = v[0] - params["AWSAccessKeyId"] = []string{auth.AccessKey} - } - - sarray = sarray[0:0] - for k, v := range params { - if s3ParamsToSign[k] { - for _, vi := range v { - if vi == "" { - sarray = append(sarray, k) - } else { - // "When signing you do not encode these values." - sarray = append(sarray, k+"="+vi) - } - } - } - } - if len(sarray) > 0 { - sort.StringSlice(sarray).Sort() - canonicalPath = canonicalPath + "?" + strings.Join(sarray, "&") - } - - payload := method + "\n" + md5 + "\n" + ctype + "\n" + date + "\n" + xamz + canonicalPath - hash := hmac.New(sha1.New, []byte(auth.SecretKey)) - hash.Write([]byte(payload)) - signature := make([]byte, b64.EncodedLen(hash.Size())) - b64.Encode(signature, hash.Sum(nil)) - - if expires { - params["Signature"] = []string{string(signature)} - } else { - headers["Authorization"] = []string{"AWS " + auth.AccessKey + ":" + string(signature)} - } - - if debug { - log.Printf("Signature payload: %q", payload) - log.Printf("Signature: %q", signature) - } -} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign_test.go b/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign_test.go deleted file mode 100644 index 9d9fe8d2e..000000000 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/sign_test.go +++ /dev/null @@ -1,194 +0,0 @@ -package s3_test - -import ( - "github.com/mitchellh/goamz/aws" - "github.com/mitchellh/goamz/s3" - . "github.com/motain/gocheck" -) - -// S3 ReST authentication docs: http://goo.gl/G1LrK - -var testAuth = aws.Auth{"0PN5J17HBGZHT7JJ3X82", "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o", ""} -var emptyAuth = aws.Auth{"", "", ""} - -func (s *S) TestSignExampleObjectGet(c *C) { - method := "GET" - path := "/johnsmith/photos/puppy.jpg" - headers := map[string][]string{ - "Host": {"johnsmith.s3.amazonaws.com"}, - "Date": {"Tue, 27 Mar 2007 19:36:42 +0000"}, - } - s3.Sign(testAuth, method, path, nil, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=" - c.Assert(headers["Authorization"], DeepEquals, []string{expected}) -} - -func (s *S) TestSignExampleObjectGetNoAuth(c *C) { - method := "GET" - path := "/johnsmith/photos/puppy.jpg" - headers := map[string][]string{ - "Host": {"johnsmith.s3.amazonaws.com"}, - "Date": {"Tue, 27 Mar 2007 19:36:42 +0000"}, - } - s3.Sign(emptyAuth, method, path, nil, headers) - c.Assert(headers["Authorization"], IsNil) -} - -func (s *S) TestSignExampleObjectPut(c *C) { - method := "PUT" - path := "/johnsmith/photos/puppy.jpg" - headers := map[string][]string{ - "Host": {"johnsmith.s3.amazonaws.com"}, - "Date": {"Tue, 27 Mar 2007 21:15:45 +0000"}, - "Content-Type": {"image/jpeg"}, - "Content-Length": {"94328"}, - } - s3.Sign(testAuth, method, path, nil, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=" - c.Assert(headers["Authorization"], DeepEquals, []string{expected}) -} - -func (s *S) TestSignExampleList(c *C) { - method := "GET" - path := "/johnsmith/" - params := map[string][]string{ - "prefix": {"photos"}, - "max-keys": {"50"}, - "marker": {"puppy"}, - } - headers := map[string][]string{ - "Host": {"johnsmith.s3.amazonaws.com"}, - "Date": {"Tue, 27 Mar 2007 19:42:41 +0000"}, - "User-Agent": {"Mozilla/5.0"}, - } - s3.Sign(testAuth, method, path, params, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=" - c.Assert(headers["Authorization"], DeepEquals, []string{expected}) -} - -func (s *S) TestSignExampleListNoAuth(c *C) { - method := "GET" - path := "/johnsmith/" - params := map[string][]string{ - "prefix": {"photos"}, - "max-keys": {"50"}, - "marker": {"puppy"}, - } - headers := map[string][]string{ - "Host": {"johnsmith.s3.amazonaws.com"}, - "Date": {"Tue, 27 Mar 2007 19:42:41 +0000"}, - "User-Agent": {"Mozilla/5.0"}, - } - s3.Sign(emptyAuth, method, path, params, headers) - c.Assert(headers["Authorization"], IsNil) -} - -func (s *S) TestSignExampleFetch(c *C) { - method := "GET" - path := "/johnsmith/" - params := map[string][]string{ - "acl": {""}, - } - headers := map[string][]string{ - "Host": {"johnsmith.s3.amazonaws.com"}, - "Date": {"Tue, 27 Mar 2007 19:44:46 +0000"}, - } - s3.Sign(testAuth, method, path, params, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=" - c.Assert(headers["Authorization"], DeepEquals, []string{expected}) -} - -func (s *S) TestSignExampleFetchNoAuth(c *C) { - method := "GET" - path := "/johnsmith/" - params := map[string][]string{ - "acl": {""}, - } - headers := map[string][]string{ - "Host": {"johnsmith.s3.amazonaws.com"}, - "Date": {"Tue, 27 Mar 2007 19:44:46 +0000"}, - } - s3.Sign(emptyAuth, method, path, params, headers) - c.Assert(headers["Authorization"], IsNil) -} - -func (s *S) TestSignExampleDelete(c *C) { - method := "DELETE" - path := "/johnsmith/photos/puppy.jpg" - params := map[string][]string{} - headers := map[string][]string{ - "Host": {"s3.amazonaws.com"}, - "Date": {"Tue, 27 Mar 2007 21:20:27 +0000"}, - "User-Agent": {"dotnet"}, - "x-amz-date": {"Tue, 27 Mar 2007 21:20:26 +0000"}, - } - s3.Sign(testAuth, method, path, params, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=" - c.Assert(headers["Authorization"], DeepEquals, []string{expected}) -} - -func (s *S) TestSignExampleUpload(c *C) { - method := "PUT" - path := "/static.johnsmith.net/db-backup.dat.gz" - params := map[string][]string{} - headers := map[string][]string{ - "Host": {"static.johnsmith.net:8080"}, - "Date": {"Tue, 27 Mar 2007 21:06:08 +0000"}, - "User-Agent": {"curl/7.15.5"}, - "x-amz-acl": {"public-read"}, - "content-type": {"application/x-download"}, - "Content-MD5": {"4gJE4saaMU4BqNR0kLY+lw=="}, - "X-Amz-Meta-ReviewedBy": {"joe@johnsmith.net,jane@johnsmith.net"}, - "X-Amz-Meta-FileChecksum": {"0x02661779"}, - "X-Amz-Meta-ChecksumAlgorithm": {"crc32"}, - "Content-Disposition": {"attachment; filename=database.dat"}, - "Content-Encoding": {"gzip"}, - "Content-Length": {"5913339"}, - } - s3.Sign(testAuth, method, path, params, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=" - c.Assert(headers["Authorization"], DeepEquals, []string{expected}) -} - -func (s *S) TestSignExampleListAllMyBuckets(c *C) { - method := "GET" - path := "/" - headers := map[string][]string{ - "Host": {"s3.amazonaws.com"}, - "Date": {"Wed, 28 Mar 2007 01:29:59 +0000"}, - } - s3.Sign(testAuth, method, path, nil, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:Db+gepJSUbZKwpx1FR0DLtEYoZA=" - c.Assert(headers["Authorization"], DeepEquals, []string{expected}) -} - -func (s *S) TestSignExampleUnicodeKeys(c *C) { - method := "GET" - path := "/dictionary/fran%C3%A7ais/pr%c3%a9f%c3%a8re" - headers := map[string][]string{ - "Host": {"s3.amazonaws.com"}, - "Date": {"Wed, 28 Mar 2007 01:49:49 +0000"}, - } - s3.Sign(testAuth, method, path, nil, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:dxhSBHoI6eVSPcXJqEghlUzZMnY=" - c.Assert(headers["Authorization"], DeepEquals, []string{expected}) -} - -// Not included in AWS documentation - -func (s *S) TestSignWithIAMToken(c *C) { - method := "GET" - path := "/" - headers := map[string][]string{ - "Host": {"s3.amazonaws.com"}, - "Date": {"Wed, 28 Mar 2007 01:29:59 +0000"}, - } - - authWithToken := testAuth - authWithToken.Token = "totallysecret" - - s3.Sign(authWithToken, method, path, nil, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:SJ0yQO7NpHyXJ7zkxY+/fGQ6aUw=" - c.Assert(headers["Authorization"], DeepEquals, []string{expected}) - c.Assert(headers["x-amz-security-token"], DeepEquals, []string{authWithToken.Token}) -} diff --git a/Godeps/_workspace/src/github.com/vaughan0/go-ini/LICENSE b/Godeps/_workspace/src/github.com/vaughan0/go-ini/LICENSE deleted file mode 100644 index 968b45384..000000000 --- a/Godeps/_workspace/src/github.com/vaughan0/go-ini/LICENSE +++ /dev/null @@ -1,14 +0,0 @@ -Copyright (c) 2013 Vaughan Newton - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the Software without restriction, including without limitation the -rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit -persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the -Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Godeps/_workspace/src/github.com/vaughan0/go-ini/README.md b/Godeps/_workspace/src/github.com/vaughan0/go-ini/README.md deleted file mode 100644 index d5cd4e74b..000000000 --- a/Godeps/_workspace/src/github.com/vaughan0/go-ini/README.md +++ /dev/null @@ -1,70 +0,0 @@ -go-ini -====== - -INI parsing library for Go (golang). - -View the API documentation [here](http://godoc.org/github.com/vaughan0/go-ini). - -Usage ------ - -Parse an INI file: - -```go -import "github.com/vaughan0/go-ini" - -file, err := ini.LoadFile("myfile.ini") -``` - -Get data from the parsed file: - -```go -name, ok := file.Get("person", "name") -if !ok { - panic("'name' variable missing from 'person' section") -} -``` - -Iterate through values in a section: - -```go -for key, value := range file["mysection"] { - fmt.Printf("%s => %s\n", key, value) -} -``` - -Iterate through sections in a file: - -```go -for name, section := range file { - fmt.Printf("Section name: %s\n", name) -} -``` - -File Format ------------ - -INI files are parsed by go-ini line-by-line. Each line may be one of the following: - - * A section definition: [section-name] - * A property: key = value - * A comment: #blahblah _or_ ;blahblah - * Blank. The line will be ignored. - -Properties defined before any section headers are placed in the default section, which has -the empty string as it's key. - -Example: - -```ini -# I am a comment -; So am I! - -[apples] -colour = red or green -shape = applish - -[oranges] -shape = square -colour = blue -``` diff --git a/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini.go b/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini.go deleted file mode 100644 index 81aeb32f8..000000000 --- a/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini.go +++ /dev/null @@ -1,123 +0,0 @@ -// Package ini provides functions for parsing INI configuration files. -package ini - -import ( - "bufio" - "fmt" - "io" - "os" - "regexp" - "strings" -) - -var ( - sectionRegex = regexp.MustCompile(`^\[(.*)\]$`) - assignRegex = regexp.MustCompile(`^([^=]+)=(.*)$`) -) - -// ErrSyntax is returned when there is a syntax error in an INI file. -type ErrSyntax struct { - Line int - Source string // The contents of the erroneous line, without leading or trailing whitespace -} - -func (e ErrSyntax) Error() string { - return fmt.Sprintf("invalid INI syntax on line %d: %s", e.Line, e.Source) -} - -// A File represents a parsed INI file. -type File map[string]Section - -// A Section represents a single section of an INI file. -type Section map[string]string - -// Returns a named Section. A Section will be created if one does not already exist for the given name. -func (f File) Section(name string) Section { - section := f[name] - if section == nil { - section = make(Section) - f[name] = section - } - return section -} - -// Looks up a value for a key in a section and returns that value, along with a boolean result similar to a map lookup. -func (f File) Get(section, key string) (value string, ok bool) { - if s := f[section]; s != nil { - value, ok = s[key] - } - return -} - -// Loads INI data from a reader and stores the data in the File. -func (f File) Load(in io.Reader) (err error) { - bufin, ok := in.(*bufio.Reader) - if !ok { - bufin = bufio.NewReader(in) - } - return parseFile(bufin, f) -} - -// Loads INI data from a named file and stores the data in the File. -func (f File) LoadFile(file string) (err error) { - in, err := os.Open(file) - if err != nil { - return - } - defer in.Close() - return f.Load(in) -} - -func parseFile(in *bufio.Reader, file File) (err error) { - section := "" - lineNum := 0 - for done := false; !done; { - var line string - if line, err = in.ReadString('\n'); err != nil { - if err == io.EOF { - done = true - } else { - return - } - } - lineNum++ - line = strings.TrimSpace(line) - if len(line) == 0 { - // Skip blank lines - continue - } - if line[0] == ';' || line[0] == '#' { - // Skip comments - continue - } - - if groups := assignRegex.FindStringSubmatch(line); groups != nil { - key, val := groups[1], groups[2] - key, val = strings.TrimSpace(key), strings.TrimSpace(val) - file.Section(section)[key] = val - } else if groups := sectionRegex.FindStringSubmatch(line); groups != nil { - name := strings.TrimSpace(groups[1]) - section = name - // Create the section if it does not exist - file.Section(section) - } else { - return ErrSyntax{lineNum, line} - } - - } - return nil -} - -// Loads and returns a File from a reader. -func Load(in io.Reader) (File, error) { - file := make(File) - err := file.Load(in) - return file, err -} - -// Loads and returns an INI File from a file on disk. -func LoadFile(filename string) (File, error) { - file := make(File) - err := file.LoadFile(filename) - return file, err -} diff --git a/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_linux_test.go b/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_linux_test.go deleted file mode 100644 index 38a6f0004..000000000 --- a/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_linux_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package ini - -import ( - "reflect" - "syscall" - "testing" -) - -func TestLoadFile(t *testing.T) { - originalOpenFiles := numFilesOpen(t) - - file, err := LoadFile("test.ini") - if err != nil { - t.Fatal(err) - } - - if originalOpenFiles != numFilesOpen(t) { - t.Error("test.ini not closed") - } - - if !reflect.DeepEqual(file, File{"default": {"stuff": "things"}}) { - t.Error("file not read correctly") - } -} - -func numFilesOpen(t *testing.T) (num uint64) { - var rlimit syscall.Rlimit - err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit) - if err != nil { - t.Fatal(err) - } - maxFds := int(rlimit.Cur) - - var stat syscall.Stat_t - for i := 0; i < maxFds; i++ { - if syscall.Fstat(i, &stat) == nil { - num++ - } else { - return - } - } - return -} diff --git a/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_test.go b/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_test.go deleted file mode 100644 index 06a4d05ea..000000000 --- a/Godeps/_workspace/src/github.com/vaughan0/go-ini/ini_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package ini - -import ( - "reflect" - "strings" - "testing" -) - -func TestLoad(t *testing.T) { - src := ` - # Comments are ignored - - herp = derp - - [foo] - hello=world - whitespace should = not matter - ; sneaky semicolon-style comment - multiple = equals = signs - - [bar] - this = that` - - file, err := Load(strings.NewReader(src)) - if err != nil { - t.Fatal(err) - } - check := func(section, key, expect string) { - if value, _ := file.Get(section, key); value != expect { - t.Errorf("Get(%q, %q): expected %q, got %q", section, key, expect, value) - } - } - - check("", "herp", "derp") - check("foo", "hello", "world") - check("foo", "whitespace should", "not matter") - check("foo", "multiple", "equals = signs") - check("bar", "this", "that") -} - -func TestSyntaxError(t *testing.T) { - src := ` - # Line 2 - [foo] - bar = baz - # Here's an error on line 6: - wut? - herp = derp` - _, err := Load(strings.NewReader(src)) - t.Logf("%T: %v", err, err) - if err == nil { - t.Fatal("expected an error, got nil") - } - syntaxErr, ok := err.(ErrSyntax) - if !ok { - t.Fatal("expected an error of type ErrSyntax") - } - if syntaxErr.Line != 6 { - t.Fatal("incorrect line number") - } - if syntaxErr.Source != "wut?" { - t.Fatal("incorrect source") - } -} - -func TestDefinedSectionBehaviour(t *testing.T) { - check := func(src string, expect File) { - file, err := Load(strings.NewReader(src)) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(file, expect) { - t.Errorf("expected %v, got %v", expect, file) - } - } - // No sections for an empty file - check("", File{}) - // Default section only if there are actually values for it - check("foo=bar", File{"": {"foo": "bar"}}) - // User-defined sections should always be present, even if empty - check("[a]\n[b]\nfoo=bar", File{ - "a": {}, - "b": {"foo": "bar"}, - }) - check("foo=bar\n[a]\nthis=that", File{ - "": {"foo": "bar"}, - "a": {"this": "that"}, - }) -} diff --git a/Godeps/_workspace/src/github.com/vaughan0/go-ini/test.ini b/Godeps/_workspace/src/github.com/vaughan0/go-ini/test.ini deleted file mode 100644 index d13c999e2..000000000 --- a/Godeps/_workspace/src/github.com/vaughan0/go-ini/test.ini +++ /dev/null @@ -1,2 +0,0 @@ -[default] -stuff = things diff --git a/Godeps/_workspace/src/golang.org/x/net/context/context_test.go b/Godeps/_workspace/src/golang.org/x/net/context/context_test.go index faf67722a..e64afa64c 100644 --- a/Godeps/_workspace/src/golang.org/x/net/context/context_test.go +++ b/Godeps/_workspace/src/golang.org/x/net/context/context_test.go @@ -375,7 +375,7 @@ func TestAllocs(t *testing.T) { <-c.Done() }, limit: 8, - gccgoLimit: 13, + gccgoLimit: 15, }, { desc: "WithCancel(bg)", diff --git a/Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/cancelreq.go b/Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/cancelreq.go new file mode 100644 index 000000000..48610e362 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/cancelreq.go @@ -0,0 +1,18 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.5 + +package ctxhttp + +import "net/http" + +func canceler(client *http.Client, req *http.Request) func() { + ch := make(chan struct{}) + req.Cancel = ch + + return func() { + close(ch) + } +} diff --git a/Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/cancelreq_go14.go b/Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/cancelreq_go14.go new file mode 100644 index 000000000..56bcbadb8 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/cancelreq_go14.go @@ -0,0 +1,23 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.5 + +package ctxhttp + +import "net/http" + +type requestCanceler interface { + CancelRequest(*http.Request) +} + +func canceler(client *http.Client, req *http.Request) func() { + rc, ok := client.Transport.(requestCanceler) + if !ok { + return func() {} + } + return func() { + rc.CancelRequest(req) + } +} diff --git a/Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/ctxhttp.go b/Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/ctxhttp.go new file mode 100644 index 000000000..9f3488813 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/ctxhttp.go @@ -0,0 +1,79 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ctxhttp provides helper functions for performing context-aware HTTP requests. +package ctxhttp + +import ( + "io" + "net/http" + "net/url" + "strings" + + "golang.org/x/net/context" +) + +// Do sends an HTTP request with the provided http.Client and returns an HTTP response. +// If the client is nil, http.DefaultClient is used. +// If the context is canceled or times out, ctx.Err() will be returned. +func Do(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) { + if client == nil { + client = http.DefaultClient + } + + // Request cancelation changed in Go 1.5, see cancelreq.go and cancelreq_go14.go. + cancel := canceler(client, req) + + type responseAndError struct { + resp *http.Response + err error + } + result := make(chan responseAndError, 1) + + go func() { + resp, err := client.Do(req) + result <- responseAndError{resp, err} + }() + + select { + case <-ctx.Done(): + cancel() + return nil, ctx.Err() + case r := <-result: + return r.resp, r.err + } +} + +// Get issues a GET request via the Do function. +func Get(ctx context.Context, client *http.Client, url string) (*http.Response, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + return Do(ctx, client, req) +} + +// Head issues a HEAD request via the Do function. +func Head(ctx context.Context, client *http.Client, url string) (*http.Response, error) { + req, err := http.NewRequest("HEAD", url, nil) + if err != nil { + return nil, err + } + return Do(ctx, client, req) +} + +// Post issues a POST request via the Do function. +func Post(ctx context.Context, client *http.Client, url string, bodyType string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequest("POST", url, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", bodyType) + return Do(ctx, client, req) +} + +// PostForm issues a POST request via the Do function. +func PostForm(ctx context.Context, client *http.Client, url string, data url.Values) (*http.Response, error) { + return Post(ctx, client, url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) +} diff --git a/Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/ctxhttp_test.go b/Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/ctxhttp_test.go new file mode 100644 index 000000000..47b53d7f1 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/ctxhttp_test.go @@ -0,0 +1,72 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ctxhttp + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + "time" + + "golang.org/x/net/context" +) + +const ( + requestDuration = 100 * time.Millisecond + requestBody = "ok" +) + +func TestNoTimeout(t *testing.T) { + ctx := context.Background() + resp, err := doRequest(ctx) + + if resp == nil || err != nil { + t.Fatalf("error received from client: %v %v", err, resp) + } +} +func TestCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(requestDuration / 2) + cancel() + }() + + resp, err := doRequest(ctx) + + if resp != nil || err == nil { + t.Fatalf("expected error, didn't get one. resp: %v", resp) + } + if err != ctx.Err() { + t.Fatalf("expected error from context but got: %v", err) + } +} + +func TestCancelAfterRequest(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + resp, err := doRequest(ctx) + + // Cancel before reading the body. + // Request.Body should still be readable after the context is canceled. + cancel() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil || string(b) != requestBody { + t.Fatalf("could not read body: %q %v", b, err) + } +} + +func doRequest(ctx context.Context) (*http.Response, error) { + var okHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(requestDuration) + w.Write([]byte(requestBody)) + }) + + serv := httptest.NewServer(okHandler) + defer serv.Close() + + return Get(ctx, nil, serv.URL) +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/attempt.go b/Godeps/_workspace/src/gopkg.in/amz.v3/aws/attempt.go similarity index 100% rename from Godeps/_workspace/src/github.com/mitchellh/goamz/aws/attempt.go rename to Godeps/_workspace/src/gopkg.in/amz.v3/aws/attempt.go diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/attempt_test.go b/Godeps/_workspace/src/gopkg.in/amz.v3/aws/attempt_test.go similarity index 95% rename from Godeps/_workspace/src/github.com/mitchellh/goamz/aws/attempt_test.go rename to Godeps/_workspace/src/gopkg.in/amz.v3/aws/attempt_test.go index 1fda5bf3c..761424289 100644 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/aws/attempt_test.go +++ b/Godeps/_workspace/src/gopkg.in/amz.v3/aws/attempt_test.go @@ -1,9 +1,11 @@ package aws_test import ( - "github.com/mitchellh/goamz/aws" - . "github.com/motain/gocheck" "time" + + . "gopkg.in/check.v1" + + "gopkg.in/amz.v3/aws" ) func (S) TestAttemptTiming(c *C) { diff --git a/Godeps/_workspace/src/gopkg.in/amz.v3/aws/aws.go b/Godeps/_workspace/src/gopkg.in/amz.v3/aws/aws.go new file mode 100644 index 000000000..f2ad7f57c --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/amz.v3/aws/aws.go @@ -0,0 +1,268 @@ +// +// goamz - Go packages to interact with the Amazon Web Services. +// +// https://wiki.ubuntu.com/goamz +// +// Copyright (c) 2011 Canonical Ltd. +// + +package aws + +import ( + "errors" + "os" + "strings" +) + +// Region defines the URLs where AWS services may be accessed. +// +// See http://goo.gl/d8BP1 for more details. +type Region struct { + Name string // the canonical name of this region. + EC2Endpoint string + S3Endpoint string + S3BucketEndpoint string // Not needed by AWS S3. Use ${bucket} for bucket name. + S3LocationConstraint bool // true if this region requires a LocationConstraint declaration. + S3LowercaseBucket bool // true if the region requires bucket names to be lower case. + SDBEndpoint string // not all regions have simpleDB, fro eg. Frankfurt (eu-central-1) does not + SNSEndpoint string + SQSEndpoint string + IAMEndpoint string +} + +func (r Region) ResolveS3BucketEndpoint(bucketName string) string { + if r.S3BucketEndpoint != "" { + return strings.ToLower(strings.Replace(r.S3BucketEndpoint, "${bucket}", bucketName, -1)) + } + return strings.ToLower(r.S3Endpoint + "/" + bucketName + "/") +} + +var USEast = Region{ + "us-east-1", // US East (N. Virginia) + "https://ec2.us-east-1.amazonaws.com", + "https://s3.amazonaws.com", + "", + false, + false, + "https://sdb.amazonaws.com", + "https://sns.us-east-1.amazonaws.com", + "https://sqs.us-east-1.amazonaws.com", + "https://iam.amazonaws.com", +} + +var USWest = Region{ + "us-west-1", //US West (N. California) + "https://ec2.us-west-1.amazonaws.com", + "https://s3-us-west-1.amazonaws.com", + "", + true, + true, + "https://sdb.us-west-1.amazonaws.com", + "https://sns.us-west-1.amazonaws.com", + "https://sqs.us-west-1.amazonaws.com", + "https://iam.amazonaws.com", +} + +var USWest2 = Region{ + "us-west-2", // US West (Oregon) + "https://ec2.us-west-2.amazonaws.com", + "https://s3-us-west-2.amazonaws.com", + "", + true, + true, + "https://sdb.us-west-2.amazonaws.com", + "https://sns.us-west-2.amazonaws.com", + "https://sqs.us-west-2.amazonaws.com", + "https://iam.amazonaws.com", +} + +var USGovWest = Region{ + "us-gov-west-1", // Isolated regions, AWS GovCloud (US) + "https://ec2.us-gov-west-1.amazonaws.com", + "https://s3-us-gov-west-1.amazonaws.com", + "", + true, + true, + "", + "https://sns.us-gov-west-1.amazonaws.com", + "https://sqs.us-gov-west-1.amazonaws.com", + "https://iam.us-gov.amazonaws.com", +} + +var EUWest = Region{ + "eu-west-1", // EU (Ireland) + "https://ec2.eu-west-1.amazonaws.com", + "https://s3-eu-west-1.amazonaws.com", + "", + true, + true, + "https://sdb.eu-west-1.amazonaws.com", + "https://sns.eu-west-1.amazonaws.com", + "https://sqs.eu-west-1.amazonaws.com", + "https://iam.amazonaws.com", +} + +var EUCentral = Region{ + "eu-central-1", // EU (Frankfurt) + "https://ec2.eu-central-1.amazonaws.com", + "https://s3-eu-central-1.amazonaws.com", + "", + true, + true, + "", + "https://sns.eu-central-1.amazonaws.com", + "https://sqs.eu-central-1.amazonaws.com", + "https://iam.amazonaws.com", +} + +var APSoutheast = Region{ + "ap-southeast-1", // Asia Pacific (Singapore) + "https://ec2.ap-southeast-1.amazonaws.com", + "https://s3-ap-southeast-1.amazonaws.com", + "", + true, + true, + "https://sdb.ap-southeast-1.amazonaws.com", + "https://sns.ap-southeast-1.amazonaws.com", + "https://sqs.ap-southeast-1.amazonaws.com", + "https://iam.amazonaws.com", +} + +var APSoutheast2 = Region{ + "ap-southeast-2", //Asia Pacific (Sydney) + "https://ec2.ap-southeast-2.amazonaws.com", + "https://s3-ap-southeast-2.amazonaws.com", + "", + true, + true, + "https://sdb.ap-southeast-2.amazonaws.com", + "https://sns.ap-southeast-2.amazonaws.com", + "https://sqs.ap-southeast-2.amazonaws.com", + "https://iam.amazonaws.com", +} + +var APNortheast = Region{ + "ap-northeast-1", //Asia Pacific (Tokyo) + "https://ec2.ap-northeast-1.amazonaws.com", + "https://s3-ap-northeast-1.amazonaws.com", + "", + true, + true, + "https://sdb.ap-northeast-1.amazonaws.com", + "https://sns.ap-northeast-1.amazonaws.com", + "https://sqs.ap-northeast-1.amazonaws.com", + "https://iam.amazonaws.com", +} + +var SAEast = Region{ + "sa-east-1", // South America (Sao Paulo) + "https://ec2.sa-east-1.amazonaws.com", + "https://s3-sa-east-1.amazonaws.com", + "", + true, + true, + "https://sdb.sa-east-1.amazonaws.com", + "https://sns.sa-east-1.amazonaws.com", + "https://sqs.sa-east-1.amazonaws.com", + "https://iam.amazonaws.com", +} + +var CNNorth = Region{ + "cn-north-1", // Isolated regions, China (Beijing) + "https://ec2.cn-north-1.amazonaws.com.cn", + "https://s3.cn-north-1.amazonaws.com.cn", + "", + true, + true, + "https://sdb.cn-north-1.amazonaws.com.cn", + "https://sns.cn-north-1.amazonaws.com.cn", + "https://sqs.cn-north-1.amazonaws.com.cn", + "https://iam.cn-north-1.amazonaws.com.cn", +} + +var Regions = map[string]Region{ + APNortheast.Name: APNortheast, + APSoutheast.Name: APSoutheast, + APSoutheast2.Name: APSoutheast2, + EUWest.Name: EUWest, + EUCentral.Name: EUCentral, + USEast.Name: USEast, + USWest.Name: USWest, + USWest2.Name: USWest2, + USGovWest.Name: USGovWest, + SAEast.Name: SAEast, + CNNorth.Name: CNNorth, +} + +type Auth struct { + AccessKey, SecretKey string +} + +var unreserved = make([]bool, 128) +var hex = "0123456789ABCDEF" + +func init() { + // RFC3986 + u := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234567890-_.~" + for _, c := range u { + unreserved[c] = true + } +} + +// EnvAuth creates an Auth based on environment information. +// The AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment +// variables are used as the first preference, but EC2_ACCESS_KEY +// and EC2_SECRET_KEY or AWS_ACCESS_KEY and AWS_SECRET_KEY +// environment variables are also supported. +func EnvAuth() (auth Auth, err error) { + auth.AccessKey = os.Getenv("AWS_ACCESS_KEY_ID") + auth.SecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY") + // first fallbaback to EC2_ env variable + if auth.AccessKey == "" && auth.SecretKey == "" { + auth.AccessKey = os.Getenv("EC2_ACCESS_KEY") + auth.SecretKey = os.Getenv("EC2_SECRET_KEY") + } + // second fallbaback to AWS_ env variable + if auth.AccessKey == "" && auth.SecretKey == "" { + auth.AccessKey = os.Getenv("AWS_ACCESS_KEY") + auth.SecretKey = os.Getenv("AWS_SECRET_KEY") + } + if auth.AccessKey == "" { + err = errors.New("AWS_ACCESS_KEY_ID not found in environment") + } + if auth.SecretKey == "" { + err = errors.New("AWS_SECRET_ACCESS_KEY not found in environment") + } + return +} + +// Encode takes a string and URI-encodes it in a way suitable +// to be used in AWS signatures. +func Encode(s string) string { + encode := false + for i := 0; i != len(s); i++ { + c := s[i] + if c > 127 || !unreserved[c] { + encode = true + break + } + } + if !encode { + return s + } + e := make([]byte, len(s)*3) + ei := 0 + for i := 0; i != len(s); i++ { + c := s[i] + if c > 127 || !unreserved[c] { + e[ei] = '%' + e[ei+1] = hex[c>>4] + e[ei+2] = hex[c&0xF] + ei += 3 + } else { + e[ei] = c + ei += 1 + } + } + return string(e[:ei]) +} diff --git a/Godeps/_workspace/src/gopkg.in/amz.v3/aws/aws_test.go b/Godeps/_workspace/src/gopkg.in/amz.v3/aws/aws_test.go new file mode 100644 index 000000000..32fe8e56b --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/amz.v3/aws/aws_test.go @@ -0,0 +1,84 @@ +package aws_test + +import ( + "os" + "strings" + "testing" + + . "gopkg.in/check.v1" + + "gopkg.in/amz.v3/aws" +) + +func Test(t *testing.T) { + TestingT(t) +} + +var _ = Suite(&S{}) + +type S struct { + environ []string +} + +func (s *S) SetUpSuite(c *C) { + s.environ = os.Environ() +} + +func (s *S) TearDownTest(c *C) { + os.Clearenv() + for _, kv := range s.environ { + l := strings.SplitN(kv, "=", 2) + os.Setenv(l[0], l[1]) + } +} + +func (s *S) TestEnvAuthNoSecret(c *C) { + os.Clearenv() + _, err := aws.EnvAuth() + c.Assert(err, ErrorMatches, "AWS_SECRET_ACCESS_KEY not found in environment") +} + +func (s *S) TestEnvAuthNoAccess(c *C) { + os.Clearenv() + os.Setenv("AWS_SECRET_ACCESS_KEY", "foo") + _, err := aws.EnvAuth() + c.Assert(err, ErrorMatches, "AWS_ACCESS_KEY_ID not found in environment") +} + +func (s *S) TestEnvAuth(c *C) { + os.Clearenv() + os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") + os.Setenv("AWS_ACCESS_KEY_ID", "access") + auth, err := aws.EnvAuth() + c.Assert(err, IsNil) + c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) +} + +func (s *S) TestEnvAuthLegacy(c *C) { + os.Clearenv() + os.Setenv("EC2_SECRET_KEY", "secret") + os.Setenv("EC2_ACCESS_KEY", "access") + auth, err := aws.EnvAuth() + c.Assert(err, IsNil) + c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) +} + +func (s *S) TestEnvAuthAws(c *C) { + os.Clearenv() + os.Setenv("AWS_SECRET_KEY", "secret") + os.Setenv("AWS_ACCESS_KEY", "access") + auth, err := aws.EnvAuth() + c.Assert(err, IsNil) + c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) +} + +func (s *S) TestEncode(c *C) { + c.Assert(aws.Encode("foo"), Equals, "foo") + c.Assert(aws.Encode("/"), Equals, "%2F") +} + +func (s *S) TestRegionsAreNamed(c *C) { + for n, r := range aws.Regions { + c.Assert(n, Equals, r.Name) + } +} diff --git a/Godeps/_workspace/src/gopkg.in/amz.v3/aws/sign.go b/Godeps/_workspace/src/gopkg.in/amz.v3/aws/sign.go new file mode 100644 index 000000000..06c96e63b --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/amz.v3/aws/sign.go @@ -0,0 +1,447 @@ +package aws + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "net/url" + "sort" + "strings" + "time" +) + +var debug = log.New( + // Remove the c-style comment header to front of line to debug information. + /*os.Stdout, //*/ ioutil.Discard, + "DEBUG: ", + log.LstdFlags, +) + +type Signer func(*http.Request, Auth) error + +// Ensure our signers meet the interface +var _ Signer = SignV2 +var _ Signer = SignV4Factory("", "") + +type hasher func(io.Reader) (string, error) + +const ( + ISO8601BasicFormat = "20060102T150405Z" + ISO8601BasicFormatShort = "20060102" +) + +// SignV2 signs an HTTP request utilizing version 2 of the AWS +// signature, and the given credentials. +func SignV2(req *http.Request, auth Auth) (err error) { + + queryVals := req.URL.Query() + queryVals.Set("AWSAccessKeyId", auth.AccessKey) + queryVals.Set("SignatureVersion", "2") + queryVals.Set("SignatureMethod", "HmacSHA256") + + uriStr := canonicalURI(req.URL) + queryStr := canonicalQueryString(queryVals) + + payload := new(bytes.Buffer) + if err := errorCollector( + fprintfWrapper(payload, "%s\n", requestMethodVerb(req.Method)), + fprintfWrapper(payload, "%s\n", req.Host), + fprintfWrapper(payload, "%s\n", uriStr), + fprintfWrapper(payload, "%s", queryStr), + ); err != nil { + return err + } + + hash := hmac.New(sha256.New, []byte(auth.SecretKey)) + hash.Write(payload.Bytes()) + signature := make([]byte, base64.StdEncoding.EncodedLen(hash.Size())) + base64.StdEncoding.Encode(signature, hash.Sum(nil)) + + queryVals.Set("Signature", string(signature)) + req.URL.RawQuery = queryVals.Encode() + + return nil +} + +// SignV4Factory returns a version 4 Signer which will utilize the +// given region name. +func SignV4Factory(regionName, serviceName string) Signer { + return func(req *http.Request, auth Auth) error { + return SignV4(req, auth, regionName, serviceName) + } +} + +func SignV4URL(req *http.Request, auth Auth, regionName, svcName string, expires time.Duration) error { + reqTime, err := requestTime(req) + if err != nil { + return err + } + + req.Header.Del("date") + + credScope := credentialScope(reqTime, regionName, svcName) + + queryVals := req.URL.Query() + queryVals.Set("X-Amz-Algorithm", "AWS4-HMAC-SHA256") + queryVals.Set("X-Amz-Credential", auth.AccessKey+"/"+credScope) + queryVals.Set("X-Amz-Date", reqTime.Format(ISO8601BasicFormat)) + queryVals.Set("X-Amz-Expires", fmt.Sprintf("%d", int(expires.Seconds()))) + queryVals.Set("X-Amz-SignedHeaders", "host") + req.URL.RawQuery = queryVals.Encode() + + _, canonReqHash, _, err := canonicalRequest(req, sha256Hasher, false) + if err != nil { + return err + } + + var strToSign string + if strToSign, err = stringToSign(reqTime, canonReqHash, credScope); err != nil { + return err + } + + key := signingKey(reqTime, auth.SecretKey, regionName, svcName) + signature := fmt.Sprintf("%x", hmacHasher(key, strToSign)) + + debug.Printf("strToSign:\n\"\"\"\n%s\n\"\"\"", strToSign) + + queryVals.Set("X-Amz-Signature", signature) + + req.URL.RawQuery = queryVals.Encode() + + return nil +} + +// SignV4 signs an HTTP request utilizing version 4 of the AWS +// signature, and the given credentials. +func SignV4(req *http.Request, auth Auth, regionName, svcName string) (err error) { + + var reqTime time.Time + if reqTime, err = requestTime(req); err != nil { + return err + } + + // Remove any existing authorization headers as they will corrupt + // the signing. + delete(req.Header, "Authorization") + delete(req.Header, "authorization") + + credScope := credentialScope(reqTime, regionName, svcName) + + _, canonReqHash, sortedHdrNames, err := canonicalRequest(req, sha256Hasher, true) + if err != nil { + return err + } + + var strToSign string + if strToSign, err = stringToSign(reqTime, canonReqHash, credScope); err != nil { + return err + } + + key := signingKey(reqTime, auth.SecretKey, regionName, svcName) + signature := fmt.Sprintf("%x", hmacHasher(key, strToSign)) + + debug.Printf("strToSign:\n\"\"\"\n%s\n\"\"\"", strToSign) + + var authHdrVal string + if authHdrVal, err = authHeaderString( + req.Header, + auth.AccessKey, + signature, + credScope, + sortedHdrNames, + ); err != nil { + return err + } + + req.Header.Set("Authorization", authHdrVal) + + return nil +} + +// Task 1: Create a Canonical Request. +// Returns the canonical request, and its hash. +func canonicalRequest( + req *http.Request, + hasher hasher, + calcPayHash bool, +) (canReq, canReqHash string, sortedHdrNames []string, err error) { + + payHash := "UNSIGNED-PAYLOAD" + if calcPayHash { + if payHash, err = payloadHash(req, hasher); err != nil { + return + } + req.Header.Set("x-amz-content-sha256", payHash) + } + + sortedHdrNames = sortHeaderNames(req.Header, "host") + var canHdr string + if canHdr, err = canonicalHeaders(sortedHdrNames, req.Host, req.Header); err != nil { + return + } + + debug.Printf("canHdr:\n\"\"\"\n%s\n\"\"\"", canHdr) + debug.Printf("signedHeader: %s\n\n", strings.Join(sortedHdrNames, ";")) + + uriStr := canonicalURI(req.URL) + queryStr := canonicalQueryString(req.URL.Query()) + + c := new(bytes.Buffer) + if err := errorCollector( + fprintfWrapper(c, "%s\n", requestMethodVerb(req.Method)), + fprintfWrapper(c, "%s\n", uriStr), + fprintfWrapper(c, "%s\n", queryStr), + fprintfWrapper(c, "%s\n", canHdr), + fprintfWrapper(c, "%s\n", strings.Join(sortedHdrNames, ";")), + fprintfWrapper(c, "%s", payHash), + ); err != nil { + return "", "", nil, err + } + + canReq = c.String() + debug.Printf("canReq:\n\"\"\"\n%s\n\"\"\"", canReq) + canReqHash, err = hasher(bytes.NewBuffer([]byte(canReq))) + + return canReq, canReqHash, sortedHdrNames, err +} + +// Task 2: Create a string to Sign +// Returns a string in the defined format to sign for the authorization header. +func stringToSign( + t time.Time, + canonReqHash string, + credScope string, +) (string, error) { + w := new(bytes.Buffer) + if err := errorCollector( + fprintfWrapper(w, "AWS4-HMAC-SHA256\n"), + fprintfWrapper(w, "%s\n", t.Format(ISO8601BasicFormat)), + fprintfWrapper(w, "%s\n", credScope), + fprintfWrapper(w, "%s", canonReqHash), + ); err != nil { + return "", err + } + + return w.String(), nil +} + +// Task 3: Calculate the Signature +// Returns a derived signing key. +func signingKey(t time.Time, secretKey, regionName, svcName string) []byte { + + kSecret := secretKey + kDate := hmacHasher([]byte("AWS4"+kSecret), t.Format(ISO8601BasicFormatShort)) + kRegion := hmacHasher(kDate, regionName) + kService := hmacHasher(kRegion, svcName) + kSigning := hmacHasher(kService, "aws4_request") + + return kSigning +} + +// Task 4: Add the Signing Information to the Request +// Returns a string to be placed in the Authorization header for the request. +func authHeaderString( + header http.Header, + accessKey, + signature string, + credScope string, + sortedHeaderNames []string, +) (string, error) { + w := new(bytes.Buffer) + if err := errorCollector( + fprintfWrapper(w, "AWS4-HMAC-SHA256 "), + fprintfWrapper(w, "Credential=%s/%s, ", accessKey, credScope), + fprintfWrapper(w, "SignedHeaders=%s, ", strings.Join(sortedHeaderNames, ";")), + fprintfWrapper(w, "Signature=%s", signature), + ); err != nil { + return "", err + } + + return w.String(), nil +} + +func canonicalURI(u *url.URL) string { + + // The algorithm states that if the path is empty, to just use a "/". + if u.Path == "" { + return "/" + } + + // Each path segment must be URI-encoded. + segments := strings.Split(u.Path, "/") + for i, segment := range segments { + segments[i] = goToAwsUrlEncoding(url.QueryEscape(segment)) + } + + return strings.Join(segments, "/") +} + +func canonicalQueryString(queryVals url.Values) string { + + // AWS dictates that if duplicate keys exist, their values be + // sorted as well. + for _, values := range queryVals { + sort.Strings(values) + } + + return goToAwsUrlEncoding(queryVals.Encode()) +} + +func goToAwsUrlEncoding(urlEncoded string) string { + // AWS dictates that we use %20 for encoding spaces rather than +. + // All significant +s should already be encoded into their + // hexadecimal equivalents before doing the string replace. + return strings.Replace(urlEncoded, "+", "%20", -1) +} + +func canonicalHeaders(sortedHeaderNames []string, host string, hdr http.Header) (string, error) { + buffer := new(bytes.Buffer) + + for _, hName := range sortedHeaderNames { + + hdrVals := host + if hName != "host" { + canonHdrKey := http.CanonicalHeaderKey(hName) + sortedHdrVals := hdr[canonHdrKey] + sort.Strings(sortedHdrVals) + hdrVals = strings.Join(sortedHdrVals, ",") + } + + if _, err := fmt.Fprintf(buffer, "%s:%s\n", hName, hdrVals); err != nil { + return "", err + } + } + + // There is intentionally a hanging newline at the end of the + // header list. + return buffer.String(), nil +} + +// Returns a SHA256 checksum of the request body. Represented as a +// lowercase hexadecimal string. +func payloadHash(req *http.Request, hasher hasher) (string, error) { + if req.Body == nil { + return hasher(bytes.NewBuffer(nil)) + } + + return hasher(req.Body) +} + +// Retrieve the header names, lower-case them, and sort them. +func sortHeaderNames(header http.Header, injectedNames ...string) []string { + + sortedNames := injectedNames + for hName, _ := range header { + sortedNames = append(sortedNames, strings.ToLower(hName)) + } + + sort.Strings(sortedNames) + + return sortedNames +} + +func hmacHasher(key []byte, value string) []byte { + h := hmac.New(sha256.New, key) + h.Write([]byte(value)) + return h.Sum(nil) +} + +func sha256Hasher(payloadReader io.Reader) (string, error) { + hasher := sha256.New() + _, err := io.Copy(hasher, payloadReader) + + return fmt.Sprintf("%x", hasher.Sum(nil)), err +} + +func credentialScope(t time.Time, regionName, svcName string) string { + return fmt.Sprintf( + "%s/%s/%s/aws4_request", + t.Format(ISO8601BasicFormatShort), + regionName, + svcName, + ) +} + +// We do a lot of fmt.Fprintfs in this package. Create a higher-order +// function to elide the bytes written return value so we can submit +// these calls to an error collector. +func fprintfWrapper(w io.Writer, format string, vals ...interface{}) func() error { + return func() error { + _, err := fmt.Fprintf(w, format, vals...) + return err + } +} + +// Poor man's maybe monad. +func errorCollector(writers ...func() error) error { + for _, writer := range writers { + if err := writer(); err != nil { + return err + } + } + + return nil +} + +// Retrieve the request time from the request. We will attempt to +// parse whatever we find, but we will not make up a request date for +// the user (i.e.: Magic!). +func requestTime(req *http.Request) (time.Time, error) { + + // Time formats to try. We want to do everything we can to accept + // all time formats, but ultimately we may fail. In the package + // scope so it doesn't get initialized for every request. + var timeFormats = []string{ + time.RFC822, + ISO8601BasicFormat, + time.RFC1123, + time.ANSIC, + time.UnixDate, + time.RubyDate, + time.RFC822Z, + time.RFC850, + time.RFC1123Z, + time.RFC3339, + time.RFC3339Nano, + time.Kitchen, + } + + // Get a date header. + var date string + if date = req.Header.Get("x-amz-date"); date == "" { + if date = req.Header.Get("date"); date == "" { + return time.Time{}, fmt.Errorf(`Could not retrieve a request date. Please provide one in either "x-amz-date", or "date".`) + } + } + + // Start attempting to parse + for _, format := range timeFormats { + if parsedTime, err := time.Parse(format, date); err == nil { + return parsedTime, nil + } + } + + return time.Time{}, fmt.Errorf( + "Could not parse the given date. Please utilize one of the following formats: %s", + strings.Join(timeFormats, ","), + ) +} + +// http.Request's Method member returns the entire method. Derive the +// verb. +func requestMethodVerb(rawMethod string) (verb string) { + verbPlus := strings.SplitN(rawMethod, " ", 2) + switch { + case len(verbPlus) == 0: // Per docs, Method will be empty if it's GET. + verb = "GET" + default: + verb = verbPlus[0] + } + return verb +} diff --git a/Godeps/_workspace/src/gopkg.in/amz.v3/aws/sign_test.go b/Godeps/_workspace/src/gopkg.in/amz.v3/aws/sign_test.go new file mode 100644 index 000000000..458df7e42 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/amz.v3/aws/sign_test.go @@ -0,0 +1,285 @@ +package aws + +import ( + "bytes" + "fmt" + "net/http" + "time" + + . "gopkg.in/check.v1" +) + +var _ = Suite(&SigningSuite{}) + +type SigningSuite struct{} + +// TODO(katco-): The signing methodology is a "one size fits all" +// approach. The hashes we check against don't include headers that +// are added in as requisite parts for S3. That doesn't mean the tests +// are invalid, or that signing is broken for these examples, but as +// long as we're adding heads in, it's impossible to know what the new +// signature should be. We should revaluate these later. See: +// https://github.com/go-amz/amz/issues/29 +const v4skipReason = `Extra headers present - cannot predict generated signature (issue #29).` + +// EC2 ReST authentication docs: http://goo.gl/fQmAN +var testAuth = Auth{"user", "secret"} + +func (s *SigningSuite) TestV4SignedUrl(c *C) { + + auth := Auth{"AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"} + req, err := http.NewRequest("GET", "https://examplebucket.s3.amazonaws.com/test.txt", nil) + req.Header.Add("date", "Fri, 24 May 2013 00:00:00 GMT") + c.Assert(err, IsNil) + err = SignV4URL(req, auth, USEast.Name, "s3", 86400*time.Second) + c.Assert(err, IsNil) + + c.Check(req.URL.String(), Equals, "https://examplebucket.s3.amazonaws.com/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404&X-Amz-SignedHeaders=host") +} + +func (s *SigningSuite) TestV4SignedUrlReserved(c *C) { + + auth := Auth{"AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"} + req, err := http.NewRequest("GET", "https://examplebucket.s3.amazonaws.com/some:reserved,characters", nil) + req.Header.Add("date", "Fri, 24 May 2013 00:00:00 GMT") + c.Assert(err, IsNil) + err = SignV4URL(req, auth, USEast.Name, "s3", 86400*time.Second) + c.Assert(err, IsNil) + + c.Check(req.URL.String(), Equals, "https://examplebucket.s3.amazonaws.com/some:reserved,characters?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-Signature=ac81e03593d6fc22ac045b9353b0242da755be2af80b981eb13917d8b9cf20a4&X-Amz-SignedHeaders=host") +} + +func (s *SigningSuite) TestV4StringToSign(c *C) { + + mockTime, err := time.Parse(time.RFC3339, "2011-09-09T23:36:00Z") + c.Assert(err, IsNil) + stringToSign, err := stringToSign( + mockTime, + "3511de7e95d28ecd39e9513b642aee07e54f4941150d8df8bf94b328ef7e55e2", + "20110909/us-east-1/iam/aws4_request", + ) + c.Assert(err, IsNil) + + const expected = `AWS4-HMAC-SHA256 +20110909T233600Z +20110909/us-east-1/iam/aws4_request +3511de7e95d28ecd39e9513b642aee07e54f4941150d8df8bf94b328ef7e55e2` + c.Assert(stringToSign, Equals, expected) +} + +func (s *SigningSuite) TestV4CanonicalRequest(c *C) { + + c.Skip(v4skipReason) + + body := new(bytes.Buffer) + _, err := fmt.Fprint(body, "Action=ListUsers&Version=2010-05-08") + c.Assert(err, IsNil) + + req, err := http.NewRequest("POST", "https://iam.amazonaws.com", body) + c.Assert(err, IsNil) + + req.Header.Add("content-type", "application/x-www-form-urlencoded; charset=utf-8") + req.Header.Add("host", req.URL.Host) + req.Header.Add("x-amz-date", "20110909T233600Z") + + canonReq, canonReqHash, _, err := canonicalRequest( + req, + sha256Hasher, + true, + ) + c.Assert(err, IsNil) + + const expected = `POST +/ + +content-type:application/x-www-form-urlencoded; charset=utf-8 +host:iam.amazonaws.com +x-amz-date:20110909T233600Z + +content-type;host;x-amz-date +b6359072c78d70ebee1e81adcbab4f01bf2c23245fa365ef83fe8f1f955085e2` + + c.Assert(canonReq, Equals, expected) + c.Assert(canonReqHash, Equals, "3511de7e95d28ecd39e9513b642aee07e54f4941150d8df8bf94b328ef7e55e2") +} + +func (s *SigningSuite) TestV4SigningKey(c *C) { + + c.Skip(v4skipReason) + + mockTime, err := time.Parse(time.RFC3339, "2011-09-09T23:36:00Z") + c.Assert(err, IsNil) + c.Assert( + fmt.Sprintf("%v", signingKey(mockTime, testAuth.SecretKey, USEast.Name, "iam")), + Equals, + "[152 241 216 137 254 196 244 66 26 220 82 43 171 12 225 248 46 105 41 194 98 237 21 229 169 76 144 239 209 227 176 231]") +} + +func (s *SigningSuite) TestV4BasicSignatureV4(c *C) { + + c.Skip(v4skipReason) + + body := new(bytes.Buffer) + + req, err := http.NewRequest("POST / http/1.1", "https://host.foo.com", body) + c.Assert(err, IsNil) + + req.Header.Add("Host", req.URL.Host) + req.Header.Add("Date", "Mon, 09 Sep 2011 23:36:00 GMT") + + testAuth := Auth{ + AccessKey: "AKIDEXAMPLE", + SecretKey: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + } + err = SignV4(req, testAuth, USEast.Name, "host") + c.Assert(err, IsNil) + + c.Assert(req.Header.Get("Authorization"), Equals, `AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request,SignedHeaders=date;host,Signature=22902d79e148b64e7571c3565769328423fe276eae4b26f83afceda9e767f726`) +} + +func (s *SigningSuite) TestV4MoreCompleteSignature(c *C) { + + req, err := http.NewRequest("GET", "https://examplebucket.s3.amazonaws.com/test.txt", nil) + c.Assert(err, IsNil) + + req.Header.Set("x-amz-date", "20130524T000000Z") + req.Header.Set("Range", "bytes=0-9") + + testAuth := Auth{ + AccessKey: "AKIAIOSFODNN7EXAMPLE", + SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + } + err = SignV4(req, testAuth, USEast.Name, "s3") + c.Assert(err, IsNil) + c.Check(req.Header.Get("Authorization"), Equals, "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41") +} + +// +// v2 Tests +// + +func (s *SigningSuite) TestV2BasicSignature(c *C) { + req, err := http.NewRequest("GET", "http://localhost/path", nil) + c.Assert(err, IsNil) + + SignV2(req, testAuth) + + query := req.URL.Query() + + c.Assert(query.Get("SignatureVersion"), Equals, "2") + c.Assert(query.Get("SignatureMethod"), Equals, "HmacSHA256") + expected := "6lSe5QyXum0jMVc7cOUz32/52ZnL7N5RyKRk/09yiK4=" + c.Assert(query.Get("Signature"), Equals, expected) +} + +func (s *SigningSuite) TestV2ParamSignature(c *C) { + + req, err := http.NewRequest("GET", "http://localhost/path", nil) + c.Assert(err, IsNil) + + query := req.URL.Query() + for i := 1; i <= 3; i++ { + query.Add(fmt.Sprintf("param%d", i), fmt.Sprintf("value%d", i)) + } + req.URL.RawQuery = query.Encode() + + SignV2(req, testAuth) + + expected := "XWOR4+0lmK8bD8CGDGZ4kfuSPbb2JibLJiCl/OPu1oU=" + c.Assert(req.URL.Query().Get("Signature"), Equals, expected) +} + +func (s *SigningSuite) TestV2ManyParams(c *C) { + + req, err := http.NewRequest("GET", "http://localhost/path", nil) + c.Assert(err, IsNil) + + query := req.URL.Query() + orderedVals := []int{10, 2, 3, 4, 5, 6, 7, 8, 9, 1} + for i, val := range orderedVals { + query.Add(fmt.Sprintf("param%d", i+1), fmt.Sprintf("value%d", val)) + } + req.URL.RawQuery = query.Encode() + + SignV2(req, testAuth) + + expected := "di0sjxIvezUgQ1SIL6i+C/H8lL+U0CQ9frLIak8jkVg=" + c.Assert(req.URL.Query().Get("Signature"), Equals, expected) +} + +func (s *SigningSuite) TestV2Escaping(c *C) { + + req, err := http.NewRequest("GET", "http://localhost/path", nil) + c.Assert(err, IsNil) + + query := req.URL.Query() + query.Add("Nonce", "+ +") + req.URL.RawQuery = query.Encode() + + err = SignV2(req, testAuth) + c.Assert(err, IsNil) + + query = req.URL.Query() + c.Assert(query.Get("Nonce"), Equals, "+ +") + + expected := "bqffDELReIqwjg/W0DnsnVUmfLK4wXVLO4/LuG+1VFA=" + c.Assert(query.Get("Signature"), Equals, expected) +} + +func (s *SigningSuite) TestV2SignatureExample1(c *C) { + + req, err := http.NewRequest("GET", "http://sdb.amazonaws.com/", nil) + c.Assert(err, IsNil) + + query := req.URL.Query() + query.Add("Timestamp", "2009-02-01T12:53:20+00:00") + query.Add("Version", "2007-11-07") + query.Add("Action", "ListDomains") + req.URL.RawQuery = query.Encode() + + SignV2(req, Auth{"access", "secret"}) + + expected := "okj96/5ucWBSc1uR2zXVfm6mDHtgfNv657rRtt/aunQ=" + c.Assert(req.URL.Query().Get("Signature"), Equals, expected) +} + +// Tests example from: +// http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html +// Specifically, good for testing case when URL does not contain a / +func (s *SigningSuite) TestV2SignatureTutorialExample(c *C) { + + req, err := http.NewRequest("GET", "https://elasticmapreduce.amazonaws.com/", nil) + c.Assert(err, IsNil) + + query := req.URL.Query() + query.Add("Timestamp", "2011-10-03T15:19:30") + query.Add("Version", "2009-03-31") + query.Add("Action", "DescribeJobFlows") + req.URL.RawQuery = query.Encode() + + testAuth := Auth{"AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"} + err = SignV2(req, testAuth) + c.Assert(err, IsNil) + c.Assert(req.URL.Query().Get("Signature"), Equals, "i91nKc4PWAt0JJIdXwz9HxZCJDdiy6cf/Mj6vPxyYIs=") +} + +// https://bugs.launchpad.net/goamz/+bug/1022749 +func (s *SigningSuite) TestSignatureWithEndpointPath(c *C) { + + req, err := http.NewRequest("GET", "http://localhost:4444/services/Cloud", nil) + c.Assert(err, IsNil) + + queryStr := req.URL.Query() + queryStr.Add("Action", "RebootInstances") + queryStr.Add("Version", "2011-12-15") + queryStr.Add("InstanceId.1", "i-10a64379") + queryStr.Add("Timestamp", time.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC).In(time.UTC).Format(time.RFC3339)) + req.URL.RawQuery = queryStr.Encode() + + err = SignV2(req, Auth{"abc", "123"}) + c.Assert(err, IsNil) + c.Assert(req.URL.Query().Get("Signature"), Equals, "gdG/vEm+c6ehhhfkrJy3+wuVzw/rzKR42TYelMwti7M=") + err = req.ParseForm() + c.Assert(err, IsNil) + c.Assert(req.Form["Signature"], DeepEquals, []string{"gdG/vEm+c6ehhhfkrJy3+wuVzw/rzKR42TYelMwti7M="}) +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/export_test.go b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/export_test.go similarity index 62% rename from Godeps/_workspace/src/github.com/mitchellh/goamz/s3/export_test.go rename to Godeps/_workspace/src/gopkg.in/amz.v3/s3/export_test.go index 15bbd6762..9c2c11f37 100644 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/export_test.go +++ b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/export_test.go @@ -1,11 +1,17 @@ package s3 import ( - "github.com/mitchellh/goamz/aws" + "net/http" + + "gopkg.in/amz.v3/aws" ) var originalStrategy = attempts +func BuildError(resp *http.Response) error { + return buildError(resp) +} + func SetAttemptStrategy(s *aws.AttemptStrategy) { if s == nil { attempts = originalStrategy @@ -14,8 +20,8 @@ func SetAttemptStrategy(s *aws.AttemptStrategy) { } } -func Sign(auth aws.Auth, method, path string, params, headers map[string][]string) { - sign(auth, method, path, params, headers) +func AttemptStrategy() aws.AttemptStrategy { + return attempts } func SetListPartsMax(n int) { diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/multi.go b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/multi.go similarity index 58% rename from Godeps/_workspace/src/github.com/mitchellh/goamz/s3/multi.go rename to Godeps/_workspace/src/gopkg.in/amz.v3/s3/multi.go index 6e318b31d..82036399b 100644 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/multi.go +++ b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/multi.go @@ -8,6 +8,7 @@ import ( "encoding/xml" "errors" "io" + "net/http" "sort" "strconv" ) @@ -49,39 +50,66 @@ type listMultiResp struct { // // See http://goo.gl/ePioY for details. func (b *Bucket) ListMulti(prefix, delim string) (multis []*Multi, prefixes []string, err error) { - params := map[string][]string{ - "uploads": {""}, - "max-uploads": {strconv.FormatInt(int64(listMultiMax), 10)}, - "prefix": {prefix}, - "delimiter": {delim}, + + req, err := http.NewRequest("GET", b.Region.ResolveS3BucketEndpoint(b.Name), nil) + if err != nil { + return nil, nil, err } + + query := req.URL.Query() + query.Add("uploads", "") + query.Add("max-uploads", strconv.FormatInt(int64(listMultiMax), 10)) + query.Add("prefix", prefix) + query.Add("delimiter", delim) + req.URL.RawQuery = query.Encode() + + addAmazonDateHeader(req.Header) + + // We need to resign every iteration because we're changing variables. + if err := b.S3.Sign(req, b.Auth); err != nil { + return nil, nil, err + } + for attempt := attempts.Start(); attempt.Next(); { - req := &request{ - method: "GET", - bucket: b.Name, - params: params, - } - var resp listMultiResp - err := b.S3.query(req, &resp) - if shouldRetry(err) && attempt.HasNext() { - continue + + resp, err := requestRetryLoop(req, attempts) + if err == nil { + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + err = buildError(resp) + } } if err != nil { + if shouldRetry(err) && attempt.HasNext() { + continue + } return nil, nil, err } - for i := range resp.Upload { - multi := &resp.Upload[i] + + var multiResp listMultiResp + if err := xml.NewDecoder(resp.Body).Decode(&multiResp); err != nil { + return nil, nil, err + } + resp.Body.Close() + + for i := range multiResp.Upload { + multi := &multiResp.Upload[i] multi.Bucket = b multis = append(multis, multi) } - prefixes = append(prefixes, resp.CommonPrefixes...) - if !resp.IsTruncated { + prefixes = append(prefixes, multiResp.CommonPrefixes...) + if !multiResp.IsTruncated { return multis, prefixes, nil } - params["key-marker"] = []string{resp.NextKeyMarker} - params["upload-id-marker"] = []string{resp.NextUploadIdMarker} - attempt = attempts.Start() // Last request worked. + + query := req.URL.Query() + query.Set("key-marker", multiResp.NextKeyMarker) + query.Set("upload-id-marker", multiResp.NextUploadIdMarker) + req.URL.RawQuery = query.Encode() + + // Last request worked; restart our counter. + attempt = attempts.Start() } + panic("unreachable") } @@ -106,35 +134,44 @@ func (b *Bucket) Multi(key, contType string, perm ACL) (*Multi, error) { // // See http://goo.gl/XP8kL for details. func (b *Bucket) InitMulti(key string, contType string, perm ACL) (*Multi, error) { - headers := map[string][]string{ - "Content-Type": {contType}, - "Content-Length": {"0"}, - "x-amz-acl": {string(perm)}, + + req, err := http.NewRequest("POST", b.Region.ResolveS3BucketEndpoint(b.Name), nil) + if err != nil { + return nil, err } - params := map[string][]string{ - "uploads": {""}, + req.URL.Path += key + + query := req.URL.Query() + query.Add("uploads", "") + req.URL.RawQuery = query.Encode() + + req.Header.Add("Content-Type", contType) + req.Header.Add("Content-Length", "0") + req.Header.Add("x-amz-acl", string(perm)) + addAmazonDateHeader(req.Header) + + if err := b.S3.Sign(req, b.Auth); err != nil { + return nil, err } - req := &request{ - method: "POST", - bucket: b.Name, - path: key, - headers: headers, - params: params, - } - var err error - var resp struct { - UploadId string `xml:"UploadId"` - } - for attempt := attempts.Start(); attempt.Next(); { - err = b.S3.query(req, &resp) - if !shouldRetry(err) { - break + + resp, err := requestRetryLoop(req, attempts) + if err == nil { + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + err = buildError(resp) } } if err != nil { return nil, err } - return &Multi{Bucket: b, Key: key, UploadId: resp.UploadId}, nil + + var multiResp struct { + UploadId string `xml:"UploadId"` + } + if err := xml.NewDecoder(resp.Body).Decode(&multiResp); err != nil { + return nil, err + } + + return &Multi{Bucket: b, Key: key, UploadId: multiResp.UploadId}, nil } // PutPart sends part n of the multipart upload, reading all the content from r. @@ -150,45 +187,51 @@ func (m *Multi) PutPart(n int, r io.ReadSeeker) (Part, error) { } func (m *Multi) putPart(n int, r io.ReadSeeker, partSize int64, md5b64 string) (Part, error) { - headers := map[string][]string{ - "Content-Length": {strconv.FormatInt(partSize, 10)}, - "Content-MD5": {md5b64}, + _, err := r.Seek(0, 0) + if err != nil { + return Part{}, err } - params := map[string][]string{ - "uploadId": {m.UploadId}, - "partNumber": {strconv.FormatInt(int64(n), 10)}, + + req, err := http.NewRequest("PUT", m.Bucket.Region.ResolveS3BucketEndpoint(m.Bucket.Name), r) + if err != nil { + return Part{}, err } - for attempt := attempts.Start(); attempt.Next(); { - _, err := r.Seek(0, 0) - if err != nil { - return Part{}, err - } - req := &request{ - method: "PUT", - bucket: m.Bucket.Name, - path: m.Key, - headers: headers, - params: params, - payload: r, - } - err = m.Bucket.S3.prepare(req) - if err != nil { - return Part{}, err - } - resp, err := m.Bucket.S3.run(req, nil) - if shouldRetry(err) && attempt.HasNext() { - continue - } - if err != nil { - return Part{}, err - } - etag := resp.Header.Get("ETag") - if etag == "" { - return Part{}, errors.New("part upload succeeded with no ETag") - } - return Part{n, etag, partSize}, nil + req.Close = true + req.URL.Path += m.Key + req.ContentLength = partSize + + query := req.URL.Query() + query.Add("uploadId", m.UploadId) + query.Add("partNumber", strconv.FormatInt(int64(n), 10)) + req.URL.RawQuery = query.Encode() + + req.Header.Add("Content-MD5", md5b64) + addAmazonDateHeader(req.Header) + + if err := m.Bucket.S3.Sign(req, m.Bucket.Auth); err != nil { + return Part{}, err } - panic("unreachable") + // Signing may read the request body. + if _, err := r.Seek(0, 0); err != nil { + return Part{}, err + } + + resp, err := requestRetryLoop(req, attempts) + defer resp.Body.Close() + + if err != nil { + return Part{}, err + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return Part{}, buildError(resp) + } + + part := Part{n, resp.Header.Get("ETag"), partSize} + if part.ETag == "" { + return Part{}, errors.New("part upload succeeded with no ETag") + } + + return part, nil } func seekerInfo(r io.ReadSeeker) (size int64, md5hex string, md5b64 string, err error) { @@ -233,35 +276,62 @@ var listPartsMax = 1000 // // See http://goo.gl/ePioY for details. func (m *Multi) ListParts() ([]Part, error) { - params := map[string][]string{ - "uploadId": {m.UploadId}, - "max-parts": {strconv.FormatInt(int64(listPartsMax), 10)}, + + req, err := http.NewRequest("GET", m.Bucket.Region.ResolveS3BucketEndpoint(m.Bucket.Name), nil) + if err != nil { + return nil, err } + req.Close = true + req.URL.Path += m.Key + + query := req.URL.Query() + query.Add("uploadId", m.UploadId) + query.Add("max-parts", strconv.FormatInt(int64(listPartsMax), 10)) + req.URL.RawQuery = query.Encode() + var parts partSlice for attempt := attempts.Start(); attempt.Next(); { - req := &request{ - method: "GET", - bucket: m.Bucket.Name, - path: m.Key, - params: params, - } - var resp listPartsResp - err := m.Bucket.S3.query(req, &resp) - if shouldRetry(err) && attempt.HasNext() { - continue - } - if err != nil { + + addAmazonDateHeader(req.Header) + + // We need to resign every iteration because we're changing the URL. + if err := m.Bucket.S3.Sign(req, m.Bucket.Auth); err != nil { return nil, err } - parts = append(parts, resp.Part...) - if !resp.IsTruncated { - sort.Sort(parts) - return parts, nil + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } else if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + err = buildError(resp) } - params["part-number-marker"] = []string{resp.NextPartNumberMarker} - attempt = attempts.Start() // Last request worked. + + if err != nil { + if shouldRetry(err) && attempt.HasNext() { + continue + } + return nil, err + } + + var listResp listPartsResp + if err := xml.NewDecoder(resp.Body).Decode(&listResp); err != nil { + return nil, err + } + + parts = append(parts, listResp.Part...) + if listResp.IsTruncated == false { + break + } + + query.Set("part-number-marker", listResp.NextPartNumberMarker) + req.URL.RawQuery = query.Encode() + + // Last request worked; restart our counter. + attempt = attempts.Start() } - panic("unreachable") + + sort.Sort(parts) + return parts, nil } type ReaderAtSeeker interface { @@ -280,6 +350,7 @@ func (m *Multi) PutAll(r ReaderAtSeeker, partSize int64) ([]Part, error) { if err != nil && !hasCode(err, "NoSuchUpload") { return nil, err } + reuse := 0 // Index of next old part to consider reusing. current := 1 // Part number of latest good part handled. totalSize, err := r.Seek(0, 2) @@ -344,33 +415,54 @@ func (p completeParts) Swap(i, j int) { p[i], p[j] = p[j], p[i] } // // See http://goo.gl/2Z7Tw for details. func (m *Multi) Complete(parts []Part) error { - params := map[string][]string{ - "uploadId": {m.UploadId}, - } - c := completeUpload{} + + var c completeUpload for _, p := range parts { c.Parts = append(c.Parts, completePart{p.N, p.ETag}) } sort.Sort(c.Parts) + data, err := xml.Marshal(&c) if err != nil { return err } - for attempt := attempts.Start(); attempt.Next(); { - req := &request{ - method: "POST", - bucket: m.Bucket.Name, - path: m.Key, - params: params, - payload: bytes.NewReader(data), - } - err := m.Bucket.S3.query(req, nil) - if shouldRetry(err) && attempt.HasNext() { - continue - } + body := bytes.NewReader(data) + + req, err := http.NewRequest( + "POST", + m.Bucket.Region.ResolveS3BucketEndpoint(m.Bucket.Name), + body, + ) + if err != nil { return err } - panic("unreachable") + req.Close = true + req.ContentLength = int64(len(data)) + req.URL.Path += m.Key + + query := req.URL.Query() + query.Add("uploadId", m.UploadId) + req.URL.RawQuery = query.Encode() + + addAmazonDateHeader(req.Header) + + if err := m.Bucket.S3.Sign(req, m.Bucket.Auth); err != nil { + return err + } + // Signing may read the request body. + if _, err := body.Seek(0, 0); err != nil { + return err + } + + resp, err := requestRetryLoop(req, attempts) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return buildError(resp) + } + + return nil } // Abort deletes an unifinished multipart upload and any previously @@ -389,21 +481,22 @@ func (m *Multi) Complete(parts []Part) error { // // See http://goo.gl/dnyJw for details. func (m *Multi) Abort() error { - params := map[string][]string{ - "uploadId": {m.UploadId}, + + req, err := http.NewRequest("DELETE", m.Bucket.Region.ResolveS3BucketEndpoint(m.Bucket.Name), nil) + if err != nil { + return nil } - for attempt := attempts.Start(); attempt.Next(); { - req := &request{ - method: "DELETE", - bucket: m.Bucket.Name, - path: m.Key, - params: params, - } - err := m.Bucket.S3.query(req, nil) - if shouldRetry(err) && attempt.HasNext() { - continue - } + req.URL.Path += m.Key + + query := req.URL.Query() + query.Add("uploadId", m.UploadId) + req.URL.RawQuery = query.Encode() + + addAmazonDateHeader(req.Header) + + if err := m.Bucket.S3.Sign(req, m.Bucket.Auth); err != nil { return err } - panic("unreachable") + _, err = requestRetryLoop(req, attempts) + return err } diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/multi_test.go b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/multi_test.go similarity index 92% rename from Godeps/_workspace/src/github.com/mitchellh/goamz/s3/multi_test.go rename to Godeps/_workspace/src/gopkg.in/amz.v3/s3/multi_test.go index fbb03e046..44295d9a3 100644 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/multi_test.go +++ b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/multi_test.go @@ -2,17 +2,20 @@ package s3_test import ( "encoding/xml" - "github.com/mitchellh/goamz/s3" - . "github.com/motain/gocheck" "io" "io/ioutil" "strings" + + . "gopkg.in/check.v1" + + "gopkg.in/amz.v3/s3" ) func (s *S) TestInitMulti(c *C) { testServer.Response(200, nil, InitMultiResultDump) - b := s.s3.Bucket("sample") + b, err := s.s3.Bucket("sample") + c.Assert(err, IsNil) multi, err := b.InitMulti("multi", "text/plain", s3.Private) c.Assert(err, IsNil) @@ -29,12 +32,13 @@ func (s *S) TestInitMulti(c *C) { func (s *S) TestMultiNoPreviousUpload(c *C) { // Don't retry the NoSuchUpload error. - s.DisableRetries() + s3.RetryAttempts(false) testServer.Response(404, nil, NoSuchUploadErrorDump) testServer.Response(200, nil, InitMultiResultDump) - b := s.s3.Bucket("sample") + b, err := s.s3.Bucket("sample") + c.Assert(err, IsNil) multi, err := b.Multi("multi", "text/plain", s3.Private) c.Assert(err, IsNil) @@ -56,7 +60,8 @@ func (s *S) TestMultiNoPreviousUpload(c *C) { func (s *S) TestMultiReturnOld(c *C) { testServer.Response(200, nil, ListMultiResultDump) - b := s.s3.Bucket("sample") + b, err := s.s3.Bucket("sample") + c.Assert(err, IsNil) multi, err := b.Multi("multi1", "text/plain", s3.Private) c.Assert(err, IsNil) @@ -76,7 +81,8 @@ func (s *S) TestListParts(c *C) { testServer.Response(404, nil, NoSuchUploadErrorDump) // :-( testServer.Response(200, nil, ListPartsResultDump2) - b := s.s3.Bucket("sample") + b, err := s.s3.Bucket("sample") + c.Assert(err, IsNil) multi, err := b.InitMulti("multi", "text/plain", s3.Private) c.Assert(err, IsNil) @@ -116,7 +122,8 @@ func (s *S) TestPutPart(c *C) { testServer.Response(200, nil, InitMultiResultDump) testServer.Response(200, headers, "") - b := s.s3.Bucket("sample") + b, err := s.s3.Bucket("sample") + c.Assert(err, IsNil) multi, err := b.InitMulti("multi", "text/plain", s3.Private) c.Assert(err, IsNil) @@ -147,7 +154,7 @@ func readAll(r io.Reader) string { func (s *S) TestPutAllNoPreviousUpload(c *C) { // Don't retry the NoSuchUpload error. - s.DisableRetries() + s3.RetryAttempts(false) etag1 := map[string]string{"ETag": `"etag1"`} etag2 := map[string]string{"ETag": `"etag2"`} @@ -158,17 +165,18 @@ func (s *S) TestPutAllNoPreviousUpload(c *C) { testServer.Response(200, etag2, "") testServer.Response(200, etag3, "") - b := s.s3.Bucket("sample") + b, err := s.s3.Bucket("sample") + c.Assert(err, IsNil) multi, err := b.InitMulti("multi", "text/plain", s3.Private) c.Assert(err, IsNil) parts, err := multi.PutAll(strings.NewReader("part1part2last"), 5) - c.Assert(parts, HasLen, 3) - c.Assert(parts[0].ETag, Equals, `"etag1"`) - c.Assert(parts[1].ETag, Equals, `"etag2"`) - c.Assert(parts[2].ETag, Equals, `"etag3"`) c.Assert(err, IsNil) + c.Assert(parts, HasLen, 3) + c.Check(parts[0].ETag, Equals, `"etag1"`) + c.Check(parts[1].ETag, Equals, `"etag2"`) + c.Check(parts[2].ETag, Equals, `"etag3"`) // Init testServer.WaitRequest() @@ -205,14 +213,15 @@ func (s *S) TestPutAllNoPreviousUpload(c *C) { func (s *S) TestPutAllZeroSizeFile(c *C) { // Don't retry the NoSuchUpload error. - s.DisableRetries() + s3.RetryAttempts(false) etag1 := map[string]string{"ETag": `"etag1"`} testServer.Response(200, nil, InitMultiResultDump) testServer.Response(404, nil, NoSuchUploadErrorDump) testServer.Response(200, etag1, "") - b := s.s3.Bucket("sample") + b, err := s.s3.Bucket("sample") + c.Assert(err, IsNil) multi, err := b.InitMulti("multi", "text/plain", s3.Private) c.Assert(err, IsNil) @@ -247,7 +256,8 @@ func (s *S) TestPutAllResume(c *C) { testServer.Response(200, nil, ListPartsResultDump2) testServer.Response(200, etag2, "") - b := s.s3.Bucket("sample") + b, err := s.s3.Bucket("sample") + c.Assert(err, IsNil) multi, err := b.InitMulti("multi", "text/plain", s3.Private) c.Assert(err, IsNil) @@ -293,7 +303,8 @@ func (s *S) TestMultiComplete(c *C) { testServer.Response(200, nil, InternalErrorDump) testServer.Response(200, nil, "") - b := s.s3.Bucket("sample") + b, err := s.s3.Bucket("sample") + c.Assert(err, IsNil) multi, err := b.InitMulti("multi", "text/plain", s3.Private) c.Assert(err, IsNil) @@ -301,8 +312,9 @@ func (s *S) TestMultiComplete(c *C) { err = multi.Complete([]s3.Part{{2, `"ETag2"`, 32}, {1, `"ETag1"`, 64}}) c.Assert(err, IsNil) - testServer.WaitRequest() - req := testServer.WaitRequest() + // Grab the 2nd request. + req := testServer.WaitRequests(2)[1] + c.Assert(req.Method, Equals, "POST") c.Assert(req.URL.Path, Equals, "/sample/multi") c.Assert(req.Form.Get("uploadId"), Matches, "JNbR_[A-Za-z0-9.]+QQ--") @@ -315,8 +327,7 @@ func (s *S) TestMultiComplete(c *C) { } } - dec := xml.NewDecoder(req.Body) - err = dec.Decode(&payload) + err = xml.NewDecoder(req.Body).Decode(&payload) c.Assert(err, IsNil) c.Assert(payload.XMLName.Local, Equals, "CompleteMultipartUpload") @@ -331,7 +342,8 @@ func (s *S) TestMultiAbort(c *C) { testServer.Response(200, nil, InitMultiResultDump) testServer.Response(200, nil, "") - b := s.s3.Bucket("sample") + b, err := s.s3.Bucket("sample") + c.Assert(err, IsNil) multi, err := b.InitMulti("multi", "text/plain", s3.Private) c.Assert(err, IsNil) @@ -349,7 +361,8 @@ func (s *S) TestMultiAbort(c *C) { func (s *S) TestListMulti(c *C) { testServer.Response(200, nil, ListMultiResultDump) - b := s.s3.Bucket("sample") + b, err := s.s3.Bucket("sample") + c.Assert(err, IsNil) multis, prefixes, err := b.ListMulti("", "/") c.Assert(err, IsNil) diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/responses_test.go b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/responses_test.go similarity index 83% rename from Godeps/_workspace/src/github.com/mitchellh/goamz/s3/responses_test.go rename to Godeps/_workspace/src/gopkg.in/amz.v3/s3/responses_test.go index 98bba563e..7ceaba81b 100644 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/responses_test.go +++ b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/responses_test.go @@ -196,46 +196,3 @@ var InternalErrorDump = ` kjhwqk ` - -var GetKeyHeaderDump = map[string]string{ - "x-amz-id-2": "ef8yU9AS1ed4OpIszj7UDNEHGran", - "x-amz-request-id": "318BC8BC143432E5", - "x-amz-version-id": "3HL4kqtJlcpXroDTDmjVBH40Nrjfkd", - "Date": "Wed, 28 Oct 2009 22:32:00 GMT", - "Last-Modified": "Sun, 1 Jan 2006 12:00:00 GMT", - "ETag": "fba9dede5f27731c9771645a39863328", - "Content-Length": "434234", - "Content-Type": "text/plain", -} - -var GetListBucketsDump = ` - - - - bb5c0f63b0b25f2d0 - joe - - - - bucket1 - 2012-01-01T02:03:04.000Z - - - bucket2 - 2014-01-11T02:03:04.000Z - - - -` - -var MultiDelDump = ` - - - - a.go - - - b.go - - -` diff --git a/Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3.go b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3.go new file mode 100644 index 000000000..3f3472477 --- /dev/null +++ b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3.go @@ -0,0 +1,566 @@ +// +// goamz - Go packages to interact with the Amazon Web Services. +// +// https://wiki.ubuntu.com/goamz +// +// Copyright (c) 2011 Canonical Ltd. +// + +package s3 + +import ( + "bytes" + "encoding/xml" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "gopkg.in/amz.v3/aws" +) + +const debug = false + +// The S3 type encapsulates operations with an S3 region. +type S3 struct { + aws.Auth + aws.Region + Sign aws.Signer + private byte // Reserve the right of using private data. +} + +// The Bucket type encapsulates operations with an S3 bucket. +type Bucket struct { + *S3 + Name string +} + +// The Owner type represents the owner of the object in an S3 bucket. +type Owner struct { + ID string + DisplayName string +} + +var ( + attempts = defaultAttempts + defaultAttempts = aws.AttemptStrategy{ + Min: 5, + Total: 5 * time.Second, + Delay: 200 * time.Millisecond, + } +) + +// RetryAttempts sets whether failing S3 requests may be retried to cope +// with eventual consistency or temporary failures. It should not be +// called while operations are in progress. +func RetryAttempts(retry bool) { + if retry { + attempts = defaultAttempts + } else { + attempts = aws.AttemptStrategy{} + } +} + +// New creates a new S3. +func New(auth aws.Auth, region aws.Region) *S3 { + return &S3{auth, region, aws.SignV4Factory(region.Name, "s3"), 0} +} + +// Bucket returns a Bucket with the given name. +func (s3 *S3) Bucket(name string) (*Bucket, error) { + if strings.IndexAny(name, "/:@") >= 0 { + return nil, fmt.Errorf("bad S3 bucket: %q", name) + } + if s3.Region.S3BucketEndpoint != "" || s3.Region.S3LowercaseBucket { + name = strings.ToLower(name) + } + return &Bucket{s3, name}, nil +} + +var createBucketConfiguration = ` + %s +` + +// locationConstraint returns a *strings.Reader specifying a +// LocationConstraint if required for the region. +// +// See http://goo.gl/bh9Kq for details. +func (s3 *S3) locationConstraint() *strings.Reader { + constraint := "" + if s3.Region.S3LocationConstraint { + constraint = fmt.Sprintf(createBucketConfiguration, s3.Region.Name) + } + return strings.NewReader(constraint) +} + +type ACL string + +const ( + Private = ACL("private") + PublicRead = ACL("public-read") + PublicReadWrite = ACL("public-read-write") + AuthenticatedRead = ACL("authenticated-read") + BucketOwnerRead = ACL("bucket-owner-read") + BucketOwnerFull = ACL("bucket-owner-full-control") +) + +// Put inserts an object into the S3 bucket. +// +// See http://goo.gl/FEBPD for details. +func (b *Bucket) Put(path string, data []byte, contType string, perm ACL) error { + body := bytes.NewReader(data) + return b.PutReader(path, body, int64(len(data)), contType, perm) +} + +// PutBucket creates a new bucket. +// +// See http://goo.gl/ndjnR for details. +func (b *Bucket) PutBucket(perm ACL) error { + body := b.locationConstraint() + req, err := http.NewRequest("PUT", b.ResolveS3BucketEndpoint(b.Name), body) + if err != nil { + return err + } + req.Close = true + + addAmazonDateHeader(req.Header) + req.Header.Add("x-amz-acl", string(perm)) + + if err := b.S3.Sign(req, b.Auth); err != nil { + return err + } + // Signing may read the request body. + if _, err := body.Seek(0, 0); err != nil { + return err + } + + _, err = http.DefaultClient.Do(req) + return err +} + +// DelBucket removes an existing S3 bucket. All objects in the bucket must +// be removed before the bucket itself can be removed. +// +// See http://goo.gl/GoBrY for details. +func (b *Bucket) DelBucket() (err error) { + + req, err := http.NewRequest("DELETE", b.ResolveS3BucketEndpoint(b.Name), nil) + if err != nil { + return err + } + req.Close = true + addAmazonDateHeader(req.Header) + + if err := b.S3.Sign(req, b.Auth); err != nil { + return err + } + resp, err := requestRetryLoop(req, attempts) + if err != nil { + return err + } + resp.Body.Close() + return nil +} + +// Get retrieves an object from an S3 bucket. +// +// See http://goo.gl/isCO7 for details. +func (b *Bucket) Get(path string) (data []byte, err error) { + body, err := b.GetReader(path) + if err != nil { + return nil, err + } + defer body.Close() + + return ioutil.ReadAll(body) +} + +// GetReader retrieves an object from an S3 bucket. It is the caller's +// responsibility to call Close on rc when finished reading. +func (b *Bucket) GetReader(path string) (rc io.ReadCloser, err error) { + + req, err := http.NewRequest("GET", b.Region.ResolveS3BucketEndpoint(b.Name), nil) + if err != nil { + return nil, err + } + req.Close = true + req.URL.Path += path + + addAmazonDateHeader(req.Header) + + if err := b.S3.Sign(req, b.Auth); err != nil { + return nil, err + } + + resp, err := requestRetryLoop(req, attempts) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return nil, buildError(resp) + } + return resp.Body, nil +} + +// PutReader inserts an object into the S3 bucket by consuming data +// from r until EOF. Passing in an io.ReadSeeker for r will optimize +// the memory usage. +func (b *Bucket) PutReader(path string, r io.Reader, length int64, contType string, perm ACL) error { + return b.PutReaderWithHeader(path, r, length, contType, perm, http.Header{}) +} + +// PutReaderWithHeader inserts an object into the S3 bucket by +// consuming data from r until EOF. It also adds the headers provided +// to the request. Passing in an io.ReadSeeker for r will optimize the +// memory usage. +func (b *Bucket) PutReaderWithHeader(path string, r io.Reader, length int64, contType string, perm ACL, hdrs http.Header) error { + + // Convert the reader to a ReadSeeker so we can seek after + // signing. + seeker, ok := r.(io.ReadSeeker) + if !ok { + content, err := ioutil.ReadAll(r) + if err != nil { + return err + } + seeker = bytes.NewReader(content) + } + + req, err := http.NewRequest("PUT", b.Region.ResolveS3BucketEndpoint(b.Name), seeker) + if err != nil { + return err + } + req.Header = hdrs + req.Close = true + req.URL.Path += path + req.ContentLength = length + + req.Header.Add("Content-Type", contType) + req.Header.Add("x-amz-acl", string(perm)) + addAmazonDateHeader(req.Header) + + // Determine the current offset. + const seekFromPos = 1 + prevPos, err := seeker.Seek(0, seekFromPos) + if err != nil { + return err + } + + if err := b.S3.Sign(req, b.Auth); err != nil { + return err + } + // Signing may read the request body. + if _, err := seeker.Seek(prevPos, 0); err != nil { + return err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return buildError(resp) // closes body + } + + resp.Body.Close() + return nil +} + +// Del removes an object from the S3 bucket. +// +// See http://goo.gl/APeTt for details. +func (b *Bucket) Del(path string) error { + + req, err := http.NewRequest("DELETE", b.ResolveS3BucketEndpoint(b.Name), nil) + if err != nil { + return err + } + req.Close = true + req.URL.Path += path + + addAmazonDateHeader(req.Header) + + if err := b.S3.Sign(req, b.Auth); err != nil { + return err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + resp.Body.Close() + return nil +} + +// The ListResp type holds the results of a List bucket operation. +type ListResp struct { + Name string + Prefix string + Delimiter string + Marker string + NextMarker string + MaxKeys int + // IsTruncated is true if the results have been truncated because + // there are more keys and prefixes than can fit in MaxKeys. + // N.B. this is the opposite sense to that documented (incorrectly) in + // http://goo.gl/YjQTc + IsTruncated bool + Contents []Key + CommonPrefixes []string `xml:">Prefix"` +} + +// The Key type represents an item stored in an S3 bucket. +type Key struct { + Key string + LastModified string + Size int64 + // ETag gives the hex-encoded MD5 sum of the contents, + // surrounded with double-quotes. + ETag string + StorageClass string + Owner Owner +} + +// List returns information about objects in an S3 bucket. +// +// The prefix parameter limits the response to keys that begin with the +// specified prefix. +// +// The delim parameter causes the response to group all of the keys that +// share a common prefix up to the next delimiter in a single entry within +// the CommonPrefixes field. You can use delimiters to separate a bucket +// into different groupings of keys, similar to how folders would work. +// +// The marker parameter specifies the key to start with when listing objects +// in a bucket. Amazon S3 lists objects in alphabetical order and +// will return keys alphabetically greater than the marker. +// +// The max parameter specifies how many keys + common prefixes to return in +// the response. The default is 1000. +// +// For example, given these keys in a bucket: +// +// index.html +// index2.html +// photos/2006/January/sample.jpg +// photos/2006/February/sample2.jpg +// photos/2006/February/sample3.jpg +// photos/2006/February/sample4.jpg +// +// Listing this bucket with delimiter set to "/" would yield the +// following result: +// +// &ListResp{ +// Name: "sample-bucket", +// MaxKeys: 1000, +// Delimiter: "/", +// Contents: []Key{ +// {Key: "index.html", "index2.html"}, +// }, +// CommonPrefixes: []string{ +// "photos/", +// }, +// } +// +// Listing the same bucket with delimiter set to "/" and prefix set to +// "photos/2006/" would yield the following result: +// +// &ListResp{ +// Name: "sample-bucket", +// MaxKeys: 1000, +// Delimiter: "/", +// Prefix: "photos/2006/", +// CommonPrefixes: []string{ +// "photos/2006/February/", +// "photos/2006/January/", +// }, +// } +// +// See http://goo.gl/YjQTc for details. +func (b *Bucket) List(prefix, delim, marker string, max int) (*ListResp, error) { + + req, err := http.NewRequest("GET", b.ResolveS3BucketEndpoint(b.Name), nil) + if err != nil { + return nil, err + } + req.Close = true + + query := req.URL.Query() + query.Add("prefix", prefix) + query.Add("delimiter", delim) + query.Add("marker", marker) + if max != 0 { + query.Add("max-keys", strconv.FormatInt(int64(max), 10)) + } + req.URL.RawQuery = query.Encode() + + addAmazonDateHeader(req.Header) + + if err := b.S3.Sign(req, b.Auth); err != nil { + return nil, err + } + + resp, err := requestRetryLoop(req, attempts) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return nil, buildError(resp) // closes body + } + + var result ListResp + err = xml.NewDecoder(resp.Body).Decode(&result) + resp.Body.Close() + return &result, nil +} + +// URL returns a non-signed URL that allows retriving the +// object at path. It only works if the object is publicly +// readable (see SignedURL). +func (b *Bucket) URL(path string) string { + return b.ResolveS3BucketEndpoint(b.Name) + path +} + +// SignedURL returns a URL which can be used to fetch objects without +// signing for the given duration. +func (b *Bucket) SignedURL(path string, expires time.Duration) (string, error) { + req, err := http.NewRequest("GET", b.URL(path), nil) + if err != nil { + return "", err + } + req.Header.Add("date", time.Now().Format(aws.ISO8601BasicFormat)) + + if err := aws.SignV4URL(req, b.Auth, b.Region.Name, "s3", expires); err != nil { + return "", err + } + return req.URL.String(), nil +} + +type request struct { + method string + bucket string + path string + signpath string + params url.Values + headers http.Header + baseurl string + payload io.Reader + prepared bool +} + +func (req *request) url() (*url.URL, error) { + u, err := url.Parse(req.baseurl) + if err != nil { + return nil, fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err) + } + u.RawQuery = req.params.Encode() + u.Path = req.path + return u, nil +} + +// Error represents an error in an operation with S3. +type Error struct { + StatusCode int // HTTP status code (200, 403, ...) + Code string // EC2 error code ("UnsupportedOperation", ...) + Message string // The human-oriented error message + BucketName string + RequestId string + HostId string +} + +func (e *Error) Error() string { + return e.Message +} + +func buildError(r *http.Response) error { + if debug { + log.Printf("got error (status code %v)", r.StatusCode) + data, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Printf("\tread error: %v", err) + } else { + log.Printf("\tdata:\n%s\n\n", data) + } + r.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + } + + err := Error{} + // TODO return error if Unmarshal fails? + xml.NewDecoder(r.Body).Decode(&err) + r.Body.Close() + err.StatusCode = r.StatusCode + if err.Message == "" { + err.Message = r.Status + } + if debug { + log.Printf("err: %#v\n", err) + } + return &err +} + +func shouldRetry(err error) bool { + if err == nil { + return false + } + switch err { + case io.ErrUnexpectedEOF, io.EOF: + return true + } + switch e := err.(type) { + case *net.DNSError: + return true + case *net.OpError: + switch e.Op { + case "read", "write": + return true + } + case *Error: + switch e.Code { + case "InternalError", "NoSuchUpload", "NoSuchBucket": + return true + } + } + return false +} + +func hasCode(err error, code string) bool { + s3err, ok := err.(*Error) + return ok && s3err.Code == code +} + +// requestRetryLoop attempts to send the request until the given +// strategy says to stop. +func requestRetryLoop(req *http.Request, retryStrat aws.AttemptStrategy) (*http.Response, error) { + + for attempt := attempts.Start(); attempt.Next(); { + + if debug { + log.Printf("Full URL (in loop): %v", req.URL) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + if shouldRetry(err) && attempt.HasNext() { + continue + } + return nil, fmt.Errorf("making request: %v", err) + } + + if debug { + log.Printf("Full response (in loop): %v", resp) + } + + return resp, nil + } + + return nil, fmt.Errorf("could not complete the request within the specified retry attempts") +} + +func addAmazonDateHeader(header http.Header) { + header.Set("x-amz-date", time.Now().In(time.UTC).Format(aws.ISO8601BasicFormat)) +} diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3_test.go b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3_test.go similarity index 61% rename from Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3_test.go rename to Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3_test.go index 376c94700..7f9b4ad51 100644 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3_test.go +++ b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3_test.go @@ -5,13 +5,13 @@ import ( "io/ioutil" "net/http" "testing" - "time" - "github.com/mitchellh/goamz/aws" - "github.com/mitchellh/goamz/s3" - "github.com/mitchellh/goamz/testutil" - . "github.com/motain/gocheck" + . "gopkg.in/check.v1" + + "gopkg.in/amz.v3/aws" + "gopkg.in/amz.v3/s3" + "gopkg.in/amz.v3/testutil" ) func Test(t *testing.T) { @@ -28,12 +28,18 @@ var testServer = testutil.NewHTTPServer() func (s *S) SetUpSuite(c *C) { testServer.Start() - auth := aws.Auth{"abc", "123", ""} - s.s3 = s3.New(auth, aws.Region{Name: "faux-region-1", S3Endpoint: testServer.URL}) + s.s3 = s3.New( + aws.Auth{"abc", "123"}, + aws.Region{ + Name: "faux-region-1", + S3Endpoint: testServer.URL, + }, + ) } func (s *S) TearDownSuite(c *C) { s3.SetAttemptStrategy(nil) + testServer.Stop() } func (s *S) SetUpTest(c *C) { @@ -48,17 +54,14 @@ func (s *S) TearDownTest(c *C) { testServer.Flush() } -func (s *S) DisableRetries() { - s3.SetAttemptStrategy(&aws.AttemptStrategy{}) -} - // PutBucket docs: http://goo.gl/kBTCu func (s *S) TestPutBucket(c *C) { testServer.Response(200, nil, "") - b := s.s3.Bucket("bucket") - err := b.PutBucket(s3.Private) + b, err := s.s3.Bucket("bucket") + c.Assert(err, IsNil) + err = b.PutBucket(s3.Private) c.Assert(err, IsNil) req := testServer.WaitRequest() @@ -67,73 +70,11 @@ func (s *S) TestPutBucket(c *C) { c.Assert(req.Header["Date"], Not(Equals), "") } -// DeleteBucket docs: http://goo.gl/GoBrY - -func (s *S) TestDelBucket(c *C) { - testServer.Response(204, nil, "") - - b := s.s3.Bucket("bucket") - err := b.DelBucket() - c.Assert(err, IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, Equals, "DELETE") - c.Assert(req.URL.Path, Equals, "/bucket/") - c.Assert(req.Header["Date"], Not(Equals), "") -} - -// ListBuckets: http://goo.gl/NqlyMN - -func (s *S) TestListBuckets(c *C) { - testServer.Response(200, nil, GetListBucketsDump) - - buckets, err := s.s3.ListBuckets() - c.Assert(err, IsNil) - c.Assert(len(buckets.Buckets), Equals, 2) - c.Assert(buckets.Buckets[0].Name, Equals, "bucket1") - c.Assert(buckets.Buckets[1].Name, Equals, "bucket2") - - req := testServer.WaitRequest() - c.Assert(req.Method, Equals, "GET") - c.Assert(req.URL.Path, Equals, "/") -} - -// GetObject docs: http://goo.gl/isCO7 - -func (s *S) TestGet(c *C) { - testServer.Response(200, nil, "content") - - b := s.s3.Bucket("bucket") - data, err := b.Get("name") - - req := testServer.WaitRequest() - c.Assert(req.Method, Equals, "GET") - c.Assert(req.URL.Path, Equals, "/bucket/name") - c.Assert(req.Header["Date"], Not(Equals), "") - - c.Assert(err, IsNil) - c.Assert(string(data), Equals, "content") -} - -func (s *S) TestHead(c *C) { - testServer.Response(200, nil, "") - b := s.s3.Bucket("bucket") - resp, err := b.Head("name") - req := testServer.WaitRequest() - c.Assert(req.Method, Equals, "HEAD") - c.Assert(req.URL.Path, Equals, "/bucket/name") - c.Assert(req.Header["Date"], Not(Equals), "") - - c.Assert(err, IsNil) - body, err := ioutil.ReadAll(resp.Body) - c.Assert(err, IsNil) - c.Assert(len(body), Equals, 0) -} - func (s *S) TestURL(c *C) { testServer.Response(200, nil, "content") - b := s.s3.Bucket("bucket") + b, err := s.s3.Bucket("bucket") + c.Assert(err, IsNil) url := b.URL("name") r, err := http.Get(url) c.Assert(err, IsNil) @@ -147,10 +88,45 @@ func (s *S) TestURL(c *C) { c.Assert(req.URL.Path, Equals, "/bucket/name") } +// DeleteBucket docs: http://goo.gl/GoBrY + +func (s *S) TestDelBucket(c *C) { + testServer.Response(204, nil, "") + + b, err := s.s3.Bucket("bucket") + c.Assert(err, IsNil) + err = b.DelBucket() + c.Assert(err, IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "DELETE") + c.Assert(req.URL.Path, Equals, "/bucket/") + c.Assert(req.Header["Date"], Not(Equals), "") +} + +// GetObject docs: http://goo.gl/isCO7 + +func (s *S) TestGet(c *C) { + testServer.Response(200, nil, "content") + + b, err := s.s3.Bucket("bucket") + c.Assert(err, IsNil) + data, err := b.Get("name") + + req := testServer.WaitRequest() + c.Assert(req.Method, Equals, "GET") + c.Assert(req.URL.Path, Equals, "/bucket/name") + c.Assert(req.Header["Date"], Not(Equals), "") + + c.Assert(err, IsNil) + c.Assert(string(data), Equals, "content") +} + func (s *S) TestGetReader(c *C) { testServer.Response(200, nil, "content") - b := s.s3.Bucket("bucket") + b, err := s.s3.Bucket("bucket") + c.Assert(err, IsNil) rc, err := b.GetReader("name") c.Assert(err, IsNil) data, err := ioutil.ReadAll(rc) @@ -169,7 +145,8 @@ func (s *S) TestGetNotFound(c *C) { testServer.Response(404, nil, GetObjectErrorDump) } - b := s.s3.Bucket("non-existent-bucket") + b, err := s.s3.Bucket("non-existent-bucket") + c.Assert(err, IsNil) data, err := b.Get("non-existent") req := testServer.WaitRequest() @@ -194,30 +171,9 @@ func (s *S) TestGetNotFound(c *C) { func (s *S) TestPutObject(c *C) { testServer.Response(200, nil, "") - b := s.s3.Bucket("bucket") - err := b.Put("name", []byte("content"), "content-type", s3.Private) + b, err := s.s3.Bucket("bucket") c.Assert(err, IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, Equals, "PUT") - c.Assert(req.URL.Path, Equals, "/bucket/name") - c.Assert(req.Header["Date"], Not(DeepEquals), []string{""}) - c.Assert(req.Header["Content-Type"], DeepEquals, []string{"content-type"}) - c.Assert(req.Header["Content-Length"], DeepEquals, []string{"7"}) - //c.Assert(req.Header["Content-MD5"], DeepEquals, "...") - c.Assert(req.Header["X-Amz-Acl"], DeepEquals, []string{"private"}) -} - -func (s *S) TestPutObjectHeader(c *C) { - testServer.Response(200, nil, "") - - b := s.s3.Bucket("bucket") - err := b.PutHeader( - "name", - []byte("content"), - map[string][]string{"Content-Type": {"content-type"}}, - s3.Private, - ) + err = b.Put("name", []byte("content"), "content-type", s3.Private) c.Assert(err, IsNil) req := testServer.WaitRequest() @@ -233,9 +189,10 @@ func (s *S) TestPutObjectHeader(c *C) { func (s *S) TestPutReader(c *C) { testServer.Response(200, nil, "") - b := s.s3.Bucket("bucket") - buf := bytes.NewBufferString("content") - err := b.PutReader("name", buf, int64(buf.Len()), "content-type", s3.Private) + b, err := s.s3.Bucket("bucket") + c.Assert(err, IsNil) + buf := bytes.NewReader([]byte("content")) + err = b.PutReader("name", buf, int64(buf.Len()), "content-type", s3.Private) c.Assert(err, IsNil) req := testServer.WaitRequest() @@ -248,18 +205,15 @@ func (s *S) TestPutReader(c *C) { c.Assert(req.Header["X-Amz-Acl"], DeepEquals, []string{"private"}) } -func (s *S) TestPutReaderHeader(c *C) { +func (s *S) TestPutReaderWithHeader(c *C) { testServer.Response(200, nil, "") - b := s.s3.Bucket("bucket") - buf := bytes.NewBufferString("content") - err := b.PutReaderHeader( - "name", - buf, - int64(buf.Len()), - map[string][]string{"Content-Type": {"content-type"}}, - s3.Private, - ) + b, err := s.s3.Bucket("bucket") + c.Assert(err, IsNil) + buf := bytes.NewReader([]byte("content")) + err = b.PutReaderWithHeader("name", buf, int64(buf.Len()), "content-type", s3.Private, http.Header{ + "Cache-Control": []string{"max-age=5"}, + }) c.Assert(err, IsNil) req := testServer.WaitRequest() @@ -268,44 +222,8 @@ func (s *S) TestPutReaderHeader(c *C) { c.Assert(req.Header["Date"], Not(DeepEquals), []string{""}) c.Assert(req.Header["Content-Type"], DeepEquals, []string{"content-type"}) c.Assert(req.Header["Content-Length"], DeepEquals, []string{"7"}) - //c.Assert(req.Header["Content-MD5"], Equals, "...") - c.Assert(req.Header["X-Amz-Acl"], DeepEquals, []string{"private"}) -} - -func (s *S) TestCopy(c *C) { - testServer.Response(200, nil, "") - - b := s.s3.Bucket("bucket") - err := b.Copy( - "old/file", - "new/file", - s3.Private, - ) - c.Assert(err, IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, Equals, "PUT") - c.Assert(req.URL.Path, Equals, "/bucket/new/file") - c.Assert(req.Header["X-Amz-Copy-Source"], DeepEquals, []string{"/bucket/old/file"}) - c.Assert(req.Header["X-Amz-Acl"], DeepEquals, []string{"private"}) -} - -func (s *S) TestPlusInURL(c *C) { - testServer.Response(200, nil, "") - - b := s.s3.Bucket("bucket") - err := b.Copy( - "dir/old+f?le", - "dir/new+f?le", - s3.Private, - ) - c.Assert(err, IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, Equals, "PUT") - c.Assert(req.RequestURI, Equals, "/bucket/dir/new%2Bf%3Fle") - c.Assert(req.Header["X-Amz-Copy-Source"], DeepEquals, []string{"/bucket/dir/old%2Bf%3Fle"}) c.Assert(req.Header["X-Amz-Acl"], DeepEquals, []string{"private"}) + c.Assert(req.Header["Cache-Control"], DeepEquals, []string{"max-age=5"}) } // DelObject docs: http://goo.gl/APeTt @@ -313,8 +231,9 @@ func (s *S) TestPlusInURL(c *C) { func (s *S) TestDelObject(c *C) { testServer.Response(200, nil, "") - b := s.s3.Bucket("bucket") - err := b.Del("name") + b, err := s.s3.Bucket("bucket") + c.Assert(err, IsNil) + err = b.Del("name") c.Assert(err, IsNil) req := testServer.WaitRequest() @@ -323,32 +242,13 @@ func (s *S) TestDelObject(c *C) { c.Assert(req.Header["Date"], Not(Equals), "") } -// Delete Multiple Objects docs: http://goo.gl/WvA5sj - -func (s *S) TestMultiDelObject(c *C) { - testServer.Response(200, nil, "") - - b := s.s3.Bucket("bucket") - err := b.MultiDel([]string{"a", "b"}) - c.Assert(err, IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, Equals, "POST") - c.Assert(req.URL.Path, Equals, "/bucket/") - c.Assert(req.RequestURI, Equals, "/bucket/?delete=") - c.Assert(req.Header["Content-Md5"], DeepEquals, []string{"nos/vZNvjGs17xIyjEFlwQ=="}) - data, err := ioutil.ReadAll(req.Body) - req.Body.Close() - c.Assert(err, IsNil) - c.Assert(string(data), Equals, "falseab") -} - // Bucket List Objects docs: http://goo.gl/YjQTc func (s *S) TestList(c *C) { testServer.Response(200, nil, GetListResultDump1) - b := s.s3.Bucket("quotes") + b, err := s.s3.Bucket("quotes") + c.Assert(err, IsNil) data, err := b.List("N", "", "", 0) c.Assert(err, IsNil) @@ -387,7 +287,8 @@ func (s *S) TestList(c *C) { func (s *S) TestListWithDelimiter(c *C) { testServer.Response(200, nil, GetListResultDump2) - b := s.s3.Bucket("quotes") + b, err := s.s3.Bucket("quotes") + c.Assert(err, IsNil) data, err := b.List("photos/2006/", "/", "some-marker", 1000) c.Assert(err, IsNil) @@ -410,26 +311,11 @@ func (s *S) TestListWithDelimiter(c *C) { c.Assert(data.CommonPrefixes, DeepEquals, []string{"photos/2006/feb/", "photos/2006/jan/"}) } -func (s *S) TestGetKey(c *C) { - testServer.Response(200, GetKeyHeaderDump, "") - - b := s.s3.Bucket("bucket") - key, err := b.GetKey("name") - - req := testServer.WaitRequest() - c.Assert(req.Method, Equals, "HEAD") - c.Assert(req.URL.Path, Equals, "/bucket/name") - c.Assert(req.Header["Date"], Not(Equals), "") - - c.Assert(err, IsNil) - c.Assert(key.Key, Equals, "name") - c.Assert(key.LastModified, Equals, GetKeyHeaderDump["Last-Modified"]) - c.Assert(key.Size, Equals, int64(434234)) - c.Assert(key.ETag, Equals, GetKeyHeaderDump["ETag"]) -} - -func (s *S) TestUnescapedColon(c *C) { - b := s.s3.Bucket("bucket") - u := b.URL("foo:bar") - c.Assert(u, Equals, "http://localhost:4444/bucket/foo:bar") +func (s *S) TestRetryAttempts(c *C) { + s3.SetAttemptStrategy(nil) + orig := s3.AttemptStrategy() + s3.RetryAttempts(false) + c.Assert(s3.AttemptStrategy(), Equals, aws.AttemptStrategy{}) + s3.RetryAttempts(true) + c.Assert(s3.AttemptStrategy(), Equals, orig) } diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3i_test.go b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3i_test.go similarity index 85% rename from Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3i_test.go rename to Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3i_test.go index 9b3145d8a..875c7d498 100644 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3i_test.go +++ b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3i_test.go @@ -5,16 +5,17 @@ import ( "crypto/md5" "fmt" "io/ioutil" - "net/http" - "strings" - - "github.com/mitchellh/goamz/aws" - "github.com/mitchellh/goamz/s3" - "github.com/mitchellh/goamz/testutil" - . "github.com/motain/gocheck" "net" + "net/http" "sort" + "strings" "time" + + . "gopkg.in/check.v1" + + "gopkg.in/amz.v3/aws" + "gopkg.in/amz.v3/s3" + "gopkg.in/amz.v3/testutil" ) // AmazonServer represents an Amazon S3 server. @@ -32,7 +33,6 @@ func (s *AmazonServer) SetUp(c *C) { var _ = Suite(&AmazonClientSuite{Region: aws.USEast}) var _ = Suite(&AmazonClientSuite{Region: aws.EUWest}) -var _ = Suite(&AmazonClientSuite{Region: aws.EUCentral}) var _ = Suite(&AmazonDomainClientSuite{Region: aws.USEast}) // AmazonClientSuite tests the client against a live S3 server. @@ -71,7 +71,7 @@ func (s *AmazonDomainClientSuite) SetUpSuite(c *C) { } s.srv.SetUp(c) region := s.Region - region.S3BucketEndpoint = "https://${bucket}.s3.amazonaws.com" + region.S3BucketEndpoint += "https://s3.amazonaws.com/${bucket}/" s.s3 = s3.New(s.srv.auth, region) s.ClientTests.Cleanup() } @@ -84,8 +84,7 @@ func (s *AmazonDomainClientSuite) TearDownTest(c *C) { // It is not used as a test suite in itself, but embedded within // another type. type ClientTests struct { - s3 *s3.S3 - authIsBroken bool + s3 *s3.S3 } func (s *ClientTests) Cleanup() { @@ -99,7 +98,18 @@ func testBucket(s *s3.S3) *s3.Bucket { if len(key) >= 8 { key = s.Auth.AccessKey[:8] } - return s.Bucket(fmt.Sprintf("goamz-%s-%s", s.Region.Name, key)) + b, err := s.Bucket(strings.ToLower(fmt.Sprintf( + "goamz-%s-%s-%s", + s.Region.Name, + key, + // Add in the time element to help isolate tests from one + // another. + time.Now().Format("20060102T150405.999999999"), + ))) + if err != nil { + panic(err) + } + return b } var attempts = aws.AttemptStrategy{ @@ -143,26 +153,40 @@ func killBucket(b *s3.Bucket) { panic(message) } -func get(url string) ([]byte, error) { - for attempt := attempts.Start(); attempt.Next(); { - resp, err := http.Get(url) - if err != nil { - if attempt.HasNext() { - continue - } - return nil, err - } - data, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - if attempt.HasNext() { - continue - } - return nil, err - } - return data, err - } - panic("unreachable") +func (s *ClientTests) TestSignedUrl(c *C) { + b := testBucket(s.s3) + err := b.PutBucket(s3.PublicRead) + c.Assert(err, IsNil) + s.testSignedUrl(c, b, "name") + // Test that various special characters get escaped properly. + s.testSignedUrl(c, b, "&@$=:,!-_.*'( )") +} + +func (s *ClientTests) testSignedUrl(c *C, b *s3.Bucket, name string) { + err := b.Put(name, []byte("test for signed URLs."), "text/plain", s3.Private) + c.Assert(err, IsNil) + defer b.Del(name) + + req, err := http.NewRequest("GET", b.URL(name), nil) + c.Assert(err, IsNil) + resp, err := http.DefaultClient.Do(req) + c.Assert(err, IsNil) + err = s3.BuildError(resp) + c.Check(err, NotNil) + c.Check(err.(*s3.Error).Code, Equals, "AccessDenied") + + url, err := b.SignedURL(name, 24*time.Hour) + c.Assert(err, IsNil) + + req, err = http.NewRequest("GET", url, nil) + c.Assert(err, IsNil) + resp, err = http.DefaultClient.Do(req) + c.Assert(err, IsNil) + + body, err := ioutil.ReadAll(resp.Body) + c.Assert(err, IsNil) + + c.Check(string(body), Equals, "test for signed URLs.") } func (s *ClientTests) TestBasicFunctionality(c *C) { @@ -178,11 +202,7 @@ func (s *ClientTests) TestBasicFunctionality(c *C) { c.Assert(err, IsNil) c.Assert(string(data), Equals, "yo!") - data, err = get(b.URL("name")) - c.Assert(err, IsNil) - c.Assert(string(data), Equals, "yo!") - - buf := bytes.NewBufferString("hey!") + buf := bytes.NewReader([]byte("hey!")) err = b.PutReader("name2", buf, int64(buf.Len()), "text/plain", s3.Private) c.Assert(err, IsNil) defer b.Del("name2") @@ -194,25 +214,10 @@ func (s *ClientTests) TestBasicFunctionality(c *C) { c.Check(string(data), Equals, "hey!") rc.Close() - data, err = get(b.SignedURL("name2", time.Now().Add(time.Hour))) + data, err = b.Get("name2") c.Assert(err, IsNil) c.Assert(string(data), Equals, "hey!") - if !s.authIsBroken { - data, err = get(b.SignedURL("name2", time.Now().Add(-time.Hour))) - c.Assert(err, IsNil) - c.Assert(string(data), Matches, "(?s).*AccessDenied.*") - } - - err = b.DelBucket() - c.Assert(err, NotNil) - - s3err, ok := err.(*s3.Error) - c.Assert(ok, Equals, true) - c.Assert(s3err.Code, Equals, "BucketNotEmpty") - c.Assert(s3err.BucketName, Equals, b.Name) - c.Assert(s3err.Message, Equals, "The bucket you tried to delete is not empty") - err = b.Del("name") c.Assert(err, IsNil) err = b.Del("name2") @@ -222,33 +227,9 @@ func (s *ClientTests) TestBasicFunctionality(c *C) { c.Assert(err, IsNil) } -func (s *ClientTests) TestCopy(c *C) { - b := testBucket(s.s3) - err := b.PutBucket(s3.PublicRead) - - err = b.Put("name+1", []byte("yo!"), "text/plain", s3.PublicRead) - c.Assert(err, IsNil) - defer b.Del("name+1") - - err = b.Copy("name+1", "name+2", s3.PublicRead) - c.Assert(err, IsNil) - defer b.Del("name+2") - - data, err := b.Get("name+2") - c.Assert(err, IsNil) - c.Assert(string(data), Equals, "yo!") - - err = b.Del("name+1") - c.Assert(err, IsNil) - err = b.Del("name+2") - c.Assert(err, IsNil) - - err = b.DelBucket() - c.Assert(err, IsNil) -} - func (s *ClientTests) TestGetNotFound(c *C) { - b := s.s3.Bucket("goamz-" + s.s3.Auth.AccessKey) + b, err := s.s3.Bucket("goamz-" + s.s3.Auth.AccessKey) + c.Assert(err, IsNil) data, err := b.Get("non-existent") s3err, _ := err.(*s3.Error) @@ -261,28 +242,40 @@ func (s *ClientTests) TestGetNotFound(c *C) { // Communicate with all endpoints to see if they are alive. func (s *ClientTests) TestRegions(c *C) { - errs := make(chan error, len(aws.Regions)) + type result struct { + aws.Region + error + } + + results := make(chan result, len(aws.Regions)) for _, region := range aws.Regions { go func(r aws.Region) { s := s3.New(s.s3.Auth, r) - b := s.Bucket("goamz-" + s.Auth.AccessKey) - _, err := b.Get("non-existent") - errs <- err + b, err := s.Bucket("goamz-" + s.Auth.AccessKey) + if !c.Check(err, IsNil) { + return + } + + _, err = b.Get("non-existent") + if !c.Check(err, NotNil) { + return + } + + results <- result{r, err} }(region) } for _ = range aws.Regions { - err := <-errs - if err != nil { - s3_err, ok := err.(*s3.Error) - if ok { - c.Check(s3_err.Code, Matches, "NoSuchBucket") - } else if _, ok = err.(*net.DNSError); ok { - // Okay as well. - } else { - c.Errorf("Non-S3 error: %s", err) + result := <-results + if s3_err, ok := result.error.(*s3.Error); ok { + if result.Region == aws.CNNorth && s3_err.Code == "InvalidAccessKeyId" { + c.Log("You must utilize an account specifically for CNNorth.") + continue } + c.Check(s3_err.Code, Matches, "NoSuchBucket") + } else if _, ok = result.error.(*net.DNSError); ok { + // Okay as well. } else { - c.Errorf("Test should have errored but it seems to have succeeded") + c.Errorf("Non-S3 error: %s", result.error) } } } @@ -391,6 +384,7 @@ func (s *ClientTests) TestDoublePutBucket(c *C) { func (s *ClientTests) TestBucketList(c *C) { b := testBucket(s.s3) + defer b.DelBucket() err := b.PutBucket(s3.Private) c.Assert(err, IsNil) diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3t_test.go b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3t_test.go similarity index 86% rename from Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3t_test.go rename to Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3t_test.go index d3d49966e..37a08ba40 100644 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3t_test.go +++ b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3t_test.go @@ -1,10 +1,11 @@ package s3_test import ( - "github.com/mitchellh/goamz/aws" - "github.com/mitchellh/goamz/s3" - "github.com/mitchellh/goamz/s3/s3test" - . "github.com/motain/gocheck" + . "gopkg.in/check.v1" + + "gopkg.in/amz.v3/aws" + "gopkg.in/amz.v3/s3" + "gopkg.in/amz.v3/s3/s3test" ) type LocalServer struct { @@ -52,9 +53,6 @@ var ( func (s *LocalServerSuite) SetUpSuite(c *C) { s.srv.SetUp(c) s.clientTests.s3 = s3.New(s.srv.auth, s.srv.region) - - // TODO Sadly the fake server ignores auth completely right now. :-( - s.clientTests.authIsBroken = true s.clientTests.Cleanup() } diff --git a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3test/server.go b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3test/server.go similarity index 94% rename from Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3test/server.go rename to Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3test/server.go index 40e915e94..5080c8df6 100644 --- a/Godeps/_workspace/src/github.com/mitchellh/goamz/s3/s3test/server.go +++ b/Godeps/_workspace/src/gopkg.in/amz.v3/s3/s3test/server.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "encoding/xml" "fmt" - "github.com/mitchellh/goamz/s3" "io" "io/ioutil" "log" @@ -19,6 +18,8 @@ import ( "strings" "sync" "time" + + "gopkg.in/amz.v3/s3" ) const debug = false @@ -226,13 +227,6 @@ var pathRegexp = regexp.MustCompile("/(([^/]+)(/(.*))?)?") // resourceForURL returns a resource object for the given URL. func (srv *Server) resourceForURL(u *url.URL) (r resource) { - - if u.Path == "/" { - return serviceResource{ - buckets: srv.buckets, - } - } - m := pathRegexp.FindStringSubmatch(u.Path) if m == nil { fatalf(404, "InvalidURI", "Couldn't parse the specified URI") @@ -290,37 +284,6 @@ func (nullResource) delete(a *action) interface{} { return notAllowed() } const timeFormat = "2006-01-02T15:04:05.000Z07:00" -type serviceResource struct { - buckets map[string]*bucket -} - -func (serviceResource) put(a *action) interface{} { return notAllowed() } -func (serviceResource) post(a *action) interface{} { return notAllowed() } -func (serviceResource) delete(a *action) interface{} { return notAllowed() } - -// GET on an s3 service lists the buckets. -// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTServiceGET.html -func (r serviceResource) get(a *action) interface{} { - type respBucket struct { - Name string - } - - type response struct { - Buckets []respBucket `xml:">Bucket"` - } - - resp := response{} - - for _, bucketPtr := range r.buckets { - bkt := respBucket{ - Name: bucketPtr.name, - } - resp.Buckets = append(resp.Buckets, bkt) - } - - return &resp -} - type bucketResource struct { name string bucket *bucket // non-nil if the bucket already exists. diff --git a/backend/s3/s3.go b/backend/s3/s3.go index b16a5283d..2f5d2d786 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -8,8 +8,8 @@ import ( "io/ioutil" "strings" - "github.com/mitchellh/goamz/aws" - "github.com/mitchellh/goamz/s3" + "gopkg.in/amz.v3/aws" + "gopkg.in/amz.v3/s3" "github.com/restic/restic/backend" ) @@ -50,7 +50,12 @@ func Open(regionname, bucketname string) (backend.Backend, error) { client := s3.New(auth, aws.Regions[regionname]) - return OpenS3Bucket(client.Bucket(bucketname), bucketname), nil + s3bucket, s3err := client.Bucket(bucketname) + if s3err != nil { + return nil, s3err + } + + return OpenS3Bucket(s3bucket, bucketname), nil } // Location returns this backend's location (the bucket name). @@ -97,8 +102,8 @@ func (bb *s3Blob) Finalize(t backend.Type, name string) error { path := s3path(t, name) // Check key does not already exist - key, err := bb.b.bucket.GetKey(path) - if err == nil && key.Key == path { + _, err := bb.b.bucket.GetReader(path) + if err == nil { return errors.New("key already exists!") } @@ -153,8 +158,8 @@ func (be *S3Backend) GetReader(t backend.Type, name string, offset, length uint) func (be *S3Backend) Test(t backend.Type, name string) (bool, error) { found := false path := s3path(t, name) - key, err := be.bucket.GetKey(path) - if err == nil && key.Key == path { + _, err := be.bucket.GetReader(path) + if err == nil { found = true } diff --git a/backend/s3_test.go b/backend/s3_test.go index 1b021e92f..80cb11965 100644 --- a/backend/s3_test.go +++ b/backend/s3_test.go @@ -3,9 +3,9 @@ package backend_test import ( "testing" - "github.com/mitchellh/goamz/aws" - "github.com/mitchellh/goamz/s3" - "github.com/mitchellh/goamz/s3/s3test" + "gopkg.in/amz.v3/aws" + "gopkg.in/amz.v3/s3" + "gopkg.in/amz.v3/s3/s3test" bes3 "github.com/restic/restic/backend/s3" . "github.com/restic/restic/test" From 3bf447b422a32cd35eecc3f6586548ab63000bf1 Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Wed, 26 Aug 2015 07:44:00 -0500 Subject: [PATCH 2/3] Update tests for new s3 lib --- backend/s3_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/s3_test.go b/backend/s3_test.go index 80cb11965..0fdd40036 100644 --- a/backend/s3_test.go +++ b/backend/s3_test.go @@ -34,10 +34,11 @@ func setupS3Backend(t *testing.T) *bes3.S3Backend { S3LocationConstraint: true, // s3test server requires a LocationConstraint } - s.auth = aws.Auth{"abc", "123", ""} + s.auth = aws.Auth{"abc", "123"} service := s3.New(s.auth, s.region) - bucket := service.Bucket("testbucket") + bucket, berr := service.Bucket("testbucket") + OK(t, err) err = bucket.PutBucket("private") OK(t, err) From ccb2f00b8a8ce93d981c5277008e2ec4effa7b21 Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Wed, 26 Aug 2015 07:54:39 -0500 Subject: [PATCH 3/3] typo --- backend/s3_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/s3_test.go b/backend/s3_test.go index 0fdd40036..afaa5f372 100644 --- a/backend/s3_test.go +++ b/backend/s3_test.go @@ -38,7 +38,7 @@ func setupS3Backend(t *testing.T) *bes3.S3Backend { service := s3.New(s.auth, s.region) bucket, berr := service.Bucket("testbucket") - OK(t, err) + OK(t, berr) err = bucket.PutBucket("private") OK(t, err)