mirror of
https://github.com/octoleo/restic.git
synced 2024-11-04 20:37:49 +00:00
215 lines
6.3 KiB
Go
215 lines
6.3 KiB
Go
/*
|
|
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package minio
|
|
|
|
import (
|
|
"crypto/md5"
|
|
"crypto/sha256"
|
|
"encoding/xml"
|
|
"io"
|
|
"io/ioutil"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/minio/minio-go/pkg/s3utils"
|
|
)
|
|
|
|
// xmlDecoder provide decoded value in xml.
|
|
func xmlDecoder(body io.Reader, v interface{}) error {
|
|
d := xml.NewDecoder(body)
|
|
return d.Decode(v)
|
|
}
|
|
|
|
// sum256 calculate sha256 sum for an input byte array.
|
|
func sum256(data []byte) []byte {
|
|
hash := sha256.New()
|
|
hash.Write(data)
|
|
return hash.Sum(nil)
|
|
}
|
|
|
|
// sumMD5 calculate sumMD5 sum for an input byte array.
|
|
func sumMD5(data []byte) []byte {
|
|
hash := md5.New()
|
|
hash.Write(data)
|
|
return hash.Sum(nil)
|
|
}
|
|
|
|
// getEndpointURL - construct a new endpoint.
|
|
func getEndpointURL(endpoint string, secure bool) (*url.URL, error) {
|
|
if strings.Contains(endpoint, ":") {
|
|
host, _, err := net.SplitHostPort(endpoint)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !s3utils.IsValidIP(host) && !s3utils.IsValidDomain(host) {
|
|
msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards."
|
|
return nil, ErrInvalidArgument(msg)
|
|
}
|
|
} else {
|
|
if !s3utils.IsValidIP(endpoint) && !s3utils.IsValidDomain(endpoint) {
|
|
msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards."
|
|
return nil, ErrInvalidArgument(msg)
|
|
}
|
|
}
|
|
// If secure is false, use 'http' scheme.
|
|
scheme := "https"
|
|
if !secure {
|
|
scheme = "http"
|
|
}
|
|
|
|
// Construct a secured endpoint URL.
|
|
endpointURLStr := scheme + "://" + endpoint
|
|
endpointURL, err := url.Parse(endpointURLStr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Validate incoming endpoint URL.
|
|
if err := isValidEndpointURL(*endpointURL); err != nil {
|
|
return nil, err
|
|
}
|
|
return endpointURL, nil
|
|
}
|
|
|
|
// closeResponse close non nil response with any response Body.
|
|
// convenient wrapper to drain any remaining data on response body.
|
|
//
|
|
// Subsequently this allows golang http RoundTripper
|
|
// to re-use the same connection for future requests.
|
|
func closeResponse(resp *http.Response) {
|
|
// Callers should close resp.Body when done reading from it.
|
|
// If resp.Body is not closed, the Client's underlying RoundTripper
|
|
// (typically Transport) may not be able to re-use a persistent TCP
|
|
// connection to the server for a subsequent "keep-alive" request.
|
|
if resp != nil && resp.Body != nil {
|
|
// Drain any remaining Body and then close the connection.
|
|
// Without this closing connection would disallow re-using
|
|
// the same connection for future uses.
|
|
// - http://stackoverflow.com/a/17961593/4465767
|
|
io.Copy(ioutil.Discard, resp.Body)
|
|
resp.Body.Close()
|
|
}
|
|
}
|
|
|
|
var emptySHA256 = sum256(nil)
|
|
|
|
// Sentinel URL is the default url value which is invalid.
|
|
var sentinelURL = url.URL{}
|
|
|
|
// Verify if input endpoint URL is valid.
|
|
func isValidEndpointURL(endpointURL url.URL) error {
|
|
if endpointURL == sentinelURL {
|
|
return ErrInvalidArgument("Endpoint url cannot be empty.")
|
|
}
|
|
if endpointURL.Path != "/" && endpointURL.Path != "" {
|
|
return ErrInvalidArgument("Endpoint url cannot have fully qualified paths.")
|
|
}
|
|
if strings.Contains(endpointURL.Host, ".s3.amazonaws.com") {
|
|
if !s3utils.IsAmazonEndpoint(endpointURL) {
|
|
return ErrInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'.")
|
|
}
|
|
}
|
|
if strings.Contains(endpointURL.Host, ".googleapis.com") {
|
|
if !s3utils.IsGoogleEndpoint(endpointURL) {
|
|
return ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Verify if input expires value is valid.
|
|
func isValidExpiry(expires time.Duration) error {
|
|
expireSeconds := int64(expires / time.Second)
|
|
if expireSeconds < 1 {
|
|
return ErrInvalidArgument("Expires cannot be lesser than 1 second.")
|
|
}
|
|
if expireSeconds > 604800 {
|
|
return ErrInvalidArgument("Expires cannot be greater than 7 days.")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// make a copy of http.Header
|
|
func cloneHeader(h http.Header) http.Header {
|
|
h2 := make(http.Header, len(h))
|
|
for k, vv := range h {
|
|
vv2 := make([]string, len(vv))
|
|
copy(vv2, vv)
|
|
h2[k] = vv2
|
|
}
|
|
return h2
|
|
}
|
|
|
|
// Filter relevant response headers from
|
|
// the HEAD, GET http response. The function takes
|
|
// a list of headers which are filtered out and
|
|
// returned as a new http header.
|
|
func filterHeader(header http.Header, filterKeys []string) (filteredHeader http.Header) {
|
|
filteredHeader = cloneHeader(header)
|
|
for _, key := range filterKeys {
|
|
filteredHeader.Del(key)
|
|
}
|
|
return filteredHeader
|
|
}
|
|
|
|
// regCred matches credential string in HTTP header
|
|
var regCred = regexp.MustCompile("Credential=([A-Z0-9]+)/")
|
|
|
|
// regCred matches signature string in HTTP header
|
|
var regSign = regexp.MustCompile("Signature=([[0-9a-f]+)")
|
|
|
|
// Redact out signature value from authorization string.
|
|
func redactSignature(origAuth string) string {
|
|
if !strings.HasPrefix(origAuth, signV4Algorithm) {
|
|
// Set a temporary redacted auth
|
|
return "AWS **REDACTED**:**REDACTED**"
|
|
}
|
|
|
|
/// Signature V4 authorization header.
|
|
|
|
// Strip out accessKeyID from:
|
|
// Credential=<access-key-id>/<date>/<aws-region>/<aws-service>/aws4_request
|
|
newAuth := regCred.ReplaceAllString(origAuth, "Credential=**REDACTED**/")
|
|
|
|
// Strip out 256-bit signature from: Signature=<256-bit signature>
|
|
return regSign.ReplaceAllString(newAuth, "Signature=**REDACTED**")
|
|
}
|
|
|
|
// Get default location returns the location based on the input
|
|
// URL `u`, if region override is provided then all location
|
|
// defaults to regionOverride.
|
|
//
|
|
// If no other cases match then the location is set to `us-east-1`
|
|
// as a last resort.
|
|
func getDefaultLocation(u url.URL, regionOverride string) (location string) {
|
|
if regionOverride != "" {
|
|
return regionOverride
|
|
}
|
|
if s3utils.IsAmazonChinaEndpoint(u) {
|
|
return "cn-north-1"
|
|
}
|
|
if s3utils.IsAmazonGovCloudEndpoint(u) {
|
|
return "us-gov-west-1"
|
|
}
|
|
// Default to location to 'us-east-1'.
|
|
return "us-east-1"
|
|
}
|