2
2
mirror of https://github.com/octoleo/restic.git synced 2024-11-27 07:16:40 +00:00
restic/vendor/github.com/kurin/blazer/base/integration_test.go
2017-09-13 14:09:48 +02:00

536 lines
17 KiB
Go

// Copyright 2016, Google
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package base
import (
"bytes"
"crypto/sha1"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"os"
"reflect"
"strings"
"testing"
"time"
"context"
)
const (
apiID = "B2_ACCOUNT_ID"
apiKey = "B2_SECRET_KEY"
)
const (
bucketName = "base-tests"
smallFileName = "TeenyTiny"
largeFileName = "BigBytes"
)
type zReader struct{}
func (zReader) Read(p []byte) (int, error) {
return len(p), nil
}
func TestStorage(t *testing.T) {
id := os.Getenv(apiID)
key := os.Getenv(apiKey)
if id == "" || key == "" {
t.Skipf("B2_ACCOUNT_ID or B2_SECRET_KEY unset; skipping integration tests")
}
ctx := context.Background()
// b2_authorize_account
b2, err := AuthorizeAccount(ctx, id, key)
if err != nil {
t.Fatal(err)
}
// b2_create_bucket
infoKey := "key"
infoVal := "val"
m := map[string]string{infoKey: infoVal}
rules := []LifecycleRule{
{
Prefix: "what/",
DaysNewUntilHidden: 5,
},
}
bname := id + "-" + bucketName
bucket, err := b2.CreateBucket(ctx, bname, "", m, rules)
if err != nil {
t.Fatal(err)
}
if bucket.Info[infoKey] != infoVal {
t.Errorf("%s: bucketInfo[%q] got %q, want %q", bucket.Name, infoKey, bucket.Info[infoKey], infoVal)
}
if len(bucket.LifecycleRules) != 1 {
t.Errorf("%s: lifecycle rules: got %d rules, wanted 1", bucket.Name, len(bucket.LifecycleRules))
}
defer func() {
// b2_delete_bucket
if err := bucket.DeleteBucket(ctx); err != nil {
t.Error(err)
}
}()
// b2_update_bucket
bucket.Info["new"] = "yay"
bucket.LifecycleRules = nil // Unset options should be a noop.
newBucket, err := bucket.Update(ctx)
if err != nil {
t.Errorf("%s: update bucket: %v", bucket.Name, err)
return
}
bucket = newBucket
if bucket.Info["new"] != "yay" {
t.Errorf("%s: info key \"new\": got %s, want \"yay\"", bucket.Name, bucket.Info["new"])
}
if len(bucket.LifecycleRules) != 1 {
t.Errorf("%s: lifecycle rules: got %d rules, wanted 1", bucket.Name, len(bucket.LifecycleRules))
}
// b2_list_buckets
buckets, err := b2.ListBuckets(ctx)
if err != nil {
t.Fatal(err)
}
var found bool
for _, bucket := range buckets {
if bucket.Name == bname {
found = true
break
}
}
if !found {
t.Errorf("%s: new bucket not found", bname)
}
// b2_get_upload_url
ue, err := bucket.GetUploadURL(ctx)
if err != nil {
t.Fatal(err)
}
// b2_upload_file
smallFile := io.LimitReader(zReader{}, 1024*50) // 50k
hash := sha1.New()
buf := &bytes.Buffer{}
w := io.MultiWriter(hash, buf)
if _, err := io.Copy(w, smallFile); err != nil {
t.Error(err)
}
smallSHA1 := fmt.Sprintf("%x", hash.Sum(nil))
smallInfoMap := map[string]string{
"one": "1",
"two": "2",
}
file, err := ue.UploadFile(ctx, buf, buf.Len(), smallFileName, "application/octet-stream", smallSHA1, smallInfoMap)
if err != nil {
t.Fatal(err)
}
defer func() {
// b2_delete_file_version
if err := file.DeleteFileVersion(ctx); err != nil {
t.Error(err)
}
}()
// b2_start_large_file
largeInfoMap := map[string]string{
"one_BILLION": "1e9",
"two_TRILLION": "2eSomething, I guess 2e12",
}
lf, err := bucket.StartLargeFile(ctx, largeFileName, "application/octet-stream", largeInfoMap)
if err != nil {
t.Fatal(err)
}
// b2_get_upload_part_url
fc, err := lf.GetUploadPartURL(ctx)
if err != nil {
t.Fatal(err)
}
// b2_upload_part
largeFile := io.LimitReader(zReader{}, 10e6) // 10M
for i := 0; i < 2; i++ {
r := io.LimitReader(largeFile, 5e6) // 5M
hash := sha1.New()
buf := &bytes.Buffer{}
w := io.MultiWriter(hash, buf)
if _, err := io.Copy(w, r); err != nil {
t.Error(err)
}
if _, err := fc.UploadPart(ctx, buf, fmt.Sprintf("%x", hash.Sum(nil)), buf.Len(), i+1); err != nil {
t.Error(err)
}
}
// b2_finish_large_file
lfile, err := lf.FinishLargeFile(ctx)
if err != nil {
t.Fatal(err)
}
// b2_get_file_info
smallInfo, err := file.GetFileInfo(ctx)
if err != nil {
t.Fatal(err)
}
compareFileAndInfo(t, smallInfo, smallFileName, smallSHA1, smallInfoMap)
largeInfo, err := lfile.GetFileInfo(ctx)
if err != nil {
t.Fatal(err)
}
compareFileAndInfo(t, largeInfo, largeFileName, "none", largeInfoMap)
defer func() {
if err := lfile.DeleteFileVersion(ctx); err != nil {
t.Error(err)
}
}()
clf, err := bucket.StartLargeFile(ctx, largeFileName, "application/octet-stream", nil)
if err != nil {
t.Fatal(err)
}
// b2_cancel_large_file
if err := clf.CancelLargeFile(ctx); err != nil {
t.Fatal(err)
}
// b2_list_file_names
files, _, err := bucket.ListFileNames(ctx, 100, "", "", "")
if err != nil {
t.Fatal(err)
}
if len(files) != 2 {
t.Errorf("expected 2 files, got %d: %v", len(files), files)
}
// b2_download_file_by_name
fr, err := bucket.DownloadFileByName(ctx, smallFileName, 0, 0)
if err != nil {
t.Fatal(err)
}
if fr.SHA1 != smallSHA1 {
t.Errorf("small file SHAs don't match: got %q, want %q", fr.SHA1, smallSHA1)
}
lbuf := &bytes.Buffer{}
if _, err := io.Copy(lbuf, fr); err != nil {
t.Fatal(err)
}
if lbuf.Len() != fr.ContentLength {
t.Errorf("small file retreived lengths don't match: got %d, want %d", lbuf.Len(), fr.ContentLength)
}
// b2_hide_file
hf, err := bucket.HideFile(ctx, smallFileName)
if err != nil {
t.Fatal(err)
}
defer func() {
if err := hf.DeleteFileVersion(ctx); err != nil {
t.Error(err)
}
}()
// b2_list_file_versions
files, _, _, err = bucket.ListFileVersions(ctx, 100, "", "", "", "")
if err != nil {
t.Fatal(err)
}
if len(files) != 3 {
t.Errorf("expected 3 files, got %d: %v", len(files), files)
}
// b2_get_download_authorization
if _, err := bucket.GetDownloadAuthorization(ctx, "foo/", 24*time.Hour); err != nil {
t.Errorf("failed to get download auth token: %v", err)
}
}
// This slow motion train wreck of a type exists to axe a net connection after
// N bytes have been written. Because of the specific bug it's built to test,
// it can't just *close* the connection, so it just sleeps forever.
type wonkyNetConn struct {
net.Conn
ctx context.Context // implode once cancelled
die *bool // only implode once
n int // bytes to allow before imploding, roughly
i int // bytes written
}
func (w *wonkyNetConn) Write(b []byte) (int, error) {
if w.i > w.n && w.ctx.Err() != nil && *w.die {
*w.die = false
select {}
}
n, err := w.Conn.Write(b)
w.i += n
return n, err
}
func newWonkyNetConn(ctx context.Context, die *bool, n int, netw, addr string) (net.Conn, error) {
conn, err := net.Dial(netw, addr)
if err != nil {
return nil, err
}
return &wonkyNetConn{
Conn: conn,
ctx: ctx,
n: n,
die: die,
}, nil
}
func makeBadDialContext(ctx context.Context) func(context.Context, string, string) (net.Conn, error) {
die := true
return func(noCtx context.Context, network, addr string) (net.Conn, error) {
return newWonkyNetConn(ctx, &die, 10000, network, addr)
}
}
func TestBadUpload(t *testing.T) {
id := os.Getenv(apiID)
key := os.Getenv(apiKey)
if id == "" || key == "" {
t.Skipf("B2_ACCOUNT_ID or B2_SECRET_KEY unset; skipping integration tests")
}
ctx := context.Background()
octx, ocancel := context.WithCancel(ctx)
defer ocancel()
badTransport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: makeBadDialContext(octx),
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
b2, err := AuthorizeAccount(ctx, id, key, Transport(badTransport))
if err != nil {
t.Fatal(err)
}
bname := id + "-" + bucketName
bucket, err := b2.CreateBucket(ctx, bname, "", nil, nil)
if err != nil {
t.Fatal(err)
}
defer func() {
if err := bucket.DeleteBucket(ctx); err != nil {
t.Error(err)
}
}()
ue, err := bucket.GetUploadURL(ctx)
if err != nil {
t.Fatal(err)
}
smallFile := io.LimitReader(zReader{}, 1024*50) // 50k
hash := sha1.New()
buf := &bytes.Buffer{}
w := io.MultiWriter(hash, buf)
if _, err := io.Copy(w, smallFile); err != nil {
t.Error(err)
}
smallSHA1 := fmt.Sprintf("%x", hash.Sum(nil))
ocancel()
go func() {
ue.UploadFile(ctx, buf, buf.Len(), smallFileName, "application/octet-stream", smallSHA1, nil)
t.Fatal("this ought not to be reachable")
}()
time.Sleep(time.Second) // give this a chance to hang
// Do the whole thing again with the same upload auth, before the remote end
// notices we're gone.
smallFile = io.LimitReader(zReader{}, 1024*50) // 50k again
buf.Reset()
if _, err := io.Copy(buf, smallFile); err != nil {
t.Error(err)
}
file, err := ue.UploadFile(ctx, buf, buf.Len(), smallFileName, "application/octet-stream", smallSHA1, nil)
if err == nil {
t.Error("expected an error, got none")
if err := file.DeleteFileVersion(ctx); err != nil {
t.Error(err)
}
}
if Action(err) != AttemptNewUpload {
t.Error("Action(%v): got %v, want AttemptNewUpload", err, Action(err))
}
}
func compareFileAndInfo(t *testing.T, info *FileInfo, name, sha1 string, imap map[string]string) {
if info.Name != name {
t.Errorf("got %q, want %q", info.Name, name)
}
if info.SHA1 != sha1 {
t.Errorf("got %q, want %q", info.SHA1, sha1)
}
if !reflect.DeepEqual(info.Info, imap) {
t.Errorf("got %v, want %v", info.Info, imap)
}
}
// from https://www.backblaze.com/b2/docs/string_encoding.html
var testCases = `[
{"fullyEncoded": "%20", "minimallyEncoded": "+", "string": " "},
{"fullyEncoded": "%21", "minimallyEncoded": "!", "string": "!"},
{"fullyEncoded": "%22", "minimallyEncoded": "%22", "string": "\""},
{"fullyEncoded": "%23", "minimallyEncoded": "%23", "string": "#"},
{"fullyEncoded": "%24", "minimallyEncoded": "$", "string": "$"},
{"fullyEncoded": "%25", "minimallyEncoded": "%25", "string": "%"},
{"fullyEncoded": "%26", "minimallyEncoded": "%26", "string": "&"},
{"fullyEncoded": "%27", "minimallyEncoded": "'", "string": "'"},
{"fullyEncoded": "%28", "minimallyEncoded": "(", "string": "("},
{"fullyEncoded": "%29", "minimallyEncoded": ")", "string": ")"},
{"fullyEncoded": "%2A", "minimallyEncoded": "*", "string": "*"},
{"fullyEncoded": "%2B", "minimallyEncoded": "%2B", "string": "+"},
{"fullyEncoded": "%2C", "minimallyEncoded": "%2C", "string": ","},
{"fullyEncoded": "%2D", "minimallyEncoded": "-", "string": "-"},
{"fullyEncoded": "%2E", "minimallyEncoded": ".", "string": "."},
{"fullyEncoded": "/", "minimallyEncoded": "/", "string": "/"},
{"fullyEncoded": "%30", "minimallyEncoded": "0", "string": "0"},
{"fullyEncoded": "%31", "minimallyEncoded": "1", "string": "1"},
{"fullyEncoded": "%32", "minimallyEncoded": "2", "string": "2"},
{"fullyEncoded": "%33", "minimallyEncoded": "3", "string": "3"},
{"fullyEncoded": "%34", "minimallyEncoded": "4", "string": "4"},
{"fullyEncoded": "%35", "minimallyEncoded": "5", "string": "5"},
{"fullyEncoded": "%36", "minimallyEncoded": "6", "string": "6"},
{"fullyEncoded": "%37", "minimallyEncoded": "7", "string": "7"},
{"fullyEncoded": "%38", "minimallyEncoded": "8", "string": "8"},
{"fullyEncoded": "%39", "minimallyEncoded": "9", "string": "9"},
{"fullyEncoded": "%3A", "minimallyEncoded": ":", "string": ":"},
{"fullyEncoded": "%3B", "minimallyEncoded": ";", "string": ";"},
{"fullyEncoded": "%3C", "minimallyEncoded": "%3C", "string": "<"},
{"fullyEncoded": "%3D", "minimallyEncoded": "=", "string": "="},
{"fullyEncoded": "%3E", "minimallyEncoded": "%3E", "string": ">"},
{"fullyEncoded": "%3F", "minimallyEncoded": "%3F", "string": "?"},
{"fullyEncoded": "%40", "minimallyEncoded": "@", "string": "@"},
{"fullyEncoded": "%41", "minimallyEncoded": "A", "string": "A"},
{"fullyEncoded": "%42", "minimallyEncoded": "B", "string": "B"},
{"fullyEncoded": "%43", "minimallyEncoded": "C", "string": "C"},
{"fullyEncoded": "%44", "minimallyEncoded": "D", "string": "D"},
{"fullyEncoded": "%45", "minimallyEncoded": "E", "string": "E"},
{"fullyEncoded": "%46", "minimallyEncoded": "F", "string": "F"},
{"fullyEncoded": "%47", "minimallyEncoded": "G", "string": "G"},
{"fullyEncoded": "%48", "minimallyEncoded": "H", "string": "H"},
{"fullyEncoded": "%49", "minimallyEncoded": "I", "string": "I"},
{"fullyEncoded": "%4A", "minimallyEncoded": "J", "string": "J"},
{"fullyEncoded": "%4B", "minimallyEncoded": "K", "string": "K"},
{"fullyEncoded": "%4C", "minimallyEncoded": "L", "string": "L"},
{"fullyEncoded": "%4D", "minimallyEncoded": "M", "string": "M"},
{"fullyEncoded": "%4E", "minimallyEncoded": "N", "string": "N"},
{"fullyEncoded": "%4F", "minimallyEncoded": "O", "string": "O"},
{"fullyEncoded": "%50", "minimallyEncoded": "P", "string": "P"},
{"fullyEncoded": "%51", "minimallyEncoded": "Q", "string": "Q"},
{"fullyEncoded": "%52", "minimallyEncoded": "R", "string": "R"},
{"fullyEncoded": "%53", "minimallyEncoded": "S", "string": "S"},
{"fullyEncoded": "%54", "minimallyEncoded": "T", "string": "T"},
{"fullyEncoded": "%55", "minimallyEncoded": "U", "string": "U"},
{"fullyEncoded": "%56", "minimallyEncoded": "V", "string": "V"},
{"fullyEncoded": "%57", "minimallyEncoded": "W", "string": "W"},
{"fullyEncoded": "%58", "minimallyEncoded": "X", "string": "X"},
{"fullyEncoded": "%59", "minimallyEncoded": "Y", "string": "Y"},
{"fullyEncoded": "%5A", "minimallyEncoded": "Z", "string": "Z"},
{"fullyEncoded": "%5B", "minimallyEncoded": "%5B", "string": "["},
{"fullyEncoded": "%5C", "minimallyEncoded": "%5C", "string": "\\"},
{"fullyEncoded": "%5D", "minimallyEncoded": "%5D", "string": "]"},
{"fullyEncoded": "%5E", "minimallyEncoded": "%5E", "string": "^"},
{"fullyEncoded": "%5F", "minimallyEncoded": "_", "string": "_"},
{"fullyEncoded": "%60", "minimallyEncoded": "%60", "string": "` + "`" + `"},
{"fullyEncoded": "%61", "minimallyEncoded": "a", "string": "a"},
{"fullyEncoded": "%62", "minimallyEncoded": "b", "string": "b"},
{"fullyEncoded": "%63", "minimallyEncoded": "c", "string": "c"},
{"fullyEncoded": "%64", "minimallyEncoded": "d", "string": "d"},
{"fullyEncoded": "%65", "minimallyEncoded": "e", "string": "e"},
{"fullyEncoded": "%66", "minimallyEncoded": "f", "string": "f"},
{"fullyEncoded": "%67", "minimallyEncoded": "g", "string": "g"},
{"fullyEncoded": "%68", "minimallyEncoded": "h", "string": "h"},
{"fullyEncoded": "%69", "minimallyEncoded": "i", "string": "i"},
{"fullyEncoded": "%6A", "minimallyEncoded": "j", "string": "j"},
{"fullyEncoded": "%6B", "minimallyEncoded": "k", "string": "k"},
{"fullyEncoded": "%6C", "minimallyEncoded": "l", "string": "l"},
{"fullyEncoded": "%6D", "minimallyEncoded": "m", "string": "m"},
{"fullyEncoded": "%6E", "minimallyEncoded": "n", "string": "n"},
{"fullyEncoded": "%6F", "minimallyEncoded": "o", "string": "o"},
{"fullyEncoded": "%70", "minimallyEncoded": "p", "string": "p"},
{"fullyEncoded": "%71", "minimallyEncoded": "q", "string": "q"},
{"fullyEncoded": "%72", "minimallyEncoded": "r", "string": "r"},
{"fullyEncoded": "%73", "minimallyEncoded": "s", "string": "s"},
{"fullyEncoded": "%74", "minimallyEncoded": "t", "string": "t"},
{"fullyEncoded": "%75", "minimallyEncoded": "u", "string": "u"},
{"fullyEncoded": "%76", "minimallyEncoded": "v", "string": "v"},
{"fullyEncoded": "%77", "minimallyEncoded": "w", "string": "w"},
{"fullyEncoded": "%78", "minimallyEncoded": "x", "string": "x"},
{"fullyEncoded": "%79", "minimallyEncoded": "y", "string": "y"},
{"fullyEncoded": "%7A", "minimallyEncoded": "z", "string": "z"},
{"fullyEncoded": "%7B", "minimallyEncoded": "%7B", "string": "{"},
{"fullyEncoded": "%7C", "minimallyEncoded": "%7C", "string": "|"},
{"fullyEncoded": "%7D", "minimallyEncoded": "%7D", "string": "}"},
{"fullyEncoded": "%7E", "minimallyEncoded": "~", "string": "~"},
{"fullyEncoded": "%7F", "minimallyEncoded": "%7F", "string": "\u007f"},
{"fullyEncoded": "%E8%87%AA%E7%94%B1", "minimallyEncoded": "%E8%87%AA%E7%94%B1", "string": "\u81ea\u7531"},
{"fullyEncoded": "%F0%90%90%80", "minimallyEncoded": "%F0%90%90%80", "string": "\ud801\udc00"}
]`
type testCase struct {
Full string `json:"fullyEncoded"`
Min string `json:"minimallyEncoded"`
Raw string `json:"string"`
}
func TestEscapes(t *testing.T) {
dec := json.NewDecoder(strings.NewReader(testCases))
var tcs []testCase
if err := dec.Decode(&tcs); err != nil {
t.Fatal(err)
}
for _, tc := range tcs {
en := escape(tc.Raw)
if !(en == tc.Full || en == tc.Min) {
t.Errorf("encode %q: got %q, want %q or %q", tc.Raw, en, tc.Min, tc.Full)
}
m, err := unescape(tc.Min)
if err != nil {
t.Errorf("decode %q: %v", tc.Min, err)
}
if m != tc.Raw {
t.Errorf("decode %q: got %q, want %q", tc.Min, m, tc.Raw)
}
f, err := unescape(tc.Full)
if err != nil {
t.Errorf("decode %q: %v", tc.Full, err)
}
if f != tc.Raw {
t.Errorf("decode %q: got %q, want %q", tc.Full, f, tc.Raw)
}
}
}