diff --git a/go.mod b/go.mod index 8912e8f63..47d0cda41 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/restic/restic require ( bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512 - cloud.google.com/go v0.66.0 // indirect + cloud.google.com/go/storage v1.12.0 github.com/Azure/azure-sdk-for-go v46.1.0+incompatible github.com/Azure/go-autorest/autorest v0.11.6 // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect @@ -44,8 +44,6 @@ require ( golang.org/x/sys v0.0.0-20200918174421-af09f7315aff golang.org/x/text v0.3.3 google.golang.org/api v0.32.0 - google.golang.org/genproto v0.0.0-20200918140846-d0d605568037 // indirect - google.golang.org/grpc v1.32.0 // indirect gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect gopkg.in/ini.v1 v1.61.0 // indirect gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 diff --git a/go.sum b/go.sum index a89f0672f..77e7cf80d 100644 --- a/go.sum +++ b/go.sum @@ -33,7 +33,10 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4= +cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v46.1.0+incompatible h1:e9xxveqrMFRJgj44gychg6jYGfZbwwKhW4wGq9LEG8Q= github.com/Azure/azure-sdk-for-go v46.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -138,7 +141,9 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0 h1:pMen7vLs8nvgEYhywH3KDWJIJTeEr2ULsVWHWYHQyBs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -169,6 +174,7 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -307,6 +313,7 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -315,6 +322,7 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -456,7 +464,10 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c h1:AQsh/7arPVFDBraQa8x7GoVnwnGg1kM7J2ySI0kF5WU= golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20200918232735-d647fc253266 h1:k7tVuG0g1JwmD3Jh8oAl1vQ1C3jb4Hi/dUl1wWDBJpQ= +golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -524,8 +535,8 @@ google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200918140846-d0d605568037 h1:ujwz1DPMeHwCvo36rK5shXhAzc4GMRecrqQFaMZJBKQ= -google.golang.org/genproto v0.0.0-20200918140846-d0d605568037/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5 h1:B9nroC8SSX5GtbVvxPF9tYIVkaCpjhVLOrlAY8ONzm8= +google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index b0d805824..096420be7 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -3,13 +3,13 @@ package gs import ( "context" - "fmt" "io" "net/http" "os" "path" "strings" + "cloud.google.com/go/storage" "github.com/pkg/errors" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" @@ -18,8 +18,8 @@ import ( "golang.org/x/oauth2" "golang.org/x/oauth2/google" "google.golang.org/api/googleapi" + "google.golang.org/api/iterator" "google.golang.org/api/option" - storage "google.golang.org/api/storage/v1" ) // Backend stores data in a GCS bucket. @@ -30,10 +30,11 @@ import ( // * storage.objects.get // * storage.objects.list type Backend struct { - service *storage.Service + gcsClient *storage.Client projectID string sem *backend.Semaphore bucketName string + bucket *storage.BucketHandle prefix string listMaxItems int backend.Layout @@ -42,7 +43,7 @@ type Backend struct { // Ensure that *Backend implements restic.Backend. var _ restic.Backend = &Backend{} -func getStorageService(rt http.RoundTripper) (*storage.Service, error) { +func getStorageClient(rt http.RoundTripper) (*storage.Client, error) { // create a new HTTP client httpClient := &http.Client{ Transport: rt, @@ -59,20 +60,28 @@ func getStorageService(rt http.RoundTripper) (*storage.Service, error) { }) } else { var err error - ts, err = google.DefaultTokenSource(ctx, storage.DevstorageReadWriteScope) + ts, err = google.DefaultTokenSource(ctx, storage.ScopeReadWrite) if err != nil { return nil, err } } - client := oauth2.NewClient(ctx, ts) + oauthClient := oauth2.NewClient(ctx, ts) - service, err := storage.NewService(ctx, option.WithHTTPClient(client)) + gcsClient, err := storage.NewClient(ctx, option.WithHTTPClient(oauthClient)) if err != nil { return nil, err } - return service, nil + return gcsClient, nil +} + +func (be *Backend) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) { + _, err := bucket.Attrs(ctx) + if err == storage.ErrBucketNotExist { + return false, nil + } + return err == nil, err } const defaultListMaxItems = 1000 @@ -80,9 +89,9 @@ const defaultListMaxItems = 1000 func open(cfg Config, rt http.RoundTripper) (*Backend, error) { debug.Log("open, config %#v", cfg) - service, err := getStorageService(rt) + gcsClient, err := getStorageClient(rt) if err != nil { - return nil, errors.Wrap(err, "getStorageService") + return nil, errors.Wrap(err, "getStorageClient") } sem, err := backend.NewSemaphore(cfg.Connections) @@ -91,10 +100,11 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { } be := &Backend{ - service: service, + gcsClient: gcsClient, projectID: cfg.ProjectID, sem: sem, bucketName: cfg.Bucket, + bucket: gcsClient.Bucket(cfg.Bucket), prefix: cfg.Prefix, Layout: &backend.DefaultLayout{ Path: cfg.Prefix, @@ -123,47 +133,19 @@ func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) { } // Try to determine if the bucket exists. If it does not, try to create it. - // - // A Get call has three typical error cases: - // - // * nil: Bucket exists and we have access to the metadata (returned). - // - // * 403: Bucket exists and we do not have access to the metadata. We - // don't have storage.buckets.get permission to the bucket, but we may - // still be able to access objects in the bucket. - // - // * 404: Bucket doesn't exist. - // - // Determining if the bucket is accessible is best-effort because the - // 403 case is ambiguous. - if _, err := be.service.Buckets.Get(be.bucketName).Do(); err != nil { - gerr, ok := err.(*googleapi.Error) - if !ok { - // Don't know what to do with this error. - return nil, errors.Wrap(err, "service.Buckets.Get") + ctx := context.Background() + exists, err := be.bucketExists(ctx, be.bucket) + if err != nil { + return nil, errors.Wrap(err, "service.Buckets.Get") + } + + if !exists { + // Bucket doesn't exist, try to create it. + if err := be.bucket.Create(ctx, be.projectID, nil); err != nil { + // Always an error, as the bucket definitely doesn't exist. + return nil, errors.Wrap(err, "service.Buckets.Insert") } - switch gerr.Code { - case 403: - // Bucket exists, but we don't know if it is - // accessible. Optimistically assume it is; if not, - // future Backend calls will fail. - debug.Log("Unable to determine if bucket %s is accessible (err %v). Continuing as if it is.", be.bucketName, err) - case 404: - // Bucket doesn't exist, try to create it. - bucket := &storage.Bucket{ - Name: be.bucketName, - } - - if _, err := be.service.Buckets.Insert(be.projectID, bucket).Do(); err != nil { - // Always an error, as the bucket definitely - // doesn't exist. - return nil, errors.Wrap(err, "service.Buckets.Insert") - } - default: - // Don't know what to do with this error. - return nil, errors.Wrap(err, "service.Buckets.Get") - } } return be, nil @@ -245,13 +227,10 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe // // restic typically writes small blobs (4MB-30MB), so the resumable // uploads are not providing significant benefit anyways. - cs := googleapi.ChunkSize(0) - - info, err := be.service.Objects.Insert(be.bucketName, - &storage.Object{ - Name: objName, - Size: uint64(rd.Length()), - }).Media(rd, cs).Do() + w := be.bucket.Object(objName).NewWriter(ctx) + w.ChunkSize = 0 + wbytes, err := io.Copy(w, rd) + w.Close() be.sem.ReleaseToken() @@ -260,7 +239,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe return errors.Wrap(err, "service.Objects.Insert") } - debug.Log("%v -> %v bytes", objName, info.Size) + debug.Log("%v -> %v bytes", objName, wbytes) return nil } @@ -295,29 +274,23 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, if length < 0 { return nil, errors.Errorf("invalid length %d", length) } + if length == 0 { + // negative length indicates read till end to GCS lib + length = -1 + } objName := be.Filename(h) be.sem.GetToken() - var byteRange string - if length > 0 { - byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length-1)) - } else { - byteRange = fmt.Sprintf("bytes=%d-", offset) - } - - req := be.service.Objects.Get(be.bucketName, objName) - // https://cloud.google.com/storage/docs/json_api/v1/parameters#range - req.Header().Set("Range", byteRange) - res, err := req.Download() + r, err := be.bucket.Object(objName).NewRangeReader(ctx, offset, int64(length)) if err != nil { be.sem.ReleaseToken() return nil, err } closeRd := wrapReader{ - ReadCloser: res.Body, + ReadCloser: r, f: func() { debug.Log("Close()") be.sem.ReleaseToken() @@ -334,15 +307,15 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf objName := be.Filename(h) be.sem.GetToken() - obj, err := be.service.Objects.Get(be.bucketName, objName).Do() + attr, err := be.bucket.Object(objName).Attrs(ctx) be.sem.ReleaseToken() if err != nil { - debug.Log("GetObject() err %v", err) + debug.Log("GetObjectAttributes() err %v", err) return restic.FileInfo{}, errors.Wrap(err, "service.Objects.Get") } - return restic.FileInfo{Size: int64(obj.Size), Name: h.Name}, nil + return restic.FileInfo{Size: attr.Size, Name: h.Name}, nil } // Test returns true if a blob of the given type and name exists in the backend. @@ -351,7 +324,7 @@ func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) { objName := be.Filename(h) be.sem.GetToken() - _, err := be.service.Objects.Get(be.bucketName, objName).Do() + _, err := be.bucket.Object(objName).Attrs(ctx) be.sem.ReleaseToken() if err == nil { @@ -366,13 +339,11 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { objName := be.Filename(h) be.sem.GetToken() - err := be.service.Objects.Delete(be.bucketName, objName).Do() + err := be.bucket.Object(objName).Delete(ctx) be.sem.ReleaseToken() - if er, ok := err.(*googleapi.Error); ok { - if er.Code == 404 { - err = nil - } + if err == storage.ErrObjectNotExist { + err = nil } debug.Log("Remove(%v) at %v -> err %v", h, objName, err) @@ -394,47 +365,36 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F ctx, cancel := context.WithCancel(ctx) defer cancel() - listReq := be.service.Objects.List(be.bucketName).Context(ctx).Prefix(prefix).MaxResults(int64(be.listMaxItems)) + itr := be.bucket.Objects(ctx, &storage.Query{Prefix: prefix}) + for { be.sem.GetToken() - obj, err := listReq.Do() + attrs, err := itr.Next() be.sem.ReleaseToken() + if err == iterator.Done { + break + } + if err != nil { + return err + } + m := strings.TrimPrefix(attrs.Name, prefix) + if m == "" { + continue + } + fi := restic.FileInfo{ + Name: path.Base(m), + Size: int64(attrs.Size), + } + + err = fn(fi) if err != nil { return err } - debug.Log("returned %v items", len(obj.Items)) - - for _, item := range obj.Items { - m := strings.TrimPrefix(item.Name, prefix) - if m == "" { - continue - } - - if ctx.Err() != nil { - return ctx.Err() - } - - fi := restic.FileInfo{ - Name: path.Base(m), - Size: int64(item.Size), - } - - err := fn(fi) - if err != nil { - return err - } - - if ctx.Err() != nil { - return ctx.Err() - } + if ctx.Err() != nil { + return ctx.Err() } - - if obj.NextPageToken == "" { - break - } - listReq.PageToken(obj.NextPageToken) } return ctx.Err()