From 7e6bfdae7909da7a1f9da76e1be063001c8b34c3 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 23 Jan 2018 23:12:52 +0100 Subject: [PATCH] backend/rest: Implement REST API v2 --- internal/backend/rest/rest.go | 86 +++++++++++++- internal/backend/rest/rest_int_test.go | 150 +++++++++++++++++++++++++ 2 files changed, 231 insertions(+), 5 deletions(-) create mode 100644 internal/backend/rest/rest_int_test.go diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index ed879dc88..1f60b1f2f 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -30,6 +30,11 @@ type restBackend struct { backend.Layout } +const ( + contentTypeV1 = "application/vnd.x.restic.rest.v1" + contentTypeV2 = "application/vnd.x.restic.rest.v2" +) + // Open opens the REST backend with the given config. func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) { client := &http.Client{Transport: rt} @@ -111,8 +116,15 @@ func (b *restBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) ( // make sure that client.Post() cannot close the reader by wrapping it rd = ioutil.NopCloser(rd) + req, err := http.NewRequest(http.MethodPost, b.Filename(h), rd) + if err != nil { + return errors.Wrap(err, "NewRequest") + } + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Accept", contentTypeV2) + b.sem.GetToken() - resp, err := ctxhttp.Post(ctx, b.client, b.Filename(h), "binary/octet-stream", rd) + resp, err := ctxhttp.Do(ctx, b.client, req) b.sem.ReleaseToken() if resp != nil { @@ -180,7 +192,8 @@ func (b *restBackend) Load(ctx context.Context, h restic.Handle, length int, off if length > 0 { byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1) } - req.Header.Add("Range", byteRange) + req.Header.Set("Range", byteRange) + req.Header.Set("Accept", contentTypeV2) debug.Log("Load(%v) send range %v", h, byteRange) b.sem.GetToken() @@ -214,8 +227,14 @@ func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInf return restic.FileInfo{}, err } + req, err := http.NewRequest(http.MethodHead, b.Filename(h), nil) + if err != nil { + return restic.FileInfo{}, errors.Wrap(err, "NewRequest") + } + req.Header.Set("Accept", contentTypeV2) + b.sem.GetToken() - resp, err := ctxhttp.Head(ctx, b.client, b.Filename(h)) + resp, err := ctxhttp.Do(ctx, b.client, req) b.sem.ReleaseToken() if err != nil { return restic.FileInfo{}, errors.Wrap(err, "client.Head") @@ -267,6 +286,8 @@ func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error { if err != nil { return errors.Wrap(err, "http.NewRequest") } + req.Header.Set("Accept", contentTypeV2) + b.sem.GetToken() resp, err := ctxhttp.Do(ctx, b.client, req) b.sem.ReleaseToken() @@ -300,17 +321,35 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti url += "/" } + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return errors.Wrap(err, "NewRequest") + } + req.Header.Set("Accept", contentTypeV2) + b.sem.GetToken() - resp, err := ctxhttp.Get(ctx, b.client, url) + resp, err := ctxhttp.Do(ctx, b.client, req) b.sem.ReleaseToken() if err != nil { return errors.Wrap(err, "Get") } + if resp.Header.Get("Content-Type") == contentTypeV2 { + return b.listv2(ctx, t, resp, fn) + } + + return b.listv1(ctx, t, resp, fn) +} + +// listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET +// /data/`) only returns the names of the files, so we need to issue an HTTP +// HEAD request for each file. +func (b *restBackend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error { + debug.Log("parsing API v1 response") dec := json.NewDecoder(resp.Body) var list []string - if err = dec.Decode(&list); err != nil { + if err := dec.Decode(&list); err != nil { return errors.Wrap(err, "Decode") } @@ -338,6 +377,43 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti return ctx.Err() } +// listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET +// /data/`) returns the names and sizes of all files. +func (b *restBackend) listv2(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error { + debug.Log("parsing API v2 response") + dec := json.NewDecoder(resp.Body) + + var list []struct { + Name string `json:"name"` + Size int64 `json:"size"` + } + if err := dec.Decode(&list); err != nil { + return errors.Wrap(err, "Decode") + } + + for _, item := range list { + if ctx.Err() != nil { + return ctx.Err() + } + + fi := restic.FileInfo{ + Name: item.Name, + Size: item.Size, + } + + err := fn(fi) + if err != nil { + return err + } + + if ctx.Err() != nil { + return ctx.Err() + } + } + + return ctx.Err() +} + // Close closes all open files. func (b *restBackend) Close() error { // this does not need to do anything, all open files are closed within the diff --git a/internal/backend/rest/rest_int_test.go b/internal/backend/rest/rest_int_test.go new file mode 100644 index 000000000..ea4e265fd --- /dev/null +++ b/internal/backend/rest/rest_int_test.go @@ -0,0 +1,150 @@ +package rest_test + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strconv" + "testing" + + "github.com/restic/restic/internal/backend/rest" + "github.com/restic/restic/internal/restic" +) + +func TestListAPI(t *testing.T) { + var tests = []struct { + Name string + + ContentType string // response header + Data string // response data + Requests int + + Result []restic.FileInfo + }{ + { + Name: "content-type-unknown", + ContentType: "application/octet-stream", + Data: `[ + "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", + "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", + "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b" + ]`, + Result: []restic.FileInfo{ + {Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386}, + {Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214}, + {Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393}, + }, + Requests: 4, + }, + { + Name: "content-type-v1", + ContentType: "application/vnd.x.restic.rest.v1", + Data: `[ + "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", + "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", + "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b" + ]`, + Result: []restic.FileInfo{ + {Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386}, + {Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214}, + {Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393}, + }, + Requests: 4, + }, + { + Name: "content-type-v2", + ContentType: "application/vnd.x.restic.rest.v2", + Data: `[ + {"name": "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", "size": 1001}, + {"name": "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "size": 1002}, + {"name": "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", "size": 1003} + ]`, + Result: []restic.FileInfo{ + {Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 1001}, + {Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 1002}, + {Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 1003}, + }, + Requests: 1, + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + numRequests := 0 + srv := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + numRequests++ + t.Logf("req %v %v, accept: %v", req.Method, req.URL.Path, req.Header["Accept"]) + + var err error + switch { + case req.Method == "GET": + // list files in data/ + res.Header().Set("Content-Type", test.ContentType) + _, err = res.Write([]byte(test.Data)) + + if err != nil { + t.Fatal(err) + } + return + case req.Method == "HEAD": + // stat file in data/, use the first two bytes in the name + // of the file as the size :) + filename := req.URL.Path[6:] + len, err := strconv.ParseInt(filename[:4], 16, 64) + if err != nil { + t.Fatal(err) + } + + res.Header().Set("Content-Length", fmt.Sprintf("%d", len)) + res.WriteHeader(http.StatusOK) + return + } + + t.Errorf("unhandled request %v %v", req.Method, req.URL.Path) + })) + defer srv.Close() + + srvURL, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + + cfg := rest.Config{ + Connections: 5, + URL: srvURL, + } + + be, err := rest.Open(cfg, http.DefaultTransport) + if err != nil { + t.Fatal(err) + } + + var list []restic.FileInfo + err = be.List(context.TODO(), restic.DataFile, func(fi restic.FileInfo) error { + list = append(list, fi) + return nil + }) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(list, test.Result) { + t.Fatalf("wrong response returned, want:\n %v\ngot: %v", test.Result, list) + } + + if numRequests != test.Requests { + t.Fatalf("wrong number of HTTP requests executed, want %d, got %d", test.Requests, numRequests) + } + + defer func() { + err = be.Close() + if err != nil { + t.Fatal(err) + } + }() + }) + } +}