backend/rest: Implement REST API v2

internal/backend/rest/rest.go

@@ -30,6 +30,11 @@ type restBackend struct {
 	backend.Layout
 }
 
+const (
+	contentTypeV1 = "application/vnd.x.restic.rest.v1"
+	contentTypeV2 = "application/vnd.x.restic.rest.v2"
+)
+
 // Open opens the REST backend with the given config.
 func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) {
 	client := &http.Client{Transport: rt}
@@ -111,8 +116,15 @@ func (b *restBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (
 	// make sure that client.Post() cannot close the reader by wrapping it
 	rd = ioutil.NopCloser(rd)
 
+	req, err := http.NewRequest(http.MethodPost, b.Filename(h), rd)
+	if err != nil {
+		return errors.Wrap(err, "NewRequest")
+	}
+	req.Header.Set("Content-Type", "application/octet-stream")
+	req.Header.Set("Accept", contentTypeV2)
+
 	b.sem.GetToken()
-	resp, err := ctxhttp.Post(ctx, b.client, b.Filename(h), "binary/octet-stream", rd)
+	resp, err := ctxhttp.Do(ctx, b.client, req)
 	b.sem.ReleaseToken()
 
 	if resp != nil {
@@ -180,7 +192,8 @@ func (b *restBackend) Load(ctx context.Context, h restic.Handle, length int, off
 	if length > 0 {
 		byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
 	}
-	req.Header.Add("Range", byteRange)
+	req.Header.Set("Range", byteRange)
+	req.Header.Set("Accept", contentTypeV2)
 	debug.Log("Load(%v) send range %v", h, byteRange)
 
 	b.sem.GetToken()
@@ -214,8 +227,14 @@ func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInf
 		return restic.FileInfo{}, err
 	}
 
+	req, err := http.NewRequest(http.MethodHead, b.Filename(h), nil)
+	if err != nil {
+		return restic.FileInfo{}, errors.Wrap(err, "NewRequest")
+	}
+	req.Header.Set("Accept", contentTypeV2)
+
 	b.sem.GetToken()
-	resp, err := ctxhttp.Head(ctx, b.client, b.Filename(h))
+	resp, err := ctxhttp.Do(ctx, b.client, req)
 	b.sem.ReleaseToken()
 	if err != nil {
 		return restic.FileInfo{}, errors.Wrap(err, "client.Head")
@@ -267,6 +286,8 @@ func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
 	if err != nil {
 		return errors.Wrap(err, "http.NewRequest")
 	}
+	req.Header.Set("Accept", contentTypeV2)
+
 	b.sem.GetToken()
 	resp, err := ctxhttp.Do(ctx, b.client, req)
 	b.sem.ReleaseToken()
@@ -300,17 +321,35 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
 		url += "/"
 	}
 
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return errors.Wrap(err, "NewRequest")
+	}
+	req.Header.Set("Accept", contentTypeV2)
+
 	b.sem.GetToken()
-	resp, err := ctxhttp.Get(ctx, b.client, url)
+	resp, err := ctxhttp.Do(ctx, b.client, req)
 	b.sem.ReleaseToken()
 
 	if err != nil {
 		return errors.Wrap(err, "Get")
 	}
 
+	if resp.Header.Get("Content-Type") == contentTypeV2 {
+		return b.listv2(ctx, t, resp, fn)
+	}
+
+	return b.listv1(ctx, t, resp, fn)
+}
+
+// listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET
+// /data/`) only returns the names of the files, so we need to issue an HTTP
+// HEAD request for each file.
+func (b *restBackend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
+	debug.Log("parsing API v1 response")
 	dec := json.NewDecoder(resp.Body)
 	var list []string
-	if err = dec.Decode(&list); err != nil {
+	if err := dec.Decode(&list); err != nil {
 		return errors.Wrap(err, "Decode")
 	}
 
@@ -338,6 +377,43 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
 	return ctx.Err()
 }
 
+// listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET
+// /data/`) returns the names and sizes of all files.
+func (b *restBackend) listv2(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
+	debug.Log("parsing API v2 response")
+	dec := json.NewDecoder(resp.Body)
+
+	var list []struct {
+		Name string `json:"name"`
+		Size int64  `json:"size"`
+	}
+	if err := dec.Decode(&list); err != nil {
+		return errors.Wrap(err, "Decode")
+	}
+
+	for _, item := range list {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+
+		fi := restic.FileInfo{
+			Name: item.Name,
+			Size: item.Size,
+		}
+
+		err := fn(fi)
+		if err != nil {
+			return err
+		}
+
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+	}
+
+	return ctx.Err()
+}
+
 // Close closes all open files.
 func (b *restBackend) Close() error {
 	// this does not need to do anything, all open files are closed within the

internal/backend/rest/rest_int_test.go

@@ -0,0 +1,150 @@
+package rest_test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"reflect"
+	"strconv"
+	"testing"
+
+	"github.com/restic/restic/internal/backend/rest"
+	"github.com/restic/restic/internal/restic"
+)
+
+func TestListAPI(t *testing.T) {
+	var tests = []struct {
+		Name string
+
+		ContentType string // response header
+		Data        string // response data
+		Requests    int
+
+		Result []restic.FileInfo
+	}{
+		{
+			Name:        "content-type-unknown",
+			ContentType: "application/octet-stream",
+			Data: `[
+				"1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985",
+				"3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352",
+				"8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b"
+			]`,
+			Result: []restic.FileInfo{
+				{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386},
+				{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214},
+				{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393},
+			},
+			Requests: 4,
+		},
+		{
+			Name:        "content-type-v1",
+			ContentType: "application/vnd.x.restic.rest.v1",
+			Data: `[
+				"1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985",
+				"3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352",
+				"8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b"
+			]`,
+			Result: []restic.FileInfo{
+				{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386},
+				{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214},
+				{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393},
+			},
+			Requests: 4,
+		},
+		{
+			Name:        "content-type-v2",
+			ContentType: "application/vnd.x.restic.rest.v2",
+			Data: `[
+				{"name": "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", "size": 1001},
+				{"name": "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "size": 1002},
+				{"name": "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", "size": 1003}
+			]`,
+			Result: []restic.FileInfo{
+				{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 1001},
+				{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 1002},
+				{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 1003},
+			},
+			Requests: 1,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			numRequests := 0
+			srv := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
+				numRequests++
+				t.Logf("req %v %v, accept: %v", req.Method, req.URL.Path, req.Header["Accept"])
+
+				var err error
+				switch {
+				case req.Method == "GET":
+					// list files in data/
+					res.Header().Set("Content-Type", test.ContentType)
+					_, err = res.Write([]byte(test.Data))
+
+					if err != nil {
+						t.Fatal(err)
+					}
+					return
+				case req.Method == "HEAD":
+					// stat file in data/, use the first two bytes in the name
+					// of the file as the size :)
+					filename := req.URL.Path[6:]
+					len, err := strconv.ParseInt(filename[:4], 16, 64)
+					if err != nil {
+						t.Fatal(err)
+					}
+
+					res.Header().Set("Content-Length", fmt.Sprintf("%d", len))
+					res.WriteHeader(http.StatusOK)
+					return
+				}
+
+				t.Errorf("unhandled request %v %v", req.Method, req.URL.Path)
+			}))
+			defer srv.Close()
+
+			srvURL, err := url.Parse(srv.URL)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			cfg := rest.Config{
+				Connections: 5,
+				URL:         srvURL,
+			}
+
+			be, err := rest.Open(cfg, http.DefaultTransport)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			var list []restic.FileInfo
+			err = be.List(context.TODO(), restic.DataFile, func(fi restic.FileInfo) error {
+				list = append(list, fi)
+				return nil
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if !reflect.DeepEqual(list, test.Result) {
+				t.Fatalf("wrong response returned, want:\n  %v\ngot:  %v", test.Result, list)
+			}
+
+			if numRequests != test.Requests {
+				t.Fatalf("wrong number of HTTP requests executed, want %d, got %d", test.Requests, numRequests)
+			}
+
+			defer func() {
+				err = be.Close()
+				if err != nil {
+					t.Fatal(err)
+				}
+			}()
+		})
+	}
+}