mirror of
https://github.com/octoleo/restic.git
synced 2024-11-29 08:14:03 +00:00
backend/rest: Implement REST API v2
This commit is contained in:
Alexander Neumann
parent
0f4cbea27d
commit
7e6bfdae79
@ -30,6 +30,11 @@ type restBackend struct {
|
||||
backend.Layout
|
||||
}
|
||||
|
||||
const (
|
||||
contentTypeV1 = "application/vnd.x.restic.rest.v1"
|
||||
contentTypeV2 = "application/vnd.x.restic.rest.v2"
|
||||
)
|
||||
|
||||
// Open opens the REST backend with the given config.
|
||||
func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) {
|
||||
client := &http.Client{Transport: rt}
|
||||
@ -111,8 +116,15 @@ func (b *restBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (
|
||||
// make sure that client.Post() cannot close the reader by wrapping it
|
||||
rd = ioutil.NopCloser(rd)
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, b.Filename(h), rd)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "NewRequest")
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
req.Header.Set("Accept", contentTypeV2)
|
||||
|
||||
b.sem.GetToken()
|
||||
resp, err := ctxhttp.Post(ctx, b.client, b.Filename(h), "binary/octet-stream", rd)
|
||||
resp, err := ctxhttp.Do(ctx, b.client, req)
|
||||
b.sem.ReleaseToken()
|
||||
|
||||
if resp != nil {
|
||||
@ -180,7 +192,8 @@ func (b *restBackend) Load(ctx context.Context, h restic.Handle, length int, off
|
||||
if length > 0 {
|
||||
byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
|
||||
}
|
||||
req.Header.Add("Range", byteRange)
|
||||
req.Header.Set("Range", byteRange)
|
||||
req.Header.Set("Accept", contentTypeV2)
|
||||
debug.Log("Load(%v) send range %v", h, byteRange)
|
||||
|
||||
b.sem.GetToken()
|
||||
@ -214,8 +227,14 @@ func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInf
|
||||
return restic.FileInfo{}, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodHead, b.Filename(h), nil)
|
||||
if err != nil {
|
||||
return restic.FileInfo{}, errors.Wrap(err, "NewRequest")
|
||||
}
|
||||
req.Header.Set("Accept", contentTypeV2)
|
||||
|
||||
b.sem.GetToken()
|
||||
resp, err := ctxhttp.Head(ctx, b.client, b.Filename(h))
|
||||
resp, err := ctxhttp.Do(ctx, b.client, req)
|
||||
b.sem.ReleaseToken()
|
||||
if err != nil {
|
||||
return restic.FileInfo{}, errors.Wrap(err, "client.Head")
|
||||
@ -267,6 +286,8 @@ func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "http.NewRequest")
|
||||
}
|
||||
req.Header.Set("Accept", contentTypeV2)
|
||||
|
||||
b.sem.GetToken()
|
||||
resp, err := ctxhttp.Do(ctx, b.client, req)
|
||||
b.sem.ReleaseToken()
|
||||
@ -300,17 +321,35 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
|
||||
url += "/"
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "NewRequest")
|
||||
}
|
||||
req.Header.Set("Accept", contentTypeV2)
|
||||
|
||||
b.sem.GetToken()
|
||||
resp, err := ctxhttp.Get(ctx, b.client, url)
|
||||
resp, err := ctxhttp.Do(ctx, b.client, req)
|
||||
b.sem.ReleaseToken()
|
||||
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Get")
|
||||
}
|
||||
|
||||
if resp.Header.Get("Content-Type") == contentTypeV2 {
|
||||
return b.listv2(ctx, t, resp, fn)
|
||||
}
|
||||
|
||||
return b.listv1(ctx, t, resp, fn)
|
||||
}
|
||||
|
||||
// listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET
|
||||
// /data/`) only returns the names of the files, so we need to issue an HTTP
|
||||
// HEAD request for each file.
|
||||
func (b *restBackend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
|
||||
debug.Log("parsing API v1 response")
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
var list []string
|
||||
if err = dec.Decode(&list); err != nil {
|
||||
if err := dec.Decode(&list); err != nil {
|
||||
return errors.Wrap(err, "Decode")
|
||||
}
|
||||
|
||||
@ -338,6 +377,43 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET
|
||||
// /data/`) returns the names and sizes of all files.
|
||||
func (b *restBackend) listv2(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
|
||||
debug.Log("parsing API v2 response")
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
|
||||
var list []struct {
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
if err := dec.Decode(&list); err != nil {
|
||||
return errors.Wrap(err, "Decode")
|
||||
}
|
||||
|
||||
for _, item := range list {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
fi := restic.FileInfo{
|
||||
Name: item.Name,
|
||||
Size: item.Size,
|
||||
}
|
||||
|
||||
err := fn(fi)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// Close closes all open files.
|
||||
func (b *restBackend) Close() error {
|
||||
// this does not need to do anything, all open files are closed within the
|
||||
|
||||
150
internal/backend/rest/rest_int_test.go
Normal file
150
internal/backend/rest/rest_int_test.go
Normal file
@ -0,0 +1,150 @@
|
||||
package rest_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/backend/rest"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
func TestListAPI(t *testing.T) {
|
||||
var tests = []struct {
|
||||
Name string
|
||||
|
||||
ContentType string // response header
|
||||
Data string // response data
|
||||
Requests int
|
||||
|
||||
Result []restic.FileInfo
|
||||
}{
|
||||
{
|
||||
Name: "content-type-unknown",
|
||||
ContentType: "application/octet-stream",
|
||||
Data: `[
|
||||
"1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985",
|
||||
"3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352",
|
||||
"8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b"
|
||||
]`,
|
||||
Result: []restic.FileInfo{
|
||||
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386},
|
||||
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214},
|
||||
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393},
|
||||
},
|
||||
Requests: 4,
|
||||
},
|
||||
{
|
||||
Name: "content-type-v1",
|
||||
ContentType: "application/vnd.x.restic.rest.v1",
|
||||
Data: `[
|
||||
"1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985",
|
||||
"3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352",
|
||||
"8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b"
|
||||
]`,
|
||||
Result: []restic.FileInfo{
|
||||
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386},
|
||||
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214},
|
||||
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393},
|
||||
},
|
||||
Requests: 4,
|
||||
},
|
||||
{
|
||||
Name: "content-type-v2",
|
||||
ContentType: "application/vnd.x.restic.rest.v2",
|
||||
Data: `[
|
||||
{"name": "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", "size": 1001},
|
||||
{"name": "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "size": 1002},
|
||||
{"name": "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", "size": 1003}
|
||||
]`,
|
||||
Result: []restic.FileInfo{
|
||||
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 1001},
|
||||
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 1002},
|
||||
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 1003},
|
||||
},
|
||||
Requests: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
numRequests := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
|
||||
numRequests++
|
||||
t.Logf("req %v %v, accept: %v", req.Method, req.URL.Path, req.Header["Accept"])
|
||||
|
||||
var err error
|
||||
switch {
|
||||
case req.Method == "GET":
|
||||
// list files in data/
|
||||
res.Header().Set("Content-Type", test.ContentType)
|
||||
_, err = res.Write([]byte(test.Data))
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return
|
||||
case req.Method == "HEAD":
|
||||
// stat file in data/, use the first two bytes in the name
|
||||
// of the file as the size :)
|
||||
filename := req.URL.Path[6:]
|
||||
len, err := strconv.ParseInt(filename[:4], 16, 64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
res.Header().Set("Content-Length", fmt.Sprintf("%d", len))
|
||||
res.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
|
||||
t.Errorf("unhandled request %v %v", req.Method, req.URL.Path)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
srvURL, err := url.Parse(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cfg := rest.Config{
|
||||
Connections: 5,
|
||||
URL: srvURL,
|
||||
}
|
||||
|
||||
be, err := rest.Open(cfg, http.DefaultTransport)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var list []restic.FileInfo
|
||||
err = be.List(context.TODO(), restic.DataFile, func(fi restic.FileInfo) error {
|
||||
list = append(list, fi)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(list, test.Result) {
|
||||
t.Fatalf("wrong response returned, want:\n %v\ngot: %v", test.Result, list)
|
||||
}
|
||||
|
||||
if numRequests != test.Requests {
|
||||
t.Fatalf("wrong number of HTTP requests executed, want %d, got %d", test.Requests, numRequests)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err = be.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user