Add REST backend

This is a port of the original work by @bchapuis in
https://github.com/restic/restic/pull/253

doc/REST_backend.md

@@ -0,0 +1,56 @@
+REST Backend
+============
+
+Restic can interact with HTTP Backend that respects the following REST API.
+
+## HEAD /config
+
+Returns "200 OK" if the repository has a configuration,
+an HTTP error otherwise.
+
+## GET /config
+
+Returns the content of the configuration file if the repository has a configuration,
+an HTTP error otherwise.
+
+Response format: binary/octet-stream
+
+## POST /config
+
+Returns "200 OK" if the configuration of the request body has been saved,
+an HTTP error otherwise.
+
+## GET /{type}/
+
+Returns a JSON array containing the names of all the blobs stored for a given type.
+
+Response format: JSON
+
+## HEAD /{type}/{name}
+
+Returns "200 OK" if the blob with the given name and type is stored in the repository,
+"404 not found" otherwise. If the blob exists, the HTTP header `Content-Length`
+is set to the file size.
+
+## GET /{type}/{name}
+
+Returns the content of the blob with the given name and type if it is stored in the repository,
+"404 not found" otherwise.
+
+If the request specifies a partial read with a Range header field,
+then the status code of the response is 206 instead of 200
+and the response only contains the specified range.
+
+Response format: binary/octet-stream
+
+## POST /{type}/{name}
+
+Saves the content of the request body as a blob with the given name and type,
+an HTTP error otherwise.
+
+Request format: binary/octet-stream
+
+## DELETE /{type}/{name}
+
+Returns "200 OK" if the blob with the given name and type has been deleted from the repository,
+an HTTP error otherwise.

src/restic/backend/rest/backend_test.go

@@ -0,0 +1,87 @@
+// DO NOT EDIT, AUTOMATICALLY GENERATED
+package rest_test
+
+import (
+	"testing"
+
+	"restic/backend/test"
+)
+
+var SkipMessage string
+
+func TestRestBackendCreate(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestCreate(t)
+}
+
+func TestRestBackendOpen(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestOpen(t)
+}
+
+func TestRestBackendCreateWithConfig(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestCreateWithConfig(t)
+}
+
+func TestRestBackendLocation(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestLocation(t)
+}
+
+func TestRestBackendConfig(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestConfig(t)
+}
+
+func TestRestBackendLoad(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestLoad(t)
+}
+
+func TestRestBackendSave(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSave(t)
+}
+
+func TestRestBackendSaveFilenames(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSaveFilenames(t)
+}
+
+func TestRestBackendBackend(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestBackend(t)
+}
+
+func TestRestBackendDelete(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestDelete(t)
+}
+
+func TestRestBackendCleanup(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestCleanup(t)
+}

src/restic/backend/rest/config.go

@@ -0,0 +1,29 @@
+package rest
+
+import (
+	"errors"
+	"net/url"
+	"strings"
+)
+
+// Config contains all configuration necessary to connect to a REST server.
+type Config struct {
+	URL *url.URL
+}
+
+// ParseConfig parses the string s and extracts the REST server URL.
+func ParseConfig(s string) (interface{}, error) {
+	if !strings.HasPrefix(s, "rest:") {
+		return nil, errors.New("invalid REST backend specification")
+	}
+
+	s = s[5:]
+	u, err := url.Parse(s)
+
+	if err != nil {
+		return nil, err
+	}
+
+	cfg := Config{URL: u}
+	return cfg, nil
+}

src/restic/backend/rest/config_test.go

@@ -0,0 +1,41 @@
+package rest
+
+import (
+	"net/url"
+	"reflect"
+	"testing"
+)
+
+func parseURL(s string) *url.URL {
+	u, err := url.Parse(s)
+	if err != nil {
+		panic(err)
+	}
+
+	return u
+}
+
+var configTests = []struct {
+	s   string
+	cfg Config
+}{
+	{"rest:http://localhost:1234", Config{
+		URL: parseURL("http://localhost:1234"),
+	}},
+}
+
+func TestParseConfig(t *testing.T) {
+	for i, test := range configTests {
+		cfg, err := ParseConfig(test.s)
+		if err != nil {
+			t.Errorf("test %d:%s failed: %v", i, test.s, err)
+			continue
+		}
+
+		if !reflect.DeepEqual(cfg, test.cfg) {
+			t.Errorf("test %d:\ninput:\n  %s\n wrong config, want:\n  %v\ngot:\n  %v",
+				i, test.s, test.cfg, cfg)
+			continue
+		}
+	}
+}

src/restic/backend/rest/rest.go

@@ -0,0 +1,258 @@
+package rest
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+
+	"restic/backend"
+)
+
+const connLimit = 10
+
+// restPath returns the path to the given resource.
+func restPath(url *url.URL, h backend.Handle) string {
+	p := url.Path
+	if p == "" {
+		p = "/"
+	}
+
+	var dir string
+
+	switch h.Type {
+	case backend.Config:
+		dir = ""
+	case backend.Data:
+		dir = backend.Paths.Data
+	case backend.Snapshot:
+		dir = backend.Paths.Snapshots
+	case backend.Index:
+		dir = backend.Paths.Index
+	case backend.Lock:
+		dir = backend.Paths.Locks
+	case backend.Key:
+		dir = backend.Paths.Keys
+	default:
+		dir = string(h.Type)
+	}
+
+	return path.Join(p, dir, h.Name)
+}
+
+type restBackend struct {
+	url      *url.URL
+	connChan chan struct{}
+	client   *http.Client
+}
+
+// Open opens the REST backend with the given config.
+func Open(cfg Config) (backend.Backend, error) {
+	connChan := make(chan struct{}, connLimit)
+	for i := 0; i < connLimit; i++ {
+		connChan <- struct{}{}
+	}
+	tr := &http.Transport{}
+	client := http.Client{Transport: tr}
+
+	return &restBackend{url: cfg.URL, connChan: connChan, client: &client}, nil
+}
+
+// Location returns this backend's location (the server's URL).
+func (b *restBackend) Location() string {
+	return b.url.String()
+}
+
+// Load returns the data stored in the backend for h at the given offset
+// and saves it in p. Load has the same semantics as io.ReaderAt.
+func (b *restBackend) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
+	if err := h.Valid(); err != nil {
+		return 0, err
+	}
+
+	req, err := http.NewRequest("GET", restPath(b.url, h), nil)
+	if err != nil {
+		return 0, err
+	}
+	req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))))
+	client := *b.client
+
+	<-b.connChan
+	resp, err := client.Do(req)
+	b.connChan <- struct{}{}
+
+	if resp != nil {
+		defer func() {
+			e := resp.Body.Close()
+
+			if err == nil {
+				err = e
+			}
+		}()
+	}
+
+	if err != nil {
+		return 0, err
+	}
+	if resp.StatusCode != 206 {
+		return 0, errors.New("blob not found")
+	}
+
+	return io.ReadFull(resp.Body, p)
+}
+
+// Save stores data in the backend at the handle.
+func (b *restBackend) Save(h backend.Handle, p []byte) (err error) {
+	if err := h.Valid(); err != nil {
+		return err
+	}
+
+	client := *b.client
+
+	<-b.connChan
+	resp, err := client.Post(restPath(b.url, h), "binary/octet-stream", bytes.NewReader(p))
+	b.connChan <- struct{}{}
+
+	if resp != nil {
+		defer func() {
+			e := resp.Body.Close()
+
+			if err == nil {
+				err = e
+			}
+		}()
+	}
+
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != 200 {
+		return errors.New("blob not saved")
+	}
+
+	return nil
+}
+
+// Stat returns information about a blob.
+func (b *restBackend) Stat(h backend.Handle) (backend.BlobInfo, error) {
+	if err := h.Valid(); err != nil {
+		return backend.BlobInfo{}, err
+	}
+
+	client := *b.client
+	<-b.connChan
+	resp, err := client.Head(restPath(b.url, h))
+	b.connChan <- struct{}{}
+	if err != nil {
+		return backend.BlobInfo{}, err
+	}
+
+	if err = resp.Body.Close(); err != nil {
+		return backend.BlobInfo{}, err
+	}
+
+	if resp.StatusCode != 200 {
+		return backend.BlobInfo{}, errors.New("blob not saved")
+	}
+
+	if resp.ContentLength < 0 {
+		return backend.BlobInfo{}, errors.New("negative content length")
+	}
+
+	bi := backend.BlobInfo{
+		Size: resp.ContentLength,
+	}
+
+	return bi, nil
+}
+
+// Test returns true if a blob of the given type and name exists in the backend.
+func (b *restBackend) Test(t backend.Type, name string) (bool, error) {
+	_, err := b.Stat(backend.Handle{Type: t, Name: name})
+	if err != nil {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// Remove removes the blob with the given name and type.
+func (b *restBackend) Remove(t backend.Type, name string) error {
+	h := backend.Handle{Type: t, Name: name}
+	if err := h.Valid(); err != nil {
+		return err
+	}
+
+	req, err := http.NewRequest("DELETE", restPath(b.url, h), nil)
+	if err != nil {
+		return err
+	}
+	client := *b.client
+
+	<-b.connChan
+	resp, err := client.Do(req)
+	b.connChan <- struct{}{}
+
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != 200 {
+		return errors.New("blob not removed")
+	}
+
+	return resp.Body.Close()
+}
+
+// List returns a channel that yields all names of blobs of type t. A
+// goroutine is started for this. If the channel done is closed, sending
+// stops.
+func (b *restBackend) List(t backend.Type, done <-chan struct{}) <-chan string {
+	ch := make(chan string)
+
+	client := *b.client
+	<-b.connChan
+	resp, err := client.Get(restPath(b.url, backend.Handle{Type: t}))
+	b.connChan <- struct{}{}
+
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+
+	if err != nil {
+		close(ch)
+		return ch
+	}
+
+	dec := json.NewDecoder(resp.Body)
+	var list []string
+	if err = dec.Decode(&list); err != nil {
+		close(ch)
+		return ch
+	}
+
+	go func() {
+		defer close(ch)
+		for _, m := range list {
+			select {
+			case ch <- m:
+			case <-done:
+				return
+			}
+		}
+	}()
+
+	return ch
+}
+
+// Close closes all open files.
+func (b *restBackend) Close() error {
+	// this does not need to do anything, all open files are closed within the
+	// same function.
+	return nil
+}

src/restic/backend/rest/rest_test.go

@@ -0,0 +1,54 @@
+package rest_test
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+
+	"restic/backend"
+	"restic/backend/rest"
+	"restic/backend/test"
+	. "restic/test"
+)
+
+//go:generate go run ../test/generate_backend_tests.go
+
+func init() {
+	if TestRESTServer == "" {
+		SkipMessage = "REST test server not available"
+		return
+	}
+
+	url, err := url.Parse(TestRESTServer)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "invalid url: %v\n", err)
+		return
+	}
+
+	cfg := rest.Config{
+		URL: url,
+	}
+
+	test.CreateFn = func() (backend.Backend, error) {
+		be, err := rest.Open(cfg)
+		if err != nil {
+			return nil, err
+		}
+
+		exists, err := be.Test(backend.Config, "")
+		if err != nil {
+			return nil, err
+		}
+
+		if exists {
+			return nil, errors.New("config already exists")
+		}
+
+		return be, nil
+	}
+
+	test.OpenFn = func() (backend.Backend, error) {
+		return rest.Open(cfg)
+	}
+}

src/restic/test/backend.go

@@ -23,6 +23,7 @@ var (
 	TestWalkerPath        = getStringVar("RESTIC_TEST_PATH", ".")
 	BenchArchiveDirectory = getStringVar("RESTIC_BENCH_DIR", ".")
 	TestS3Server          = getStringVar("RESTIC_TEST_S3_SERVER", "")
+	TestRESTServer        = getStringVar("RESTIC_TEST_REST_SERVER", "")
 )
 
 func getStringVar(name, defaultValue string) string {