Merge pull request #464 from restic/rest-backend

Add REST backend

.gitignore

@@ -2,3 +2,4 @@
 /bin
 /restic
 /.vagrant
+/vendor/pkg

doc/REST_backend.md

@@ -0,0 +1,59 @@
+REST Backend
+============
+
+Restic can interact with HTTP Backend that respects the following REST API. The
+following values are valid for `{type}`: `data`, `keys`, `locks`, `snapshots`,
+`index`, `config`. `{path}` is a path to the repository, so that multiple
+different repositories can be accessed. The default path is `/`.
+
+## HEAD {path}/config
+
+Returns "200 OK" if the repository has a configuration,
+an HTTP error otherwise.
+
+## GET {path}/config
+
+Returns the content of the configuration file if the repository has a configuration,
+an HTTP error otherwise.
+
+Response format: binary/octet-stream
+
+## POST {path}/config
+
+Returns "200 OK" if the configuration of the request body has been saved,
+an HTTP error otherwise.
+
+## GET {path}/{type}/
+
+Returns a JSON array containing the names of all the blobs stored for a given type.
+
+Response format: JSON
+
+## HEAD {path}/{type}/{name}
+
+Returns "200 OK" if the blob with the given name and type is stored in the repository,
+"404 not found" otherwise. If the blob exists, the HTTP header `Content-Length`
+is set to the file size.
+
+## GET {path}/{type}/{name}
+
+Returns the content of the blob with the given name and type if it is stored in the repository,
+"404 not found" otherwise.
+
+If the request specifies a partial read with a Range header field,
+then the status code of the response is 206 instead of 200
+and the response only contains the specified range.
+
+Response format: binary/octet-stream
+
+## POST {path}/{type}/{name}
+
+Saves the content of the request body as a blob with the given name and type,
+an HTTP error otherwise.
+
+Request format: binary/octet-stream
+
+## DELETE {path}/{type}/{name}
+
+Returns "200 OK" if the blob with the given name and type has been deleted from the repository,
+an HTTP error otherwise.

src/restic/backend/rest/backend_test.go

@@ -0,0 +1,87 @@
+// DO NOT EDIT, AUTOMATICALLY GENERATED
+package rest_test
+
+import (
+	"testing"
+
+	"restic/backend/test"
+)
+
+var SkipMessage string
+
+func TestRestBackendCreate(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestCreate(t)
+}
+
+func TestRestBackendOpen(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestOpen(t)
+}
+
+func TestRestBackendCreateWithConfig(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestCreateWithConfig(t)
+}
+
+func TestRestBackendLocation(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestLocation(t)
+}
+
+func TestRestBackendConfig(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestConfig(t)
+}
+
+func TestRestBackendLoad(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestLoad(t)
+}
+
+func TestRestBackendSave(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSave(t)
+}
+
+func TestRestBackendSaveFilenames(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSaveFilenames(t)
+}
+
+func TestRestBackendBackend(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestBackend(t)
+}
+
+func TestRestBackendDelete(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestDelete(t)
+}
+
+func TestRestBackendCleanup(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestCleanup(t)
+}

src/restic/backend/rest/config.go

@@ -0,0 +1,29 @@
+package rest
+
+import (
+	"errors"
+	"net/url"
+	"strings"
+)
+
+// Config contains all configuration necessary to connect to a REST server.
+type Config struct {
+	URL *url.URL
+}
+
+// ParseConfig parses the string s and extracts the REST server URL.
+func ParseConfig(s string) (interface{}, error) {
+	if !strings.HasPrefix(s, "rest:") {
+		return nil, errors.New("invalid REST backend specification")
+	}
+
+	s = s[5:]
+	u, err := url.Parse(s)
+
+	if err != nil {
+		return nil, err
+	}
+
+	cfg := Config{URL: u}
+	return cfg, nil
+}

src/restic/backend/rest/config_test.go

@@ -0,0 +1,41 @@
+package rest
+
+import (
+	"net/url"
+	"reflect"
+	"testing"
+)
+
+func parseURL(s string) *url.URL {
+	u, err := url.Parse(s)
+	if err != nil {
+		panic(err)
+	}
+
+	return u
+}
+
+var configTests = []struct {
+	s   string
+	cfg Config
+}{
+	{"rest:http://localhost:1234", Config{
+		URL: parseURL("http://localhost:1234"),
+	}},
+}
+
+func TestParseConfig(t *testing.T) {
+	for i, test := range configTests {
+		cfg, err := ParseConfig(test.s)
+		if err != nil {
+			t.Errorf("test %d:%s failed: %v", i, test.s, err)
+			continue
+		}
+
+		if !reflect.DeepEqual(cfg, test.cfg) {
+			t.Errorf("test %d:\ninput:\n  %s\n wrong config, want:\n  %v\ngot:\n  %v",
+				i, test.s, test.cfg, cfg)
+			continue
+		}
+	}
+}

src/restic/backend/rest/rest.go

@@ -0,0 +1,264 @@
+package rest
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+
+	"restic/backend"
+)
+
+const connLimit = 10
+
+// restPath returns the path to the given resource.
+func restPath(url *url.URL, h backend.Handle) string {
+	u := *url
+
+	var dir string
+
+	switch h.Type {
+	case backend.Config:
+		dir = ""
+		h.Name = "config"
+	case backend.Data:
+		dir = backend.Paths.Data
+	case backend.Snapshot:
+		dir = backend.Paths.Snapshots
+	case backend.Index:
+		dir = backend.Paths.Index
+	case backend.Lock:
+		dir = backend.Paths.Locks
+	case backend.Key:
+		dir = backend.Paths.Keys
+	default:
+		dir = string(h.Type)
+	}
+
+	u.Path = path.Join(url.Path, dir, h.Name)
+
+	return u.String()
+}
+
+type restBackend struct {
+	url      *url.URL
+	connChan chan struct{}
+	client   *http.Client
+}
+
+// Open opens the REST backend with the given config.
+func Open(cfg Config) (backend.Backend, error) {
+	connChan := make(chan struct{}, connLimit)
+	for i := 0; i < connLimit; i++ {
+		connChan <- struct{}{}
+	}
+	tr := &http.Transport{}
+	client := http.Client{Transport: tr}
+
+	return &restBackend{url: cfg.URL, connChan: connChan, client: &client}, nil
+}
+
+// Location returns this backend's location (the server's URL).
+func (b *restBackend) Location() string {
+	return b.url.String()
+}
+
+// Load returns the data stored in the backend for h at the given offset
+// and saves it in p. Load has the same semantics as io.ReaderAt.
+func (b *restBackend) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
+	if err := h.Valid(); err != nil {
+		return 0, err
+	}
+
+	req, err := http.NewRequest("GET", restPath(b.url, h), nil)
+	if err != nil {
+		return 0, err
+	}
+	req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))))
+	client := *b.client
+
+	<-b.connChan
+	resp, err := client.Do(req)
+	b.connChan <- struct{}{}
+
+	if resp != nil {
+		defer func() {
+			e := resp.Body.Close()
+
+			if err == nil {
+				err = e
+			}
+		}()
+	}
+
+	if err != nil {
+		return 0, err
+	}
+	if resp.StatusCode != 200 && resp.StatusCode != 206 {
+		return 0, fmt.Errorf("unexpected HTTP response code %v", resp.StatusCode)
+	}
+
+	return io.ReadFull(resp.Body, p)
+}
+
+// Save stores data in the backend at the handle.
+func (b *restBackend) Save(h backend.Handle, p []byte) (err error) {
+	if err := h.Valid(); err != nil {
+		return err
+	}
+
+	client := *b.client
+
+	<-b.connChan
+	resp, err := client.Post(restPath(b.url, h), "binary/octet-stream", bytes.NewReader(p))
+	b.connChan <- struct{}{}
+
+	if resp != nil {
+		defer func() {
+			e := resp.Body.Close()
+
+			if err == nil {
+				err = e
+			}
+		}()
+	}
+
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("unexpected HTTP response code %v", resp.StatusCode)
+	}
+
+	return nil
+}
+
+// Stat returns information about a blob.
+func (b *restBackend) Stat(h backend.Handle) (backend.BlobInfo, error) {
+	if err := h.Valid(); err != nil {
+		return backend.BlobInfo{}, err
+	}
+
+	client := *b.client
+	<-b.connChan
+	resp, err := client.Head(restPath(b.url, h))
+	b.connChan <- struct{}{}
+	if err != nil {
+		return backend.BlobInfo{}, err
+	}
+
+	if err = resp.Body.Close(); err != nil {
+		return backend.BlobInfo{}, err
+	}
+
+	if resp.StatusCode != 200 {
+		return backend.BlobInfo{}, fmt.Errorf("unexpected HTTP response code %v", resp.StatusCode)
+	}
+
+	if resp.ContentLength < 0 {
+		return backend.BlobInfo{}, errors.New("negative content length")
+	}
+
+	bi := backend.BlobInfo{
+		Size: resp.ContentLength,
+	}
+
+	return bi, nil
+}
+
+// Test returns true if a blob of the given type and name exists in the backend.
+func (b *restBackend) Test(t backend.Type, name string) (bool, error) {
+	_, err := b.Stat(backend.Handle{Type: t, Name: name})
+	if err != nil {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// Remove removes the blob with the given name and type.
+func (b *restBackend) Remove(t backend.Type, name string) error {
+	h := backend.Handle{Type: t, Name: name}
+	if err := h.Valid(); err != nil {
+		return err
+	}
+
+	req, err := http.NewRequest("DELETE", restPath(b.url, h), nil)
+	if err != nil {
+		return err
+	}
+	client := *b.client
+
+	<-b.connChan
+	resp, err := client.Do(req)
+	b.connChan <- struct{}{}
+
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != 200 {
+		return errors.New("blob not removed")
+	}
+
+	return resp.Body.Close()
+}
+
+// List returns a channel that yields all names of blobs of type t. A
+// goroutine is started for this. If the channel done is closed, sending
+// stops.
+func (b *restBackend) List(t backend.Type, done <-chan struct{}) <-chan string {
+	ch := make(chan string)
+
+	url := restPath(b.url, backend.Handle{Type: t})
+	if !strings.HasSuffix(url, "/") {
+		url += "/"
+	}
+
+	client := *b.client
+	<-b.connChan
+	resp, err := client.Get(url)
+	b.connChan <- struct{}{}
+
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+
+	if err != nil {
+		close(ch)
+		return ch
+	}
+
+	dec := json.NewDecoder(resp.Body)
+	var list []string
+	if err = dec.Decode(&list); err != nil {
+		close(ch)
+		return ch
+	}
+
+	go func() {
+		defer close(ch)
+		for _, m := range list {
+			select {
+			case ch <- m:
+			case <-done:
+				return
+			}
+		}
+	}()
+
+	return ch
+}
+
+// Close closes all open files.
+func (b *restBackend) Close() error {
+	// this does not need to do anything, all open files are closed within the
+	// same function.
+	return nil
+}

src/restic/backend/rest/rest_path_test.go

@@ -0,0 +1,48 @@
+package rest
+
+import (
+	"net/url"
+	"restic/backend"
+	"testing"
+)
+
+var restPathTests = []struct {
+	Handle backend.Handle
+	URL    *url.URL
+	Result string
+}{
+	{
+		URL: parseURL("https://hostname.foo"),
+		Handle: backend.Handle{
+			Type: backend.Data,
+			Name: "foobar",
+		},
+		Result: "https://hostname.foo/data/foobar",
+	},
+	{
+		URL: parseURL("https://hostname.foo:1234/prefix/repo"),
+		Handle: backend.Handle{
+			Type: backend.Lock,
+			Name: "foobar",
+		},
+		Result: "https://hostname.foo:1234/prefix/repo/locks/foobar",
+	},
+	{
+		URL: parseURL("https://hostname.foo:1234/prefix/repo"),
+		Handle: backend.Handle{
+			Type: backend.Config,
+			Name: "foobar",
+		},
+		Result: "https://hostname.foo:1234/prefix/repo/config",
+	},
+}
+
+func TestRESTPaths(t *testing.T) {
+	for i, test := range restPathTests {
+		result := restPath(test.URL, test.Handle)
+		if result != test.Result {
+			t.Errorf("test %d: resulting URL does not match, want:\n  %#v\ngot: \n  %#v",
+				i, test.Result, result)
+		}
+	}
+}

src/restic/backend/rest/rest_test.go

@@ -0,0 +1,54 @@
+package rest_test
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+
+	"restic/backend"
+	"restic/backend/rest"
+	"restic/backend/test"
+	. "restic/test"
+)
+
+//go:generate go run ../test/generate_backend_tests.go
+
+func init() {
+	if TestRESTServer == "" {
+		SkipMessage = "REST test server not available"
+		return
+	}
+
+	url, err := url.Parse(TestRESTServer)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "invalid url: %v\n", err)
+		return
+	}
+
+	cfg := rest.Config{
+		URL: url,
+	}
+
+	test.CreateFn = func() (backend.Backend, error) {
+		be, err := rest.Open(cfg)
+		if err != nil {
+			return nil, err
+		}
+
+		exists, err := be.Test(backend.Config, "")
+		if err != nil {
+			return nil, err
+		}
+
+		if exists {
+			return nil, errors.New("config already exists")
+		}
+
+		return be, nil
+	}
+
+	test.OpenFn = func() (backend.Backend, error) {
+		return rest.Open(cfg)
+	}
+}

src/restic/backend/test/tests.go

@@ -164,7 +164,8 @@ func TestConfig(t testing.TB) {
 	// try accessing the config with different names, should all return the
 	// same config
 	for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
-		buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Config}, nil)
+		h := backend.Handle{Type: backend.Config, Name: name}
+		buf, err := backend.LoadAll(b, h, nil)
 		if err != nil {
 			t.Fatalf("unable to read config with name %q: %v", name, err)
 		}

src/restic/cmd/restic/global.go

@@ -8,15 +8,17 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/jessevdk/go-flags"
-	"golang.org/x/crypto/ssh/terminal"
 	"restic/backend"
 	"restic/backend/local"
+	"restic/backend/rest"
 	"restic/backend/s3"
 	"restic/backend/sftp"
 	"restic/debug"
 	"restic/location"
 	"restic/repository"
+
+	"github.com/jessevdk/go-flags"
+	"golang.org/x/crypto/ssh/terminal"
 )
 
 var version = "compiled manually"
@@ -247,6 +249,8 @@ func open(s string) (backend.Backend, error) {
 
 		debug.Log("open", "opening s3 repository at %#v", cfg)
 		return s3.Open(cfg)
+	case "rest":
+		return rest.Open(loc.Config.(rest.Config))
 	}
 
 	debug.Log("open", "invalid repository location: %v", s)
@@ -280,6 +284,8 @@ func create(s string) (backend.Backend, error) {
 
 		debug.Log("open", "create s3 repository at %#v", loc.Config)
 		return s3.Open(cfg)
+	case "rest":
+		return rest.Open(loc.Config.(rest.Config))
 	}
 
 	debug.Log("open", "invalid repository scheme: %v", s)

src/restic/location/location.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"restic/backend/local"
+	"restic/backend/rest"
 	"restic/backend/s3"
 	"restic/backend/sftp"
 )
@@ -27,6 +28,7 @@ var parsers = []parser{
 	{"local", local.ParseConfig},
 	{"sftp", sftp.ParseConfig},
 	{"s3", s3.ParseConfig},
+	{"rest", rest.ParseConfig},
 }
 
 // Parse extracts repository location information from the string s. If s

src/restic/location/location_test.go

@@ -1,13 +1,24 @@
 package location
 
 import (
+	"net/url"
 	"reflect"
 	"testing"
 
+	"restic/backend/rest"
 	"restic/backend/s3"
 	"restic/backend/sftp"
 )
 
+func parseURL(s string) *url.URL {
+	u, err := url.Parse(s)
+	if err != nil {
+		panic(err)
+	}
+
+	return u
+}
+
 var parseTests = []struct {
 	s string
 	u Location
@@ -101,6 +112,11 @@ var parseTests = []struct {
 			UseHTTP:  true,
 		}},
 	},
+	{"rest:http://hostname.foo:1234/", Location{Scheme: "rest",
+		Config: rest.Config{
+			URL: parseURL("http://hostname.foo:1234/"),
+		}},
+	},
 }
 
 func TestParse(t *testing.T) {

src/restic/test/backend.go

@@ -23,6 +23,7 @@ var (
 	TestWalkerPath        = getStringVar("RESTIC_TEST_PATH", ".")
 	BenchArchiveDirectory = getStringVar("RESTIC_BENCH_DIR", ".")
 	TestS3Server          = getStringVar("RESTIC_TEST_S3_SERVER", "")
+	TestRESTServer        = getStringVar("RESTIC_TEST_REST_SERVER", "")
 )
 
 func getStringVar(name, defaultValue string) string {