From 345b6c4694b218916b438a4f8460e74ef7acc846 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Tue, 13 Mar 2018 20:38:08 +0100
Subject: [PATCH 01/28] Move backend/sftp.SplitShellArgs to backend/

---
 internal/backend/sftp/sftp.go                                | 2 +-
 internal/backend/{sftp/split.go => shell_split.go}           | 2 +-
 internal/backend/{sftp/split_test.go => shell_split_test.go} | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)
 rename internal/backend/{sftp/split.go => shell_split.go} (99%)
 rename internal/backend/{sftp/split_test.go => shell_split_test.go} (99%)

diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go
index 8f7855a37..47b6871c1 100644
--- a/internal/backend/sftp/sftp.go
+++ b/internal/backend/sftp/sftp.go
@@ -179,7 +179,7 @@ func (r *SFTP) IsNotExist(err error) bool {
 
 func buildSSHCommand(cfg Config) (cmd string, args []string, err error) {
 	if cfg.Command != "" {
-		return SplitShellArgs(cfg.Command)
+		return backend.SplitShellArgs(cfg.Command)
 	}
 
 	cmd = "ssh"
diff --git a/internal/backend/sftp/split.go b/internal/backend/shell_split.go
similarity index 99%
rename from internal/backend/sftp/split.go
rename to internal/backend/shell_split.go
index d429f7e35..d28ea6034 100644
--- a/internal/backend/sftp/split.go
+++ b/internal/backend/shell_split.go
@@ -1,4 +1,4 @@
-package sftp
+package backend
 
 import (
 	"unicode"
diff --git a/internal/backend/sftp/split_test.go b/internal/backend/shell_split_test.go
similarity index 99%
rename from internal/backend/sftp/split_test.go
rename to internal/backend/shell_split_test.go
index 06241b29a..bb7963d21 100644
--- a/internal/backend/sftp/split_test.go
+++ b/internal/backend/shell_split_test.go
@@ -1,4 +1,4 @@
-package sftp
+package backend
 
 import (
 	"reflect"

From 34f27edc034aaba604f148a5514454cf177154fa Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Tue, 13 Mar 2018 20:50:37 +0100
Subject: [PATCH 02/28] Refactor SplitShellStrings

---
 internal/backend/sftp/sftp.go        |  7 +++++-
 internal/backend/shell_split.go      | 20 ++++++++---------
 internal/backend/shell_split_test.go | 32 ++++++++++------------------
 3 files changed, 26 insertions(+), 33 deletions(-)

diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go
index 47b6871c1..1f98cf56b 100644
--- a/internal/backend/sftp/sftp.go
+++ b/internal/backend/sftp/sftp.go
@@ -179,7 +179,12 @@ func (r *SFTP) IsNotExist(err error) bool {
 
 func buildSSHCommand(cfg Config) (cmd string, args []string, err error) {
 	if cfg.Command != "" {
-		return backend.SplitShellArgs(cfg.Command)
+		args, err := backend.SplitShellStrings(cfg.Command)
+		if err != nil {
+			return "", nil, err
+		}
+
+		return args[0], args[1:], nil
 	}
 
 	cmd = "ssh"
diff --git a/internal/backend/shell_split.go b/internal/backend/shell_split.go
index d28ea6034..eff527616 100644
--- a/internal/backend/shell_split.go
+++ b/internal/backend/shell_split.go
@@ -41,8 +41,8 @@ func (s *shellSplitter) isSplitChar(c rune) bool {
 	return c == '\\' || unicode.IsSpace(c)
 }
 
-// SplitShellArgs returns the list of arguments from a shell command string.
-func SplitShellArgs(data string) (cmd string, args []string, err error) {
+// SplitShellStrings returns the list of shell strings from a shell command string.
+func SplitShellStrings(data string) (strs []string, err error) {
 	s := &shellSplitter{}
 
 	// derived from strings.SplitFunc
@@ -50,7 +50,7 @@ func SplitShellArgs(data string) (cmd string, args []string, err error) {
 	for i, rune := range data {
 		if s.isSplitChar(rune) {
 			if fieldStart >= 0 {
-				args = append(args, data[fieldStart:i])
+				strs = append(strs, data[fieldStart:i])
 				fieldStart = -1
 			}
 		} else if fieldStart == -1 {
@@ -58,21 +58,19 @@ func SplitShellArgs(data string) (cmd string, args []string, err error) {
 		}
 	}
 	if fieldStart >= 0 { // Last field might end at EOF.
-		args = append(args, data[fieldStart:])
+		strs = append(strs, data[fieldStart:])
 	}
 
 	switch s.quote {
 	case '\'':
-		return "", nil, errors.New("single-quoted string not terminated")
+		return nil, errors.New("single-quoted string not terminated")
 	case '"':
-		return "", nil, errors.New("double-quoted string not terminated")
+		return nil, errors.New("double-quoted string not terminated")
 	}
 
-	if len(args) == 0 {
-		return "", nil, errors.New("command string is empty")
+	if len(strs) == 0 {
+		return nil, errors.New("command string is empty")
 	}
 
-	cmd, args = args[0], args[1:]
-
-	return cmd, args, nil
+	return strs, nil
 }
diff --git a/internal/backend/shell_split_test.go b/internal/backend/shell_split_test.go
index bb7963d21..40ae84c63 100644
--- a/internal/backend/shell_split_test.go
+++ b/internal/backend/shell_split_test.go
@@ -8,59 +8,53 @@ import (
 func TestShellSplitter(t *testing.T) {
 	var tests = []struct {
 		data string
-		cmd  string
 		args []string
 	}{
 		{
 			`foo`,
-			"foo", []string{},
+			[]string{"foo"},
 		},
 		{
 			`'foo'`,
-			"foo", []string{},
+			[]string{"foo"},
 		},
 		{
 			`foo bar baz`,
-			"foo", []string{"bar", "baz"},
+			[]string{"foo", "bar", "baz"},
 		},
 		{
 			`foo 'bar' baz`,
-			"foo", []string{"bar", "baz"},
+			[]string{"foo", "bar", "baz"},
 		},
 		{
 			`'bar box' baz`,
-			"bar box", []string{"baz"},
+			[]string{"bar box", "baz"},
 		},
 		{
 			`"bar 'box'" baz`,
-			"bar 'box'", []string{"baz"},
+			[]string{"bar 'box'", "baz"},
 		},
 		{
 			`'bar "box"' baz`,
-			`bar "box"`, []string{"baz"},
+			[]string{`bar "box"`, "baz"},
 		},
 		{
 			`\"bar box baz`,
-			`"bar`, []string{"box", "baz"},
+			[]string{`"bar`, "box", "baz"},
 		},
 		{
 			`"bar/foo/x" "box baz"`,
-			"bar/foo/x", []string{"box baz"},
+			[]string{"bar/foo/x", "box baz"},
 		},
 	}
 
 	for _, test := range tests {
 		t.Run("", func(t *testing.T) {
-			cmd, args, err := SplitShellArgs(test.data)
+			args, err := SplitShellStrings(test.data)
 			if err != nil {
 				t.Fatal(err)
 			}
 
-			if cmd != test.cmd {
-				t.Fatalf("wrong cmd returned, want:\n  %#v\ngot:\n  %#v",
-					test.cmd, cmd)
-			}
-
 			if !reflect.DeepEqual(args, test.args) {
 				t.Fatalf("wrong args returned, want:\n  %#v\ngot:\n  %#v",
 					test.args, args)
@@ -94,7 +88,7 @@ func TestShellSplitterInvalid(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run("", func(t *testing.T) {
-			cmd, args, err := SplitShellArgs(test.data)
+			args, err := SplitShellStrings(test.data)
 			if err == nil {
 				t.Fatalf("expected error not found: %v", test.err)
 			}
@@ -103,10 +97,6 @@ func TestShellSplitterInvalid(t *testing.T) {
 				t.Fatalf("expected error not found, want:\n  %q\ngot:\n  %q", test.err, err.Error())
 			}
 
-			if cmd != "" {
-				t.Fatalf("splitter returned cmd from invalid data: %v", cmd)
-			}
-
 			if len(args) > 0 {
 				t.Fatalf("splitter returned fields from invalid data: %v", args)
 			}

From cf4cf94418a777c776d6b2583719d2e7db618375 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Tue, 13 Mar 2018 21:09:16 +0100
Subject: [PATCH 03/28] Move backend/sftp.StartForeground to backend/

---
 internal/backend/{sftp => }/foreground_solaris.go | 7 +++++--
 internal/backend/{sftp => }/foreground_unix.go    | 6 +++---
 internal/backend/{sftp => }/foreground_windows.go | 4 ++--
 internal/backend/sftp/sftp.go                     | 2 +-
 4 files changed, 11 insertions(+), 8 deletions(-)
 rename internal/backend/{sftp => }/foreground_solaris.go (58%)
 rename internal/backend/{sftp => }/foreground_unix.go (91%)
 rename internal/backend/{sftp => }/foreground_windows.go (84%)

diff --git a/internal/backend/sftp/foreground_solaris.go b/internal/backend/foreground_solaris.go
similarity index 58%
rename from internal/backend/sftp/foreground_solaris.go
rename to internal/backend/foreground_solaris.go
index 5114eb9ea..501f9c1a1 100644
--- a/internal/backend/sftp/foreground_solaris.go
+++ b/internal/backend/foreground_solaris.go
@@ -1,4 +1,4 @@
-package sftp
+package backend
 
 import (
 	"os/exec"
@@ -7,7 +7,10 @@ import (
 	"github.com/restic/restic/internal/errors"
 )
 
-func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
+// StartForeground runs cmd in the foreground, by temporarily switching to the
+// new process group created for cmd. The returned function `bg` switches back
+// to the previous process group.
+func StartForeground(cmd *exec.Cmd) (bg func() error, err error) {
 	// run the command in it's own process group so that SIGINT
 	// is not sent to it.
 	cmd.SysProcAttr = &syscall.SysProcAttr{
diff --git a/internal/backend/sftp/foreground_unix.go b/internal/backend/foreground_unix.go
similarity index 91%
rename from internal/backend/sftp/foreground_unix.go
rename to internal/backend/foreground_unix.go
index 789cecb1c..1662a0250 100644
--- a/internal/backend/sftp/foreground_unix.go
+++ b/internal/backend/foreground_unix.go
@@ -1,7 +1,7 @@
 // +build !solaris
 // +build !windows
 
-package sftp
+package backend
 
 import (
 	"os"
@@ -24,10 +24,10 @@ func tcsetpgrp(fd int, pid int) error {
 	return errno
 }
 
-// startForeground runs cmd in the foreground, by temporarily switching to the
+// StartForeground runs cmd in the foreground, by temporarily switching to the
 // new process group created for cmd. The returned function `bg` switches back
 // to the previous process group.
-func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
+func StartForeground(cmd *exec.Cmd) (bg func() error, err error) {
 	// open the TTY, we need the file descriptor
 	tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0)
 	if err != nil {
diff --git a/internal/backend/sftp/foreground_windows.go b/internal/backend/foreground_windows.go
similarity index 84%
rename from internal/backend/sftp/foreground_windows.go
rename to internal/backend/foreground_windows.go
index e57b4d3a4..0e8b03a13 100644
--- a/internal/backend/sftp/foreground_windows.go
+++ b/internal/backend/foreground_windows.go
@@ -1,4 +1,4 @@
-package sftp
+package backend
 
 import (
 	"os/exec"
@@ -6,7 +6,7 @@ import (
 	"github.com/restic/restic/internal/errors"
 )
 
-// startForeground runs cmd in the foreground, by temporarily switching to the
+// StartForeground runs cmd in the foreground, by temporarily switching to the
 // new process group created for cmd. The returned function `bg` switches back
 // to the previous process group.
 func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go
index 1f98cf56b..268a3404a 100644
--- a/internal/backend/sftp/sftp.go
+++ b/internal/backend/sftp/sftp.go
@@ -65,7 +65,7 @@ func startClient(program string, args ...string) (*SFTP, error) {
 		return nil, errors.Wrap(err, "cmd.StdoutPipe")
 	}
 
-	bg, err := startForeground(cmd)
+	bg, err := backend.StartForeground(cmd)
 	if err != nil {
 		return nil, errors.Wrap(err, "cmd.Start")
 	}

From cabbbd2b14f2c6fca98ad48b620300008b389cea Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Tue, 13 Mar 2018 22:22:35 +0100
Subject: [PATCH 04/28] backend/rest: Export Content-Types

---
 internal/backend/rest/rest.go | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go
index f5254f3cc..94e3d4f72 100644
--- a/internal/backend/rest/rest.go
+++ b/internal/backend/rest/rest.go
@@ -30,9 +30,10 @@ type restBackend struct {
 	backend.Layout
 }
 
+// the REST API protocol version is decided by HTTP request headers, these are the constants.
 const (
-	contentTypeV1 = "application/vnd.x.restic.rest.v1"
-	contentTypeV2 = "application/vnd.x.restic.rest.v2"
+	ContentTypeV1 = "application/vnd.x.restic.rest.v1"
+	ContentTypeV2 = "application/vnd.x.restic.rest.v2"
 )
 
 // Open opens the REST backend with the given config.
@@ -119,7 +120,7 @@ func (b *restBackend) Save(ctx context.Context, h restic.Handle, rd restic.Rewin
 		return errors.Wrap(err, "NewRequest")
 	}
 	req.Header.Set("Content-Type", "application/octet-stream")
-	req.Header.Set("Accept", contentTypeV2)
+	req.Header.Set("Accept", ContentTypeV2)
 
 	// explicitly set the content length, this prevents chunked encoding and
 	// let's the server know what's coming.
@@ -198,7 +199,7 @@ func (b *restBackend) openReader(ctx context.Context, h restic.Handle, length in
 		byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
 	}
 	req.Header.Set("Range", byteRange)
-	req.Header.Set("Accept", contentTypeV2)
+	req.Header.Set("Accept", ContentTypeV2)
 	debug.Log("Load(%v) send range %v", h, byteRange)
 
 	b.sem.GetToken()
@@ -236,7 +237,7 @@ func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInf
 	if err != nil {
 		return restic.FileInfo{}, errors.Wrap(err, "NewRequest")
 	}
-	req.Header.Set("Accept", contentTypeV2)
+	req.Header.Set("Accept", ContentTypeV2)
 
 	b.sem.GetToken()
 	resp, err := ctxhttp.Do(ctx, b.client, req)
@@ -291,7 +292,7 @@ func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
 	if err != nil {
 		return errors.Wrap(err, "http.NewRequest")
 	}
-	req.Header.Set("Accept", contentTypeV2)
+	req.Header.Set("Accept", ContentTypeV2)
 
 	b.sem.GetToken()
 	resp, err := ctxhttp.Do(ctx, b.client, req)
@@ -330,7 +331,7 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
 	if err != nil {
 		return errors.Wrap(err, "NewRequest")
 	}
-	req.Header.Set("Accept", contentTypeV2)
+	req.Header.Set("Accept", ContentTypeV2)
 
 	b.sem.GetToken()
 	resp, err := ctxhttp.Do(ctx, b.client, req)
@@ -344,7 +345,7 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
 		return errors.Errorf("List failed, server response: %v (%v)", resp.Status, resp.StatusCode)
 	}
 
-	if resp.Header.Get("Content-Type") == contentTypeV2 {
+	if resp.Header.Get("Content-Type") == ContentTypeV2 {
 		return b.listv2(ctx, t, resp, fn)
 	}
 

From 61f6db25f4f0377e4140e012254828844b415ada Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 11 Mar 2018 22:01:37 +0100
Subject: [PATCH 05/28] CI: install rclone

---
 run_integration_tests.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/run_integration_tests.go b/run_integration_tests.go
index c040199aa..a5451560e 100644
--- a/run_integration_tests.go
+++ b/run_integration_tests.go
@@ -97,6 +97,7 @@ func (env *TravisEnvironment) Prepare() error {
 		"github.com/golang/dep/cmd/dep",
 		"github.com/restic/rest-server/cmd/rest-server",
 		"github.com/restic/calens",
+		"github.com/ncw/rclone",
 	}
 
 	for _, pkg := range pkgs {

From e377759c8101671d1bd49c3bf94ef956c4eb31ca Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Tue, 13 Mar 2018 22:30:41 +0100
Subject: [PATCH 06/28] rest: Export Backend struct

---
 internal/backend/rest/rest.go | 39 ++++++++++++++++++-----------------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go
index 94e3d4f72..3e41265a9 100644
--- a/internal/backend/rest/rest.go
+++ b/internal/backend/rest/rest.go
@@ -21,9 +21,10 @@ import (
 )
 
 // make sure the rest backend implements restic.Backend
-var _ restic.Backend = &restBackend{}
+var _ restic.Backend = &Backend{}
 
-type restBackend struct {
+// Backend uses the REST protocol to access data stored on a server.
+type Backend struct {
 	url    *url.URL
 	sem    *backend.Semaphore
 	client *http.Client
@@ -37,7 +38,7 @@ const (
 )
 
 // Open opens the REST backend with the given config.
-func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) {
+func Open(cfg Config, rt http.RoundTripper) (*Backend, error) {
 	client := &http.Client{Transport: rt}
 
 	sem, err := backend.NewSemaphore(cfg.Connections)
@@ -51,7 +52,7 @@ func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) {
 		url = url[:len(url)-1]
 	}
 
-	be := &restBackend{
+	be := &Backend{
 		url:    cfg.URL,
 		client: client,
 		Layout: &backend.RESTLayout{URL: url, Join: path.Join},
@@ -62,7 +63,7 @@ func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) {
 }
 
 // Create creates a new REST on server configured in config.
-func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
+func Create(cfg Config, rt http.RoundTripper) (*Backend, error) {
 	be, err := Open(cfg, rt)
 	if err != nil {
 		return nil, err
@@ -101,12 +102,12 @@ func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
 }
 
 // Location returns this backend's location (the server's URL).
-func (b *restBackend) Location() string {
+func (b *Backend) Location() string {
 	return b.url.String()
 }
 
 // Save stores data in the backend at the handle.
-func (b *restBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
+func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
 	if err := h.Valid(); err != nil {
 		return err
 	}
@@ -163,7 +164,7 @@ func (e ErrIsNotExist) Error() string {
 }
 
 // IsNotExist returns true if the error was caused by a non-existing file.
-func (b *restBackend) IsNotExist(err error) bool {
+func (b *Backend) IsNotExist(err error) bool {
 	err = errors.Cause(err)
 	_, ok := err.(ErrIsNotExist)
 	return ok
@@ -171,11 +172,11 @@ func (b *restBackend) IsNotExist(err error) bool {
 
 // Load runs fn with a reader that yields the contents of the file at h at the
 // given offset.
-func (b *restBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
 	return backend.DefaultLoad(ctx, h, length, offset, b.openReader, fn)
 }
 
-func (b *restBackend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
 	debug.Log("Load %v, length %v, offset %v", h, length, offset)
 	if err := h.Valid(); err != nil {
 		return nil, err
@@ -228,7 +229,7 @@ func (b *restBackend) openReader(ctx context.Context, h restic.Handle, length in
 }
 
 // Stat returns information about a blob.
-func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
+func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
 	if err := h.Valid(); err != nil {
 		return restic.FileInfo{}, err
 	}
@@ -273,7 +274,7 @@ func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInf
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (b *restBackend) Test(ctx context.Context, h restic.Handle) (bool, error) {
+func (b *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
 	_, err := b.Stat(ctx, h)
 	if err != nil {
 		return false, nil
@@ -283,7 +284,7 @@ func (b *restBackend) Test(ctx context.Context, h restic.Handle) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
+func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
 	if err := h.Valid(); err != nil {
 		return err
 	}
@@ -321,7 +322,7 @@ func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
 
 // List runs fn for each file in the backend which has the type t. When an
 // error occurs (or fn returns an error), List stops and returns it.
-func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
+func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
 	url := b.Dirname(restic.Handle{Type: t})
 	if !strings.HasSuffix(url, "/") {
 		url += "/"
@@ -355,7 +356,7 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
 // listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET
 // /data/`) only returns the names of the files, so we need to issue an HTTP
 // HEAD request for each file.
-func (b *restBackend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
+func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
 	debug.Log("parsing API v1 response")
 	dec := json.NewDecoder(resp.Body)
 	var list []string
@@ -389,7 +390,7 @@ func (b *restBackend) listv1(ctx context.Context, t restic.FileType, resp *http.
 
 // listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET
 // /data/`) returns the names and sizes of all files.
-func (b *restBackend) listv2(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
+func (b *Backend) listv2(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
 	debug.Log("parsing API v2 response")
 	dec := json.NewDecoder(resp.Body)
 
@@ -425,21 +426,21 @@ func (b *restBackend) listv2(ctx context.Context, t restic.FileType, resp *http.
 }
 
 // Close closes all open files.
-func (b *restBackend) Close() error {
+func (b *Backend) Close() error {
 	// this does not need to do anything, all open files are closed within the
 	// same function.
 	return nil
 }
 
 // Remove keys for a specified backend type.
-func (b *restBackend) removeKeys(ctx context.Context, t restic.FileType) error {
+func (b *Backend) removeKeys(ctx context.Context, t restic.FileType) error {
 	return b.List(ctx, t, func(fi restic.FileInfo) error {
 		return b.Remove(ctx, restic.Handle{Type: t, Name: fi.Name})
 	})
 }
 
 // Delete removes all data in the backend.
-func (b *restBackend) Delete(ctx context.Context) error {
+func (b *Backend) Delete(ctx context.Context) error {
 	alltypes := []restic.FileType{
 		restic.DataFile,
 		restic.KeyFile,

From fe99340e408f55a270096f08f68a2f0e2a6629c3 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Tue, 13 Mar 2018 22:30:51 +0100
Subject: [PATCH 07/28] Add rclone backend

---
 cmd/restic/global.go                        |  13 ++
 internal/backend/foreground_windows.go      |   2 +-
 internal/backend/location/location.go       |   2 +
 internal/backend/rclone/backend.go          | 225 ++++++++++++++++++++
 internal/backend/rclone/backend_test.go     |  83 ++++++++
 internal/backend/rclone/config.go           |  36 ++++
 internal/backend/rclone/config_test.go      |  33 +++
 internal/backend/rclone/stdio_conn.go       |  72 +++++++
 internal/backend/rclone/stdio_conn_go110.go |  25 +++
 internal/backend/rclone/stdio_conn_other.go |  22 ++
 10 files changed, 512 insertions(+), 1 deletion(-)
 create mode 100644 internal/backend/rclone/backend.go
 create mode 100644 internal/backend/rclone/backend_test.go
 create mode 100644 internal/backend/rclone/config.go
 create mode 100644 internal/backend/rclone/config_test.go
 create mode 100644 internal/backend/rclone/stdio_conn.go
 create mode 100644 internal/backend/rclone/stdio_conn_go110.go
 create mode 100644 internal/backend/rclone/stdio_conn_other.go

diff --git a/cmd/restic/global.go b/cmd/restic/global.go
index 5d8e5110f..0c3d805b2 100644
--- a/cmd/restic/global.go
+++ b/cmd/restic/global.go
@@ -18,6 +18,7 @@ import (
 	"github.com/restic/restic/internal/backend/gs"
 	"github.com/restic/restic/internal/backend/local"
 	"github.com/restic/restic/internal/backend/location"
+	"github.com/restic/restic/internal/backend/rclone"
 	"github.com/restic/restic/internal/backend/rest"
 	"github.com/restic/restic/internal/backend/s3"
 	"github.com/restic/restic/internal/backend/sftp"
@@ -509,6 +510,14 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 			return nil, err
 		}
 
+		debug.Log("opening rest repository at %#v", cfg)
+		return cfg, nil
+	case "rclone":
+		cfg := loc.Config.(rclone.Config)
+		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
+			return nil, err
+		}
+
 		debug.Log("opening rest repository at %#v", cfg)
 		return cfg, nil
 	}
@@ -564,6 +573,8 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 		be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt)
 	case "rest":
 		be, err = rest.Open(cfg.(rest.Config), rt)
+	case "rclone":
+		be, err = rclone.Open(cfg.(rclone.Config))
 
 	default:
 		return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
@@ -625,6 +636,8 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 		return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt)
 	case "rest":
 		return rest.Create(cfg.(rest.Config), rt)
+	case "rclone":
+		return rclone.Open(cfg.(rclone.Config))
 	}
 
 	debug.Log("invalid repository scheme: %v", s)
diff --git a/internal/backend/foreground_windows.go b/internal/backend/foreground_windows.go
index 0e8b03a13..281d5f3f8 100644
--- a/internal/backend/foreground_windows.go
+++ b/internal/backend/foreground_windows.go
@@ -9,7 +9,7 @@ import (
 // StartForeground runs cmd in the foreground, by temporarily switching to the
 // new process group created for cmd. The returned function `bg` switches back
 // to the previous process group.
-func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
+func StartForeground(cmd *exec.Cmd) (bg func() error, err error) {
 	// just start the process and hope for the best
 	err = cmd.Start()
 	if err != nil {
diff --git a/internal/backend/location/location.go b/internal/backend/location/location.go
index cc547c60b..68f4dfaff 100644
--- a/internal/backend/location/location.go
+++ b/internal/backend/location/location.go
@@ -8,6 +8,7 @@ import (
 	"github.com/restic/restic/internal/backend/b2"
 	"github.com/restic/restic/internal/backend/gs"
 	"github.com/restic/restic/internal/backend/local"
+	"github.com/restic/restic/internal/backend/rclone"
 	"github.com/restic/restic/internal/backend/rest"
 	"github.com/restic/restic/internal/backend/s3"
 	"github.com/restic/restic/internal/backend/sftp"
@@ -38,6 +39,7 @@ var parsers = []parser{
 	{"azure", azure.ParseConfig},
 	{"swift", swift.ParseConfig},
 	{"rest", rest.ParseConfig},
+	{"rclone", rclone.ParseConfig},
 }
 
 func isPath(s string) bool {
diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go
new file mode 100644
index 000000000..e2e538cb4
--- /dev/null
+++ b/internal/backend/rclone/backend.go
@@ -0,0 +1,225 @@
+package rclone
+
+import (
+	"context"
+	"crypto/tls"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"time"
+
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/backend/rest"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
+	"golang.org/x/net/context/ctxhttp"
+	"golang.org/x/net/http2"
+)
+
+// Backend is used to access data stored somewhere via rclone.
+type Backend struct {
+	*rest.Backend
+	tr         *http2.Transport
+	cmd        *exec.Cmd
+	waitCh     <-chan struct{}
+	waitResult error
+}
+
+// run starts command with args and initializes the StdioConn.
+func run(command string, args ...string) (*StdioConn, *exec.Cmd, func() error, error) {
+	cmd := exec.Command(command, args...)
+	cmd.Stderr = os.Stderr
+
+	r, stdin, err := os.Pipe()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	stdout, w, err := os.Pipe()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	cmd.Stdin = r
+	cmd.Stdout = w
+
+	bg, err := backend.StartForeground(cmd)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	c := &StdioConn{
+		stdin:  stdout,
+		stdout: stdin,
+		cmd:    cmd,
+	}
+
+	return c, cmd, bg, nil
+}
+
+// New initializes a Backend and starts the process.
+func New(cfg Config) (*Backend, error) {
+	var (
+		args []string
+		err  error
+	)
+
+	// build program args, start with the program
+	if cfg.Program != "" {
+		a, err := backend.SplitShellStrings(cfg.Program)
+		if err != nil {
+			return nil, err
+		}
+		args = append(args, a...)
+	} else {
+		args = append(args, "rclone")
+	}
+
+	// then add the arguments
+	if cfg.Args != "" {
+		a, err := backend.SplitShellStrings(cfg.Args)
+		if err != nil {
+			return nil, err
+		}
+
+		args = append(args, a...)
+	} else {
+		args = append(args, "serve", "restic", "--stdio")
+	}
+
+	// finally, add the remote
+	args = append(args, cfg.Remote)
+	arg0, args := args[0], args[1:]
+
+	debug.Log("running command: %v %v", arg0, args)
+	conn, cmd, bg, err := run(arg0, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	tr := &http2.Transport{
+		AllowHTTP: true, // this is not really HTTP, just stdin/stdout
+		DialTLS: func(network, address string, cfg *tls.Config) (net.Conn, error) {
+			debug.Log("new connection requested, %v %v", network, address)
+			return conn, nil
+		},
+	}
+
+	waitCh := make(chan struct{})
+	be := &Backend{
+		tr:     tr,
+		cmd:    cmd,
+		waitCh: waitCh,
+	}
+
+	go func() {
+		debug.Log("waiting for error result")
+		err := cmd.Wait()
+		debug.Log("Wait returned %v", err)
+		be.waitResult = err
+		close(waitCh)
+	}()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	go func() {
+		debug.Log("monitoring command to cancel first HTTP request context")
+		select {
+		case <-ctx.Done():
+			debug.Log("context has been cancelled, returning")
+		case <-be.waitCh:
+			debug.Log("command has exited, cancelling context")
+			cancel()
+		}
+	}()
+
+	// send an HTTP request to the base URL, see if the server is there
+	client := &http.Client{
+		Transport: tr,
+		Timeout:   5 * time.Second,
+	}
+
+	req, err := http.NewRequest(http.MethodGet, "http://localhost/", nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept", rest.ContentTypeV2)
+
+	res, err := ctxhttp.Do(ctx, client, req)
+	if err != nil {
+		bg()
+		_ = cmd.Process.Kill()
+		return nil, errors.Errorf("error talking HTTP to rclone: %v", err)
+	}
+
+	debug.Log("HTTP status %q returned, moving instance to background", res.Status)
+	bg()
+
+	return be, nil
+}
+
+// Open starts an rclone process with the given config.
+func Open(cfg Config) (*Backend, error) {
+	be, err := New(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	url, err := url.Parse("http://localhost/")
+	if err != nil {
+		return nil, err
+	}
+
+	restConfig := rest.Config{
+		Connections: 20,
+		URL:         url,
+	}
+
+	restBackend, err := rest.Open(restConfig, be.tr)
+	if err != nil {
+		return nil, err
+	}
+
+	be.Backend = restBackend
+	return be, nil
+}
+
+// Create initializes a new restic repo with clone.
+func Create(cfg Config) (*Backend, error) {
+	be, err := New(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	debug.Log("new backend created")
+
+	url, err := url.Parse("http://localhost/")
+	if err != nil {
+		return nil, err
+	}
+
+	restConfig := rest.Config{
+		Connections: 20,
+		URL:         url,
+	}
+
+	restBackend, err := rest.Create(restConfig, be.tr)
+	if err != nil {
+		return nil, err
+	}
+
+	be.Backend = restBackend
+	return be, nil
+}
+
+// Close terminates the backend.
+func (be *Backend) Close() error {
+	debug.Log("exting rclone")
+	be.tr.CloseIdleConnections()
+	<-be.waitCh
+	debug.Log("wait for rclone returned: %v", be.waitResult)
+	return be.waitResult
+}
diff --git a/internal/backend/rclone/backend_test.go b/internal/backend/rclone/backend_test.go
new file mode 100644
index 000000000..ab8eb8376
--- /dev/null
+++ b/internal/backend/rclone/backend_test.go
@@ -0,0 +1,83 @@
+package rclone_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/restic/restic/internal/backend/rclone"
+	"github.com/restic/restic/internal/backend/test"
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+const rcloneConfig = `
+[local]
+type = local
+`
+
+func newTestSuite(t testing.TB) *test.Suite {
+	dir, cleanup := rtest.TempDir(t)
+
+	return &test.Suite{
+		// NewConfig returns a config for a new temporary backend that will be used in tests.
+		NewConfig: func() (interface{}, error) {
+			cfgfile := filepath.Join(dir, "rclone.conf")
+			t.Logf("write rclone config to %v", cfgfile)
+			err := ioutil.WriteFile(cfgfile, []byte(rcloneConfig), 0644)
+			if err != nil {
+				return nil, err
+			}
+
+			t.Logf("use backend at %v", dir)
+
+			repodir := filepath.Join(dir, "repo")
+			err = os.Mkdir(repodir, 0755)
+			if err != nil {
+				return nil, err
+			}
+
+			cfg := rclone.NewConfig()
+			cfg.Program = fmt.Sprintf("rclone --config %q", cfgfile)
+			cfg.Remote = "local:" + repodir
+			return cfg, nil
+		},
+
+		// CreateFn is a function that creates a temporary repository for the tests.
+		Create: func(config interface{}) (restic.Backend, error) {
+			t.Logf("Create()")
+			cfg := config.(rclone.Config)
+			return rclone.Create(cfg)
+		},
+
+		// OpenFn is a function that opens a previously created temporary repository.
+		Open: func(config interface{}) (restic.Backend, error) {
+			t.Logf("Open()")
+			cfg := config.(rclone.Config)
+			return rclone.Open(cfg)
+		},
+
+		// CleanupFn removes data created during the tests.
+		Cleanup: func(config interface{}) error {
+			t.Logf("cleanup dir %v", dir)
+			cleanup()
+			return nil
+		},
+	}
+}
+
+func TestBackendRclone(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			rtest.SkipDisallowed(t, "restic/backend/rclone.TestBackendRclone")
+		}
+	}()
+
+	newTestSuite(t).RunTests(t)
+}
+
+func BenchmarkBackendREST(t *testing.B) {
+	newTestSuite(t).RunBenchmarks(t)
+}
diff --git a/internal/backend/rclone/config.go b/internal/backend/rclone/config.go
new file mode 100644
index 000000000..a833d153b
--- /dev/null
+++ b/internal/backend/rclone/config.go
@@ -0,0 +1,36 @@
+package rclone
+
+import (
+	"strings"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/options"
+)
+
+// Config contains all configuration necessary to start rclone.
+type Config struct {
+	Program string `option:"program" help:"path to rclone (default: rclone)"`
+	Args    string `option:"args"    help:"arguments for running rclone (default: restic serve --stdio)"`
+	Remote  string
+}
+
+func init() {
+	options.Register("rclone", Config{})
+}
+
+// NewConfig returns a new Config with the default values filled in.
+func NewConfig() Config {
+	return Config{}
+}
+
+// ParseConfig parses the string s and extracts the remote server URL.
+func ParseConfig(s string) (interface{}, error) {
+	if !strings.HasPrefix(s, "rclone:") {
+		return nil, errors.New("invalid rclone backend specification")
+	}
+
+	s = s[7:]
+	cfg := NewConfig()
+	cfg.Remote = s
+	return cfg, nil
+}
diff --git a/internal/backend/rclone/config_test.go b/internal/backend/rclone/config_test.go
new file mode 100644
index 000000000..05e4b2b29
--- /dev/null
+++ b/internal/backend/rclone/config_test.go
@@ -0,0 +1,33 @@
+package rclone
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseConfig(t *testing.T) {
+	var tests = []struct {
+		s   string
+		cfg Config
+	}{
+		{
+			"rclone:local:foo:/bar",
+			Config{
+				Remote: "local:foo:/bar",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run("", func(t *testing.T) {
+			cfg, err := ParseConfig(test.s)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if !reflect.DeepEqual(cfg, test.cfg) {
+				t.Fatalf("wrong config, want:\n  %v\ngot:\n  %v", test.cfg, cfg)
+			}
+		})
+	}
+}
diff --git a/internal/backend/rclone/stdio_conn.go b/internal/backend/rclone/stdio_conn.go
new file mode 100644
index 000000000..4472300ec
--- /dev/null
+++ b/internal/backend/rclone/stdio_conn.go
@@ -0,0 +1,72 @@
+package rclone
+
+import (
+	"net"
+	"os"
+	"os/exec"
+
+	"github.com/restic/restic/internal/debug"
+)
+
+// StdioConn implements a net.Conn via stdin/stdout.
+type StdioConn struct {
+	stdin                   *os.File
+	stdout                  *os.File
+	bytesWritten, bytesRead int
+	cmd                     *exec.Cmd
+}
+
+func (s *StdioConn) Read(p []byte) (int, error) {
+	n, err := s.stdin.Read(p)
+	s.bytesRead += n
+	return n, err
+}
+
+func (s *StdioConn) Write(p []byte) (int, error) {
+	n, err := s.stdout.Write(p)
+	s.bytesWritten += n
+	return n, err
+}
+
+// Close closes both streams.
+func (s *StdioConn) Close() error {
+	debug.Log("close server instance")
+	var errs []error
+
+	for _, f := range []func() error{s.stdin.Close, s.stdout.Close} {
+		err := f()
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	if len(errs) > 0 {
+		return errs[0]
+	}
+	return nil
+}
+
+// LocalAddr returns nil.
+func (s *StdioConn) LocalAddr() net.Addr {
+	return Addr{}
+}
+
+// RemoteAddr returns nil.
+func (s *StdioConn) RemoteAddr() net.Addr {
+	return Addr{}
+}
+
+// make sure StdioConn implements net.Conn
+var _ net.Conn = &StdioConn{}
+
+// Addr implements net.Addr for stdin/stdout.
+type Addr struct{}
+
+// Network returns the network type as a string.
+func (a Addr) Network() string {
+	return "stdio"
+}
+
+func (a Addr) String() string {
+	return "stdio"
+}
diff --git a/internal/backend/rclone/stdio_conn_go110.go b/internal/backend/rclone/stdio_conn_go110.go
new file mode 100644
index 000000000..b21f65f04
--- /dev/null
+++ b/internal/backend/rclone/stdio_conn_go110.go
@@ -0,0 +1,25 @@
+// +build go1.10
+
+package rclone
+
+import "time"
+
+// SetDeadline sets the read/write deadline.
+func (s *StdioConn) SetDeadline(t time.Time) error {
+	err1 := s.stdin.SetReadDeadline(t)
+	err2 := s.stdout.SetWriteDeadline(t)
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+// SetReadDeadline sets the read/write deadline.
+func (s *StdioConn) SetReadDeadline(t time.Time) error {
+	return s.stdin.SetReadDeadline(t)
+}
+
+// SetWriteDeadline sets the read/write deadline.
+func (s *StdioConn) SetWriteDeadline(t time.Time) error {
+	return s.stdout.SetWriteDeadline(t)
+}
diff --git a/internal/backend/rclone/stdio_conn_other.go b/internal/backend/rclone/stdio_conn_other.go
new file mode 100644
index 000000000..07f85961b
--- /dev/null
+++ b/internal/backend/rclone/stdio_conn_other.go
@@ -0,0 +1,22 @@
+// +build !go1.10
+
+package rclone
+
+import "time"
+
+// On Go < 1.10, it's not possible to set read/write deadlines on files, so we just ignore that.
+
+// SetDeadline sets the read/write deadline.
+func (s *StdioConn) SetDeadline(t time.Time) error {
+	return nil
+}
+
+// SetReadDeadline sets the read/write deadline.
+func (s *StdioConn) SetReadDeadline(t time.Time) error {
+	return nil
+}
+
+// SetWriteDeadline sets the read/write deadline.
+func (s *StdioConn) SetWriteDeadline(t time.Time) error {
+	return nil
+}

From 4dc0f24b38a74b63ddff59180c4c624c89a06e88 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Wed, 14 Mar 2018 20:54:48 +0100
Subject: [PATCH 08/28] backend/tests: Drain reader before returning error

---
 internal/backend/test/tests.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go
index caf0a9ac7..dec1e0bee 100644
--- a/internal/backend/test/tests.go
+++ b/internal/backend/test/tests.go
@@ -147,6 +147,10 @@ func (s *Suite) TestLoad(t *testing.T) {
 	}
 
 	err = b.Load(context.TODO(), handle, 0, 0, func(rd io.Reader) error {
+		_, err := io.Copy(ioutil.Discard, rd)
+		if err != nil {
+			t.Fatal(err)
+		}
 		return errors.Errorf("deliberate error")
 	})
 	if err == nil {

From 065fe1e54f200b118276be99dac67fb7a7cac5cd Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Wed, 14 Mar 2018 21:14:54 +0100
Subject: [PATCH 09/28] backend/rclone: Skip test if binary is unavailable

---
 internal/backend/rclone/backend_test.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/internal/backend/rclone/backend_test.go b/internal/backend/rclone/backend_test.go
index ab8eb8376..9f76b517c 100644
--- a/internal/backend/rclone/backend_test.go
+++ b/internal/backend/rclone/backend_test.go
@@ -4,11 +4,13 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"testing"
 
 	"github.com/restic/restic/internal/backend/rclone"
 	"github.com/restic/restic/internal/backend/test"
+	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
@@ -49,7 +51,12 @@ func newTestSuite(t testing.TB) *test.Suite {
 		Create: func(config interface{}) (restic.Backend, error) {
 			t.Logf("Create()")
 			cfg := config.(rclone.Config)
-			return rclone.Create(cfg)
+			be, err := rclone.Create(cfg)
+			if e, ok := errors.Cause(err).(*exec.Error); ok && e.Err == exec.ErrNotFound {
+				t.Skipf("program %q not found", e.Name)
+				return nil, nil
+			}
+			return be, err
 		},
 
 		// OpenFn is a function that opens a previously created temporary repository.

From 3622b60c130d051425a5563984f88b4bbd9a998d Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Wed, 14 Mar 2018 21:16:10 +0100
Subject: [PATCH 10/28] CI: Check that rclone backend test isn't skipped

---
 run_integration_tests.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/run_integration_tests.go b/run_integration_tests.go
index a5451560e..7c37d859c 100644
--- a/run_integration_tests.go
+++ b/run_integration_tests.go
@@ -192,6 +192,7 @@ func (env *TravisEnvironment) RunTests() error {
 		"restic/backend/rest.TestBackendREST",
 		"restic/backend/sftp.TestBackendSFTP",
 		"restic/backend/s3.TestBackendMinio",
+		"restic/backend/rclone.TestBackendRclone",
 	}
 
 	// if the test s3 repository is available, make sure that the test is not skipped

From 20352886f3829bb3f8ec9824bacfa761f500ff6e Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Wed, 14 Mar 2018 21:21:51 +0100
Subject: [PATCH 11/28] Update Gopkg.lock

---
 Gopkg.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gopkg.lock b/Gopkg.lock
index bae881508..25a3b8bed 100644
--- a/Gopkg.lock
+++ b/Gopkg.lock
@@ -232,6 +232,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "a7d099b3ce195ffc37adedb05a4386be38e6158925a1c0fe579efdc20fa11f6a"
+  inputs-digest = "d3d59414a33bb8ecc6d88a681c782a87244a565cc9d0f85615cfa0704c02800a"
   solver-name = "gps-cdcl"
   solver-version = 1

From 6d9a029e090b09d93bd0e65b444665d434f4821c Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Wed, 14 Mar 2018 21:29:54 +0100
Subject: [PATCH 12/28] backend/rclone: Prefix all error messages

---
 internal/backend/rclone/backend.go | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go
index e2e538cb4..e162c890c 100644
--- a/internal/backend/rclone/backend.go
+++ b/internal/backend/rclone/backend.go
@@ -1,8 +1,10 @@
 package rclone
 
 import (
+	"bufio"
 	"context"
 	"crypto/tls"
+	"fmt"
 	"net"
 	"net/http"
 	"net/url"
@@ -30,7 +32,18 @@ type Backend struct {
 // run starts command with args and initializes the StdioConn.
 func run(command string, args ...string) (*StdioConn, *exec.Cmd, func() error, error) {
 	cmd := exec.Command(command, args...)
-	cmd.Stderr = os.Stderr
+	p, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	// start goroutine to add a prefix to all messages printed by to stderr by rclone
+	go func() {
+		sc := bufio.NewScanner(p)
+		for sc.Scan() {
+			fmt.Fprintf(os.Stderr, "rclone: %v\n", sc.Text())
+		}
+	}()
 
 	r, stdin, err := os.Pipe()
 	if err != nil {

From 99b62c11b81e91085ba8f9500f6bbe47febac62b Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Wed, 14 Mar 2018 22:28:12 +0100
Subject: [PATCH 13/28] backend/rclone: Stop rclone in case of errors

---
 internal/backend/rclone/backend.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go
index e162c890c..c04eb5611 100644
--- a/internal/backend/rclone/backend.go
+++ b/internal/backend/rclone/backend.go
@@ -221,6 +221,7 @@ func Create(cfg Config) (*Backend, error) {
 
 	restBackend, err := rest.Create(restConfig, be.tr)
 	if err != nil {
+		_ = be.Close()
 		return nil, err
 	}
 

From fc0295016ab56bf6a1db9aa7e651cba27a7bdf37 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Thu, 15 Mar 2018 19:00:25 +0100
Subject: [PATCH 14/28] Address code review comments

---
 internal/backend/rclone/backend.go      |  2 +-
 internal/backend/rclone/backend_test.go | 26 +------------------------
 internal/backend/rclone/config.go       |  2 +-
 3 files changed, 3 insertions(+), 27 deletions(-)

diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go
index c04eb5611..40f434b3e 100644
--- a/internal/backend/rclone/backend.go
+++ b/internal/backend/rclone/backend.go
@@ -231,7 +231,7 @@ func Create(cfg Config) (*Backend, error) {
 
 // Close terminates the backend.
 func (be *Backend) Close() error {
-	debug.Log("exting rclone")
+	debug.Log("exiting rclone")
 	be.tr.CloseIdleConnections()
 	<-be.waitCh
 	debug.Log("wait for rclone returned: %v", be.waitResult)
diff --git a/internal/backend/rclone/backend_test.go b/internal/backend/rclone/backend_test.go
index 9f76b517c..16281035d 100644
--- a/internal/backend/rclone/backend_test.go
+++ b/internal/backend/rclone/backend_test.go
@@ -1,11 +1,7 @@
 package rclone_test
 
 import (
-	"fmt"
-	"io/ioutil"
-	"os"
 	"os/exec"
-	"path/filepath"
 	"testing"
 
 	"github.com/restic/restic/internal/backend/rclone"
@@ -15,35 +11,15 @@ import (
 	rtest "github.com/restic/restic/internal/test"
 )
 
-const rcloneConfig = `
-[local]
-type = local
-`
-
 func newTestSuite(t testing.TB) *test.Suite {
 	dir, cleanup := rtest.TempDir(t)
 
 	return &test.Suite{
 		// NewConfig returns a config for a new temporary backend that will be used in tests.
 		NewConfig: func() (interface{}, error) {
-			cfgfile := filepath.Join(dir, "rclone.conf")
-			t.Logf("write rclone config to %v", cfgfile)
-			err := ioutil.WriteFile(cfgfile, []byte(rcloneConfig), 0644)
-			if err != nil {
-				return nil, err
-			}
-
 			t.Logf("use backend at %v", dir)
-
-			repodir := filepath.Join(dir, "repo")
-			err = os.Mkdir(repodir, 0755)
-			if err != nil {
-				return nil, err
-			}
-
 			cfg := rclone.NewConfig()
-			cfg.Program = fmt.Sprintf("rclone --config %q", cfgfile)
-			cfg.Remote = "local:" + repodir
+			cfg.Remote = dir
 			return cfg, nil
 		},
 
diff --git a/internal/backend/rclone/config.go b/internal/backend/rclone/config.go
index a833d153b..fa4ce88df 100644
--- a/internal/backend/rclone/config.go
+++ b/internal/backend/rclone/config.go
@@ -10,7 +10,7 @@ import (
 // Config contains all configuration necessary to start rclone.
 type Config struct {
 	Program string `option:"program" help:"path to rclone (default: rclone)"`
-	Args    string `option:"args"    help:"arguments for running rclone (default: restic serve --stdio)"`
+	Args    string `option:"args"    help:"arguments for running rclone (default: serve restic --stdio)"`
 	Remote  string
 }
 

From 4d5c7a87492e3e5c6382d7532c225a38c993456a Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Thu, 15 Mar 2018 21:22:14 +0100
Subject: [PATCH 15/28] backend/rclone: Make sure rclone terminates

---
 internal/backend/rclone/backend.go    | 50 ++++++++++++++++++++++-----
 internal/backend/rclone/stdio_conn.go | 29 +++++++++-------
 2 files changed, 59 insertions(+), 20 deletions(-)

diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go
index 40f434b3e..67bd1eab7 100644
--- a/internal/backend/rclone/backend.go
+++ b/internal/backend/rclone/backend.go
@@ -10,6 +10,7 @@ import (
 	"net/url"
 	"os"
 	"os/exec"
+	"sync"
 	"time"
 
 	"github.com/restic/restic/internal/backend"
@@ -27,18 +28,25 @@ type Backend struct {
 	cmd        *exec.Cmd
 	waitCh     <-chan struct{}
 	waitResult error
+	wg         *sync.WaitGroup
+	conn       *StdioConn
 }
 
 // run starts command with args and initializes the StdioConn.
-func run(command string, args ...string) (*StdioConn, *exec.Cmd, func() error, error) {
+func run(command string, args ...string) (*StdioConn, *exec.Cmd, *sync.WaitGroup, func() error, error) {
 	cmd := exec.Command(command, args...)
+
 	p, err := cmd.StderrPipe()
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, nil, nil, err
 	}
 
+	var wg sync.WaitGroup
+
 	// start goroutine to add a prefix to all messages printed by to stderr by rclone
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
 		sc := bufio.NewScanner(p)
 		for sc.Scan() {
 			fmt.Fprintf(os.Stderr, "rclone: %v\n", sc.Text())
@@ -47,12 +55,12 @@ func run(command string, args ...string) (*StdioConn, *exec.Cmd, func() error, e
 
 	r, stdin, err := os.Pipe()
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, nil, nil, err
 	}
 
 	stdout, w, err := os.Pipe()
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, nil, nil, err
 	}
 
 	cmd.Stdin = r
@@ -60,7 +68,7 @@ func run(command string, args ...string) (*StdioConn, *exec.Cmd, func() error, e
 
 	bg, err := backend.StartForeground(cmd)
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, nil, nil, err
 	}
 
 	c := &StdioConn{
@@ -69,7 +77,7 @@ func run(command string, args ...string) (*StdioConn, *exec.Cmd, func() error, e
 		cmd:    cmd,
 	}
 
-	return c, cmd, bg, nil
+	return c, cmd, &wg, bg, nil
 }
 
 // New initializes a Backend and starts the process.
@@ -107,15 +115,20 @@ func New(cfg Config) (*Backend, error) {
 	arg0, args := args[0], args[1:]
 
 	debug.Log("running command: %v %v", arg0, args)
-	conn, cmd, bg, err := run(arg0, args...)
+	conn, cmd, wg, bg, err := run(arg0, args...)
 	if err != nil {
 		return nil, err
 	}
 
+	dialCount := 0
 	tr := &http2.Transport{
 		AllowHTTP: true, // this is not really HTTP, just stdin/stdout
 		DialTLS: func(network, address string, cfg *tls.Config) (net.Conn, error) {
 			debug.Log("new connection requested, %v %v", network, address)
+			if dialCount > 0 {
+				panic("dial count > 0")
+			}
+			dialCount++
 			return conn, nil
 		},
 	}
@@ -125,9 +138,13 @@ func New(cfg Config) (*Backend, error) {
 		tr:     tr,
 		cmd:    cmd,
 		waitCh: waitCh,
+		conn:   conn,
+		wg:     wg,
 	}
 
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
 		debug.Log("waiting for error result")
 		err := cmd.Wait()
 		debug.Log("Wait returned %v", err)
@@ -138,7 +155,9 @@ func New(cfg Config) (*Backend, error) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
 		debug.Log("monitoring command to cancel first HTTP request context")
 		select {
 		case <-ctx.Done():
@@ -160,6 +179,7 @@ func New(cfg Config) (*Backend, error) {
 		return nil, err
 	}
 	req.Header.Set("Accept", rest.ContentTypeV2)
+	req.Cancel = ctx.Done()
 
 	res, err := ctxhttp.Do(ctx, client, req)
 	if err != nil {
@@ -229,11 +249,25 @@ func Create(cfg Config) (*Backend, error) {
 	return be, nil
 }
 
+const waitForExit = 5 * time.Second
+
 // Close terminates the backend.
 func (be *Backend) Close() error {
 	debug.Log("exiting rclone")
 	be.tr.CloseIdleConnections()
-	<-be.waitCh
+
+	select {
+	case <-be.waitCh:
+		debug.Log("rclone exited")
+	case <-time.After(waitForExit):
+		debug.Log("timeout, closing file descriptors")
+		err := be.conn.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	be.wg.Wait()
 	debug.Log("wait for rclone returned: %v", be.waitResult)
 	return be.waitResult
 }
diff --git a/internal/backend/rclone/stdio_conn.go b/internal/backend/rclone/stdio_conn.go
index 4472300ec..bb4928176 100644
--- a/internal/backend/rclone/stdio_conn.go
+++ b/internal/backend/rclone/stdio_conn.go
@@ -4,6 +4,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"sync"
 
 	"github.com/restic/restic/internal/debug"
 )
@@ -14,6 +15,7 @@ type StdioConn struct {
 	stdout                  *os.File
 	bytesWritten, bytesRead int
 	cmd                     *exec.Cmd
+	close                   sync.Once
 }
 
 func (s *StdioConn) Read(p []byte) (int, error) {
@@ -29,21 +31,24 @@ func (s *StdioConn) Write(p []byte) (int, error) {
 }
 
 // Close closes both streams.
-func (s *StdioConn) Close() error {
-	debug.Log("close server instance")
-	var errs []error
+func (s *StdioConn) Close() (err error) {
+	s.close.Do(func() {
+		debug.Log("close stdio connection")
+		var errs []error
 
-	for _, f := range []func() error{s.stdin.Close, s.stdout.Close} {
-		err := f()
-		if err != nil {
-			errs = append(errs, err)
+		for _, f := range []func() error{s.stdin.Close, s.stdout.Close} {
+			err := f()
+			if err != nil {
+				errs = append(errs, err)
+			}
 		}
-	}
 
-	if len(errs) > 0 {
-		return errs[0]
-	}
-	return nil
+		if len(errs) > 0 {
+			err = errs[0]
+		}
+	})
+
+	return err
 }
 
 // LocalAddr returns nil.

From 17312d3a982d8435c7be1f59fede433fb48ed757 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Thu, 15 Mar 2018 21:37:18 +0100
Subject: [PATCH 16/28] backend/rest: Ensure base URL ends with slash

This makes it easier for rclone.
---
 doc/100_references.rst               |  2 +-
 internal/backend/rest/config.go      |  4 +++
 internal/backend/rest/config_test.go | 40 +++++++++++++++++-----------
 3 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/doc/100_references.rst b/doc/100_references.rst
index dc2b40ee7..44dcc6673 100644
--- a/doc/100_references.rst
+++ b/doc/100_references.rst
@@ -681,7 +681,7 @@ return JSON. Any different value for this header means API version 1.
 
 The placeholder ``{path}`` in this document is a path to the repository, so
 that multiple different repositories can be accessed. The default path is
-``/``.
+``/``. The path must end with a slash.
 
 POST {path}?create=true
 =======================
diff --git a/internal/backend/rest/config.go b/internal/backend/rest/config.go
index 34d7958fb..60c6bf92b 100644
--- a/internal/backend/rest/config.go
+++ b/internal/backend/rest/config.go
@@ -32,6 +32,10 @@ func ParseConfig(s string) (interface{}, error) {
 	}
 
 	s = s[5:]
+	if !strings.HasSuffix(s, "/") {
+		s += "/"
+	}
+
 	u, err := url.Parse(s)
 
 	if err != nil {
diff --git a/internal/backend/rest/config_test.go b/internal/backend/rest/config_test.go
index ed5413323..2d8e32a73 100644
--- a/internal/backend/rest/config_test.go
+++ b/internal/backend/rest/config_test.go
@@ -19,24 +19,34 @@ var configTests = []struct {
 	s   string
 	cfg Config
 }{
-	{"rest:http://localhost:1234", Config{
-		URL:         parseURL("http://localhost:1234"),
-		Connections: 5,
-	}},
+	{
+		s: "rest:http://localhost:1234",
+		cfg: Config{
+			URL:         parseURL("http://localhost:1234/"),
+			Connections: 5,
+		},
+	},
+	{
+		s: "rest:http://localhost:1234/",
+		cfg: Config{
+			URL:         parseURL("http://localhost:1234/"),
+			Connections: 5,
+		},
+	},
 }
 
 func TestParseConfig(t *testing.T) {
-	for i, test := range configTests {
-		cfg, err := ParseConfig(test.s)
-		if err != nil {
-			t.Errorf("test %d:%s failed: %v", i, test.s, err)
-			continue
-		}
+	for _, test := range configTests {
+		t.Run("", func(t *testing.T) {
+			cfg, err := ParseConfig(test.s)
+			if err != nil {
+				t.Fatalf("%s failed: %v", test.s, err)
+			}
 
-		if !reflect.DeepEqual(cfg, test.cfg) {
-			t.Errorf("test %d:\ninput:\n  %s\n wrong config, want:\n  %v\ngot:\n  %v",
-				i, test.s, test.cfg, cfg)
-			continue
-		}
+			if !reflect.DeepEqual(cfg, test.cfg) {
+				t.Fatalf("\ninput: %s\n wrong config, want:\n  %v\ngot:\n  %v",
+					test.s, test.cfg, cfg)
+			}
+		})
 	}
 }

From 518bf4e5f6712b11b9b2087972668fc2f1aa5733 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Thu, 15 Mar 2018 22:48:21 +0100
Subject: [PATCH 17/28] doc: Correct verbatim text in the manual

---
 doc/030_preparing_a_new_repo.rst | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst
index 44f8db365..c20489d82 100644
--- a/doc/030_preparing_a_new_repo.rst
+++ b/doc/030_preparing_a_new_repo.rst
@@ -139,7 +139,7 @@ If you use TLS, restic will use the system's CA certificates to verify the
 server certificate. When the verification fails, restic refuses to proceed and
 exits with an error. If you have your own self-signed certificate, or a custom
 CA certificate should be used for verification, you can pass restic the
-certificate filename via the `--cacert` option.
+certificate filename via the ``--cacert`` option.
 
 REST server uses exactly the same directory structure as local backend,
 so you should be able to access it both locally and via HTTP, even
@@ -306,8 +306,8 @@ bucket does not exist yet, it will be created:
     Please note that knowledge of your password is required to access the repository.
     Losing your password means that your data is irrecoverably lost.
 
-The number of concurrent connections to the B2 service can be set with the `-o
-b2.connections=10`. By default, at most five parallel connections are
+The number of concurrent connections to the B2 service can be set with the ``-o
+b2.connections=10``. By default, at most five parallel connections are
 established.
 
 Microsoft Azure Blob Storage
@@ -321,7 +321,7 @@ account name and key as follows:
     $ export AZURE_ACCOUNT_NAME=<ACCOUNT_NAME>
     $ export AZURE_ACCOUNT_KEY=<SECRET_KEY>
 
-Afterwards you can initialize a repository in a container called `foo` in the
+Afterwards you can initialize a repository in a container called ``foo`` in the
 root path like this:
 
 .. code-block:: console
@@ -334,7 +334,7 @@ root path like this:
     [...]
 
 The number of concurrent connections to the Azure Blob Storage service can be set with the
-`-o azure.connections=10`. By default, at most five parallel connections are
+``-o azure.connections=10``. By default, at most five parallel connections are
 established.
 
 Google Cloud Storage
@@ -369,7 +369,7 @@ located on an instance with default service accounts then these should work out
 the box.
 
 Once authenticated, you can use the ``gs:`` backend type to create a new
-repository in the bucket `foo` at the root path:
+repository in the bucket ``foo`` at the root path:
 
 .. code-block:: console
 
@@ -381,7 +381,7 @@ repository in the bucket `foo` at the root path:
     [...]
 
 The number of concurrent connections to the GCS service can be set with the
-`-o gs.connections=10`. By default, at most five parallel connections are
+``-o gs.connections=10``. By default, at most five parallel connections are
 established.
 
 .. _service account: https://cloud.google.com/storage/docs/authentication#service_accounts

From 4172fcd1679587f8a7a875de0e0755e18d5f5656 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Thu, 15 Mar 2018 22:48:30 +0100
Subject: [PATCH 18/28] doc: Add rclone backend

---
 doc/030_preparing_a_new_repo.rst | 89 ++++++++++++++++++++++++++++++++
 1 file changed, 89 insertions(+)

diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst
index c20489d82..4c0f23fad 100644
--- a/doc/030_preparing_a_new_repo.rst
+++ b/doc/030_preparing_a_new_repo.rst
@@ -387,6 +387,95 @@ established.
 .. _service account: https://cloud.google.com/storage/docs/authentication#service_accounts
 .. _create a service account key: https://cloud.google.com/storage/docs/authentication#generating-a-private-key
 
+Other Services via rclone
+*************************
+
+The program `rclone`_ can be used to access many other different services and
+store data there. First, you need to install and `configure`_ rclone. When you
+configure a remote named ``foo``, you can then call restic as follows to
+initiate a new repository in the path ``bar`` in the repo:
+
+.. code-block:: console
+
+    $ restic -r rclone:foo:bar init
+
+Restic takes care of starting and stopping rclone.
+
+As a more concrete example, suppose you have configured a remote named
+``b2prod`` for Backblaze B2 with rclone, with a bucket called ``yggdrasil``.
+You can then use rclone to list files in the bucket like this:
+
+.. code-block:: console
+
+    $ rclone ls b2prod:yggdrasil
+
+In order to create a new repository in the root directory of the bucket, call
+restic like this:
+
+.. code-block:: console
+
+    $ restic -r rclone:b2prod:yggdrasil
+
+If you want to use the path ``foo/bar/baz`` in the bucket instead, pass this to
+restic:
+
+.. code-block:: console
+
+    $ restic -r rclone:b2prod:yggdrasil/foo/bar/baz
+
+Listing the files of an empty repository directly with rclone should return a
+listing similar to the following:
+
+
+.. code-block:: console
+
+    $ rclone ls b2prod:yggdrasil/foo/bar/baz
+        155 bar/baz/config
+        448 bar/baz/keys/4bf9c78049de689d73a56ed0546f83b8416795295cda12ec7fb9465af3900b44
+
+The rclone backend has two additional options:
+
+ * ``-o rclone.program`` specifies the path to rclone, the default value is just ``rclone``
+ * ``-o rclone.args`` allows setting the arguments passed to rclone, by default this is ``serve restic --stdio``
+
+In order to start rclone, restic will build a list of arguments by joining the
+following lists (in this order): ``rclone.program``, ``rclone.args`` and as the
+last parameter the value that follows the ``rclone:`` prefix of the repository
+specification.
+
+So, calling restic like this
+
+.. code-block:: console
+
+    $ restic -o rclone.program="/path/to/rclone" \
+      -o rclone.args="serve restic --stdio --bwlimit 1M --verbose" \
+      -r rclone:b2:foo/bar
+
+runs rclone as follows:
+
+.. code-block:: console
+
+    $ /path/to/rclone serve restic --stdio --bwlimit 1M --verbose b2:foo/bar
+
+Manually setting ``rclone.program`` also allows running a remote instance of
+rclone e.g. via SSH on a server, for example:
+
+.. code-block:: console
+
+    $ restic -o rclone.program="ssh user@host rclone" -r rclone:b2:foo/bar
+
+The rclone command may also be hard-coded in the SSH configuration or the
+user's public key, in this case it may be sufficient to just start the SSH
+connection (and it's irrelevant what's passed after ``rclone:`` in the
+repository specification):
+
+.. code-block:: console
+
+    $ restic -o rclone.program="ssh user@host" -r rclone:x
+
+.. _rclone: https://rclone.org/
+.. _configure: https://rclone.org/docs/
+
 Password prompt on Windows
 **************************
 

From 362d5afec40ad47f7d3b547691f862743fc3811c Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Thu, 15 Mar 2018 22:48:39 +0100
Subject: [PATCH 19/28] Add entry to changelog

---
 changelog/unreleased/issue-1561 | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 changelog/unreleased/issue-1561

diff --git a/changelog/unreleased/issue-1561 b/changelog/unreleased/issue-1561
new file mode 100644
index 000000000..5cca7f9a5
--- /dev/null
+++ b/changelog/unreleased/issue-1561
@@ -0,0 +1,10 @@
+Enhancement: Allow using rclone to access other services
+
+We've added the ability to use rclone to store backup data on all backends that
+it supports. This was done in collaboration with Nick, the author of rclone.
+You can now use it to first configure a service, then restic manages the rest
+(starting and stopping rclone). For details, please see the manual.
+
+https://github.com/restic/restic/issues/1561
+https://github.com/restic/restic/pull/1657
+https://rclone.org

From 011217e4bfbafc4aaeb453e5bdaebbb5b3f230ea Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sat, 17 Mar 2018 13:47:44 +0100
Subject: [PATCH 20/28] backend/rclone: Improve documentation and README

---
 README.rst                       |  1 +
 doc/030_preparing_a_new_repo.rst | 14 ++++++++++++--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index efb31c1e1..d366d3d1f 100644
--- a/README.rst
+++ b/README.rst
@@ -57,6 +57,7 @@ Therefore, restic supports the following backends for storing backups natively:
 - `BackBlaze B2 <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#backblaze-b2>`__
 - `Microsoft Azure Blob Storage <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#microsoft-azure-blob-storage>`__
 - `Google Cloud Storage <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#google-cloud-storage>`__
+- And many other services via the `rclone <https://rclone.org>`__ `Backend <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone>`__
 
 Design Principles
 -----------------
diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst
index 4c0f23fad..caec81e98 100644
--- a/doc/030_preparing_a_new_repo.rst
+++ b/doc/030_preparing_a_new_repo.rst
@@ -391,7 +391,9 @@ Other Services via rclone
 *************************
 
 The program `rclone`_ can be used to access many other different services and
-store data there. First, you need to install and `configure`_ rclone. When you
+store data there. First, you need to install and `configure`_ rclone.  The
+general backend specification format is ``rclone:<remote>:<path>``, the
+``<remote>:<path>`` component will be directly passed to rclone. When you
 configure a remote named ``foo``, you can then call restic as follows to
 initiate a new repository in the path ``bar`` in the repo:
 
@@ -426,13 +428,20 @@ restic:
 Listing the files of an empty repository directly with rclone should return a
 listing similar to the following:
 
-
 .. code-block:: console
 
     $ rclone ls b2prod:yggdrasil/foo/bar/baz
         155 bar/baz/config
         448 bar/baz/keys/4bf9c78049de689d73a56ed0546f83b8416795295cda12ec7fb9465af3900b44
 
+Rclone can be `configured with environment variables`_, so for instance
+configuring a bandwidth limit for rclone cat be achieve by setting the
+``RCLONE_BWLIMIT`` environment variable:
+
+.. code-block:: console
+
+    $ export RCLONE_BWLIMIT=1M
+
 The rclone backend has two additional options:
 
  * ``-o rclone.program`` specifies the path to rclone, the default value is just ``rclone``
@@ -475,6 +484,7 @@ repository specification):
 
 .. _rclone: https://rclone.org/
 .. _configure: https://rclone.org/docs/
+.. _configured with environment variables: https://rclone.org/docs/#environment-variables
 
 Password prompt on Windows
 **************************

From 737d93860ab50dd3fd52cf24eae04d214b2826e6 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 18 Mar 2018 12:54:59 +0100
Subject: [PATCH 21/28] Extend first timeout to 60 seconds.

---
 internal/backend/rclone/backend.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go
index 67bd1eab7..9431119e0 100644
--- a/internal/backend/rclone/backend.go
+++ b/internal/backend/rclone/backend.go
@@ -171,7 +171,7 @@ func New(cfg Config) (*Backend, error) {
 	// send an HTTP request to the base URL, see if the server is there
 	client := &http.Client{
 		Transport: tr,
-		Timeout:   5 * time.Second,
+		Timeout:   60 * time.Second,
 	}
 
 	req, err := http.NewRequest(http.MethodGet, "http://localhost/", nil)

From e978b36713254a24c215a1990d72ed1ea09161a7 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 18 Mar 2018 13:04:16 +0100
Subject: [PATCH 22/28] doc: Add hint how to debug rclone

---
 doc/030_preparing_a_new_repo.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst
index caec81e98..7b1ea0f2c 100644
--- a/doc/030_preparing_a_new_repo.rst
+++ b/doc/030_preparing_a_new_repo.rst
@@ -442,6 +442,8 @@ configuring a bandwidth limit for rclone cat be achieve by setting the
 
     $ export RCLONE_BWLIMIT=1M
 
+For debugging rclone, you can set the environment variable ``RCLONE_VERBOSE=2``.
+
 The rclone backend has two additional options:
 
  * ``-o rclone.program`` specifies the path to rclone, the default value is just ``rclone``

From 1beeb7d0ddaa8f81a90095787e1c04eae0f6cba2 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 18 Mar 2018 13:26:19 +0100
Subject: [PATCH 23/28] doc/REST: Make documentation match reality

---
 doc/100_references.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/100_references.rst b/doc/100_references.rst
index 44dcc6673..be22defa0 100644
--- a/doc/100_references.rst
+++ b/doc/100_references.rst
@@ -672,8 +672,8 @@ The following values are valid for ``{type}``:
 The API version is selected via the ``Accept`` HTTP header in the request. The
 following values are defined:
 
- * ``application/vnd.x.restic.rest.v1+json`` or empty: Select API version 1
- * ``application/vnd.x.restic.rest.v2+json``: Select API version 2
+ * ``application/vnd.x.restic.rest.v1`` or empty: Select API version 1
+ * ``application/vnd.x.restic.rest.v2``: Select API version 2
 
 The server will respond with the value of the highest version it supports in
 the ``Content-Type`` HTTP response header for the HTTP requests which should

From 360ff1806a4540672e9893697ba19eb6087b7782 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 18 Mar 2018 13:47:49 +0100
Subject: [PATCH 24/28] doc: Fix instructions for rclone backend

---
 doc/030_preparing_a_new_repo.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst
index 7b1ea0f2c..a74e2563c 100644
--- a/doc/030_preparing_a_new_repo.rst
+++ b/doc/030_preparing_a_new_repo.rst
@@ -416,14 +416,14 @@ restic like this:
 
 .. code-block:: console
 
-    $ restic -r rclone:b2prod:yggdrasil
+    $ restic -r rclone:b2prod:yggdrasil init
 
 If you want to use the path ``foo/bar/baz`` in the bucket instead, pass this to
 restic:
 
 .. code-block:: console
 
-    $ restic -r rclone:b2prod:yggdrasil/foo/bar/baz
+    $ restic -r rclone:b2prod:yggdrasil/foo/bar/baz init
 
 Listing the files of an empty repository directly with rclone should return a
 listing similar to the following:

From 0b776e63e781ea03df8d20673d19aa5663d82e58 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 18 Mar 2018 20:30:02 +0100
Subject: [PATCH 25/28] backend/rclone: Request random file name

When `/` is requested, rclone returns the list of all files in the
remote, which is not what we want (and it can take quite some time).
---
 internal/backend/rclone/backend.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go
index 9431119e0..e431ec5a9 100644
--- a/internal/backend/rclone/backend.go
+++ b/internal/backend/rclone/backend.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"math/rand"
 	"net"
 	"net/http"
 	"net/url"
@@ -174,7 +175,11 @@ func New(cfg Config) (*Backend, error) {
 		Timeout:   60 * time.Second,
 	}
 
-	req, err := http.NewRequest(http.MethodGet, "http://localhost/", nil)
+	// request a random file which does not exist. we just want to test when
+	// rclone is able to accept HTTP requests.
+	url := fmt.Sprintf("http://localhost/file-%d", rand.Uint64())
+
+	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return nil, err
 	}

From c43c94776b13053afccef4341a488e68d102f700 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sat, 24 Mar 2018 18:37:36 +0100
Subject: [PATCH 26/28] rclone: Make concurrent connections configurable

---
 internal/backend/rclone/backend.go     |  2 +-
 internal/backend/rclone/config.go      | 11 +++++++----
 internal/backend/rclone/config_test.go |  3 ++-
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go
index e431ec5a9..c37576ba7 100644
--- a/internal/backend/rclone/backend.go
+++ b/internal/backend/rclone/backend.go
@@ -212,7 +212,7 @@ func Open(cfg Config) (*Backend, error) {
 	}
 
 	restConfig := rest.Config{
-		Connections: 20,
+		Connections: cfg.Connections,
 		URL:         url,
 	}
 
diff --git a/internal/backend/rclone/config.go b/internal/backend/rclone/config.go
index fa4ce88df..c9a309282 100644
--- a/internal/backend/rclone/config.go
+++ b/internal/backend/rclone/config.go
@@ -9,9 +9,10 @@ import (
 
 // Config contains all configuration necessary to start rclone.
 type Config struct {
-	Program string `option:"program" help:"path to rclone (default: rclone)"`
-	Args    string `option:"args"    help:"arguments for running rclone (default: serve restic --stdio)"`
-	Remote  string
+	Program     string `option:"program" help:"path to rclone (default: rclone)"`
+	Args        string `option:"args"    help:"arguments for running rclone (default: serve restic --stdio)"`
+	Remote      string
+	Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
 }
 
 func init() {
@@ -20,7 +21,9 @@ func init() {
 
 // NewConfig returns a new Config with the default values filled in.
 func NewConfig() Config {
-	return Config{}
+	return Config{
+		Connections: 5,
+	}
 }
 
 // ParseConfig parses the string s and extracts the remote server URL.
diff --git a/internal/backend/rclone/config_test.go b/internal/backend/rclone/config_test.go
index 05e4b2b29..a59e5fb53 100644
--- a/internal/backend/rclone/config_test.go
+++ b/internal/backend/rclone/config_test.go
@@ -13,7 +13,8 @@ func TestParseConfig(t *testing.T) {
 		{
 			"rclone:local:foo:/bar",
 			Config{
-				Remote: "local:foo:/bar",
+				Remote:      "local:foo:/bar",
+				Connections: 5,
 			},
 		},
 	}

From 86f4b0373030fefce306f712fd97b40a6c46dcef Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Fri, 30 Mar 2018 09:52:46 +0200
Subject: [PATCH 27/28] Remove unneeded byte counters

---
 internal/backend/rclone/stdio_conn.go | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/internal/backend/rclone/stdio_conn.go b/internal/backend/rclone/stdio_conn.go
index bb4928176..4abbb7c9a 100644
--- a/internal/backend/rclone/stdio_conn.go
+++ b/internal/backend/rclone/stdio_conn.go
@@ -11,22 +11,19 @@ import (
 
 // StdioConn implements a net.Conn via stdin/stdout.
 type StdioConn struct {
-	stdin                   *os.File
-	stdout                  *os.File
-	bytesWritten, bytesRead int
-	cmd                     *exec.Cmd
-	close                   sync.Once
+	stdin  *os.File
+	stdout *os.File
+	cmd    *exec.Cmd
+	close  sync.Once
 }
 
 func (s *StdioConn) Read(p []byte) (int, error) {
 	n, err := s.stdin.Read(p)
-	s.bytesRead += n
 	return n, err
 }
 
 func (s *StdioConn) Write(p []byte) (int, error) {
 	n, err := s.stdout.Write(p)
-	s.bytesWritten += n
 	return n, err
 }
 

From 3f48e0e0f4a943246590d9dd77f5f00710e299a4 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 1 Apr 2018 10:34:30 +0200
Subject: [PATCH 28/28] Add extra options to rclone

For details see https://github.com/restic/restic/pull/1657#issuecomment-377707486
---
 doc/030_preparing_a_new_repo.rst   | 10 +++++++---
 internal/backend/rclone/backend.go |  4 +++-
 internal/backend/rclone/config.go  |  2 +-
 3 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst
index a74e2563c..7d865d9a5 100644
--- a/doc/030_preparing_a_new_repo.rst
+++ b/doc/030_preparing_a_new_repo.rst
@@ -447,7 +447,10 @@ For debugging rclone, you can set the environment variable ``RCLONE_VERBOSE=2``.
 The rclone backend has two additional options:
 
  * ``-o rclone.program`` specifies the path to rclone, the default value is just ``rclone``
- * ``-o rclone.args`` allows setting the arguments passed to rclone, by default this is ``serve restic --stdio``
+ * ``-o rclone.args`` allows setting the arguments passed to rclone, by default this is ``serve restic --stdio --b2-hard-delete --drive-use-trash=false``
+
+The reason why the two last parameters (``--b2-hard-delete`` and
+``--drive-use-trash=false``) can be found in the corresponding GitHub `issue #1657`_.
 
 In order to start rclone, restic will build a list of arguments by joining the
 following lists (in this order): ``rclone.program``, ``rclone.args`` and as the
@@ -459,14 +462,14 @@ So, calling restic like this
 .. code-block:: console
 
     $ restic -o rclone.program="/path/to/rclone" \
-      -o rclone.args="serve restic --stdio --bwlimit 1M --verbose" \
+      -o rclone.args="serve restic --stdio --bwlimit 1M --b2-hard-delete --verbose" \
       -r rclone:b2:foo/bar
 
 runs rclone as follows:
 
 .. code-block:: console
 
-    $ /path/to/rclone serve restic --stdio --bwlimit 1M --verbose b2:foo/bar
+    $ /path/to/rclone serve restic --stdio --bwlimit 1M --b2-hard-delete --verbose b2:foo/bar
 
 Manually setting ``rclone.program`` also allows running a remote instance of
 rclone e.g. via SSH on a server, for example:
@@ -487,6 +490,7 @@ repository specification):
 .. _rclone: https://rclone.org/
 .. _configure: https://rclone.org/docs/
 .. _configured with environment variables: https://rclone.org/docs/#environment-variables
+.. _issue #1657: https://github.com/restic/restic/pull/1657#issuecomment-377707486
 
 Password prompt on Windows
 **************************
diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go
index c37576ba7..e19cfa6c8 100644
--- a/internal/backend/rclone/backend.go
+++ b/internal/backend/rclone/backend.go
@@ -108,7 +108,9 @@ func New(cfg Config) (*Backend, error) {
 
 		args = append(args, a...)
 	} else {
-		args = append(args, "serve", "restic", "--stdio")
+		args = append(args,
+			"serve", "restic", "--stdio",
+			"--b2-hard-delete", "--drive-use-trash=false")
 	}
 
 	// finally, add the remote
diff --git a/internal/backend/rclone/config.go b/internal/backend/rclone/config.go
index c9a309282..c2c5d88f9 100644
--- a/internal/backend/rclone/config.go
+++ b/internal/backend/rclone/config.go
@@ -10,7 +10,7 @@ import (
 // Config contains all configuration necessary to start rclone.
 type Config struct {
 	Program     string `option:"program" help:"path to rclone (default: rclone)"`
-	Args        string `option:"args"    help:"arguments for running rclone (default: serve restic --stdio)"`
+	Args        string `option:"args"    help:"arguments for running rclone (default: serve restic --stdio --b2-hard-delete --drive-use-trash=false)"`
 	Remote      string
 	Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
 }