Add rclone backend

cmd/restic/global.go

@@ -18,6 +18,7 @@ import (
 	"github.com/restic/restic/internal/backend/gs"
 	"github.com/restic/restic/internal/backend/local"
 	"github.com/restic/restic/internal/backend/location"
+	"github.com/restic/restic/internal/backend/rclone"
 	"github.com/restic/restic/internal/backend/rest"
 	"github.com/restic/restic/internal/backend/s3"
 	"github.com/restic/restic/internal/backend/sftp"
@@ -509,6 +510,14 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 			return nil, err
 		}
 
+		debug.Log("opening rest repository at %#v", cfg)
+		return cfg, nil
+	case "rclone":
+		cfg := loc.Config.(rclone.Config)
+		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
+			return nil, err
+		}
+
 		debug.Log("opening rest repository at %#v", cfg)
 		return cfg, nil
 	}
@@ -564,6 +573,8 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 		be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt)
 	case "rest":
 		be, err = rest.Open(cfg.(rest.Config), rt)
+	case "rclone":
+		be, err = rclone.Open(cfg.(rclone.Config))
 
 	default:
 		return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
@@ -625,6 +636,8 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 		return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt)
 	case "rest":
 		return rest.Create(cfg.(rest.Config), rt)
+	case "rclone":
+		return rclone.Open(cfg.(rclone.Config))
 	}
 
 	debug.Log("invalid repository scheme: %v", s)

internal/backend/foreground_windows.go

@@ -9,7 +9,7 @@ import (
 // StartForeground runs cmd in the foreground, by temporarily switching to the
 // new process group created for cmd. The returned function `bg` switches back
 // to the previous process group.
-func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
+func StartForeground(cmd *exec.Cmd) (bg func() error, err error) {
 	// just start the process and hope for the best
 	err = cmd.Start()
 	if err != nil {

internal/backend/location/location.go

@@ -8,6 +8,7 @@ import (
 	"github.com/restic/restic/internal/backend/b2"
 	"github.com/restic/restic/internal/backend/gs"
 	"github.com/restic/restic/internal/backend/local"
+	"github.com/restic/restic/internal/backend/rclone"
 	"github.com/restic/restic/internal/backend/rest"
 	"github.com/restic/restic/internal/backend/s3"
 	"github.com/restic/restic/internal/backend/sftp"
@@ -38,6 +39,7 @@ var parsers = []parser{
 	{"azure", azure.ParseConfig},
 	{"swift", swift.ParseConfig},
 	{"rest", rest.ParseConfig},
+	{"rclone", rclone.ParseConfig},
 }
 
 func isPath(s string) bool {

internal/backend/rclone/backend.go

@@ -0,0 +1,225 @@
+package rclone
+
+import (
+	"context"
+	"crypto/tls"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"time"
+
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/backend/rest"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
+	"golang.org/x/net/context/ctxhttp"
+	"golang.org/x/net/http2"
+)
+
+// Backend is used to access data stored somewhere via rclone.
+type Backend struct {
+	*rest.Backend
+	tr         *http2.Transport
+	cmd        *exec.Cmd
+	waitCh     <-chan struct{}
+	waitResult error
+}
+
+// run starts command with args and initializes the StdioConn.
+func run(command string, args ...string) (*StdioConn, *exec.Cmd, func() error, error) {
+	cmd := exec.Command(command, args...)
+	cmd.Stderr = os.Stderr
+
+	r, stdin, err := os.Pipe()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	stdout, w, err := os.Pipe()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	cmd.Stdin = r
+	cmd.Stdout = w
+
+	bg, err := backend.StartForeground(cmd)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	c := &StdioConn{
+		stdin:  stdout,
+		stdout: stdin,
+		cmd:    cmd,
+	}
+
+	return c, cmd, bg, nil
+}
+
+// New initializes a Backend and starts the process.
+func New(cfg Config) (*Backend, error) {
+	var (
+		args []string
+		err  error
+	)
+
+	// build program args, start with the program
+	if cfg.Program != "" {
+		a, err := backend.SplitShellStrings(cfg.Program)
+		if err != nil {
+			return nil, err
+		}
+		args = append(args, a...)
+	} else {
+		args = append(args, "rclone")
+	}
+
+	// then add the arguments
+	if cfg.Args != "" {
+		a, err := backend.SplitShellStrings(cfg.Args)
+		if err != nil {
+			return nil, err
+		}
+
+		args = append(args, a...)
+	} else {
+		args = append(args, "serve", "restic", "--stdio")
+	}
+
+	// finally, add the remote
+	args = append(args, cfg.Remote)
+	arg0, args := args[0], args[1:]
+
+	debug.Log("running command: %v %v", arg0, args)
+	conn, cmd, bg, err := run(arg0, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	tr := &http2.Transport{
+		AllowHTTP: true, // this is not really HTTP, just stdin/stdout
+		DialTLS: func(network, address string, cfg *tls.Config) (net.Conn, error) {
+			debug.Log("new connection requested, %v %v", network, address)
+			return conn, nil
+		},
+	}
+
+	waitCh := make(chan struct{})
+	be := &Backend{
+		tr:     tr,
+		cmd:    cmd,
+		waitCh: waitCh,
+	}
+
+	go func() {
+		debug.Log("waiting for error result")
+		err := cmd.Wait()
+		debug.Log("Wait returned %v", err)
+		be.waitResult = err
+		close(waitCh)
+	}()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	go func() {
+		debug.Log("monitoring command to cancel first HTTP request context")
+		select {
+		case <-ctx.Done():
+			debug.Log("context has been cancelled, returning")
+		case <-be.waitCh:
+			debug.Log("command has exited, cancelling context")
+			cancel()
+		}
+	}()
+
+	// send an HTTP request to the base URL, see if the server is there
+	client := &http.Client{
+		Transport: tr,
+		Timeout:   5 * time.Second,
+	}
+
+	req, err := http.NewRequest(http.MethodGet, "http://localhost/", nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept", rest.ContentTypeV2)
+
+	res, err := ctxhttp.Do(ctx, client, req)
+	if err != nil {
+		bg()
+		_ = cmd.Process.Kill()
+		return nil, errors.Errorf("error talking HTTP to rclone: %v", err)
+	}
+
+	debug.Log("HTTP status %q returned, moving instance to background", res.Status)
+	bg()
+
+	return be, nil
+}
+
+// Open starts an rclone process with the given config.
+func Open(cfg Config) (*Backend, error) {
+	be, err := New(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	url, err := url.Parse("http://localhost/")
+	if err != nil {
+		return nil, err
+	}
+
+	restConfig := rest.Config{
+		Connections: 20,
+		URL:         url,
+	}
+
+	restBackend, err := rest.Open(restConfig, be.tr)
+	if err != nil {
+		return nil, err
+	}
+
+	be.Backend = restBackend
+	return be, nil
+}
+
+// Create initializes a new restic repo with clone.
+func Create(cfg Config) (*Backend, error) {
+	be, err := New(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	debug.Log("new backend created")
+
+	url, err := url.Parse("http://localhost/")
+	if err != nil {
+		return nil, err
+	}
+
+	restConfig := rest.Config{
+		Connections: 20,
+		URL:         url,
+	}
+
+	restBackend, err := rest.Create(restConfig, be.tr)
+	if err != nil {
+		return nil, err
+	}
+
+	be.Backend = restBackend
+	return be, nil
+}
+
+// Close terminates the backend.
+func (be *Backend) Close() error {
+	debug.Log("exting rclone")
+	be.tr.CloseIdleConnections()
+	<-be.waitCh
+	debug.Log("wait for rclone returned: %v", be.waitResult)
+	return be.waitResult
+}

internal/backend/rclone/backend_test.go

@@ -0,0 +1,83 @@
+package rclone_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/restic/restic/internal/backend/rclone"
+	"github.com/restic/restic/internal/backend/test"
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+const rcloneConfig = `
+[local]
+type = local
+`
+
+func newTestSuite(t testing.TB) *test.Suite {
+	dir, cleanup := rtest.TempDir(t)
+
+	return &test.Suite{
+		// NewConfig returns a config for a new temporary backend that will be used in tests.
+		NewConfig: func() (interface{}, error) {
+			cfgfile := filepath.Join(dir, "rclone.conf")
+			t.Logf("write rclone config to %v", cfgfile)
+			err := ioutil.WriteFile(cfgfile, []byte(rcloneConfig), 0644)
+			if err != nil {
+				return nil, err
+			}
+
+			t.Logf("use backend at %v", dir)
+
+			repodir := filepath.Join(dir, "repo")
+			err = os.Mkdir(repodir, 0755)
+			if err != nil {
+				return nil, err
+			}
+
+			cfg := rclone.NewConfig()
+			cfg.Program = fmt.Sprintf("rclone --config %q", cfgfile)
+			cfg.Remote = "local:" + repodir
+			return cfg, nil
+		},
+
+		// CreateFn is a function that creates a temporary repository for the tests.
+		Create: func(config interface{}) (restic.Backend, error) {
+			t.Logf("Create()")
+			cfg := config.(rclone.Config)
+			return rclone.Create(cfg)
+		},
+
+		// OpenFn is a function that opens a previously created temporary repository.
+		Open: func(config interface{}) (restic.Backend, error) {
+			t.Logf("Open()")
+			cfg := config.(rclone.Config)
+			return rclone.Open(cfg)
+		},
+
+		// CleanupFn removes data created during the tests.
+		Cleanup: func(config interface{}) error {
+			t.Logf("cleanup dir %v", dir)
+			cleanup()
+			return nil
+		},
+	}
+}
+
+func TestBackendRclone(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			rtest.SkipDisallowed(t, "restic/backend/rclone.TestBackendRclone")
+		}
+	}()
+
+	newTestSuite(t).RunTests(t)
+}
+
+func BenchmarkBackendREST(t *testing.B) {
+	newTestSuite(t).RunBenchmarks(t)
+}

internal/backend/rclone/config.go

@@ -0,0 +1,36 @@
+package rclone
+
+import (
+	"strings"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/options"
+)
+
+// Config contains all configuration necessary to start rclone.
+type Config struct {
+	Program string `option:"program" help:"path to rclone (default: rclone)"`
+	Args    string `option:"args"    help:"arguments for running rclone (default: restic serve --stdio)"`
+	Remote  string
+}
+
+func init() {
+	options.Register("rclone", Config{})
+}
+
+// NewConfig returns a new Config with the default values filled in.
+func NewConfig() Config {
+	return Config{}
+}
+
+// ParseConfig parses the string s and extracts the remote server URL.
+func ParseConfig(s string) (interface{}, error) {
+	if !strings.HasPrefix(s, "rclone:") {
+		return nil, errors.New("invalid rclone backend specification")
+	}
+
+	s = s[7:]
+	cfg := NewConfig()
+	cfg.Remote = s
+	return cfg, nil
+}

internal/backend/rclone/config_test.go

@@ -0,0 +1,33 @@
+package rclone
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseConfig(t *testing.T) {
+	var tests = []struct {
+		s   string
+		cfg Config
+	}{
+		{
+			"rclone:local:foo:/bar",
+			Config{
+				Remote: "local:foo:/bar",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run("", func(t *testing.T) {
+			cfg, err := ParseConfig(test.s)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if !reflect.DeepEqual(cfg, test.cfg) {
+				t.Fatalf("wrong config, want:\n  %v\ngot:\n  %v", test.cfg, cfg)
+			}
+		})
+	}
+}

internal/backend/rclone/stdio_conn.go

@@ -0,0 +1,72 @@
+package rclone
+
+import (
+	"net"
+	"os"
+	"os/exec"
+
+	"github.com/restic/restic/internal/debug"
+)
+
+// StdioConn implements a net.Conn via stdin/stdout.
+type StdioConn struct {
+	stdin                   *os.File
+	stdout                  *os.File
+	bytesWritten, bytesRead int
+	cmd                     *exec.Cmd
+}
+
+func (s *StdioConn) Read(p []byte) (int, error) {
+	n, err := s.stdin.Read(p)
+	s.bytesRead += n
+	return n, err
+}
+
+func (s *StdioConn) Write(p []byte) (int, error) {
+	n, err := s.stdout.Write(p)
+	s.bytesWritten += n
+	return n, err
+}
+
+// Close closes both streams.
+func (s *StdioConn) Close() error {
+	debug.Log("close server instance")
+	var errs []error
+
+	for _, f := range []func() error{s.stdin.Close, s.stdout.Close} {
+		err := f()
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	if len(errs) > 0 {
+		return errs[0]
+	}
+	return nil
+}
+
+// LocalAddr returns nil.
+func (s *StdioConn) LocalAddr() net.Addr {
+	return Addr{}
+}
+
+// RemoteAddr returns nil.
+func (s *StdioConn) RemoteAddr() net.Addr {
+	return Addr{}
+}
+
+// make sure StdioConn implements net.Conn
+var _ net.Conn = &StdioConn{}
+
+// Addr implements net.Addr for stdin/stdout.
+type Addr struct{}
+
+// Network returns the network type as a string.
+func (a Addr) Network() string {
+	return "stdio"
+}
+
+func (a Addr) String() string {
+	return "stdio"
+}

internal/backend/rclone/stdio_conn_go110.go

@@ -0,0 +1,25 @@
+// +build go1.10
+
+package rclone
+
+import "time"
+
+// SetDeadline sets the read/write deadline.
+func (s *StdioConn) SetDeadline(t time.Time) error {
+	err1 := s.stdin.SetReadDeadline(t)
+	err2 := s.stdout.SetWriteDeadline(t)
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+// SetReadDeadline sets the read/write deadline.
+func (s *StdioConn) SetReadDeadline(t time.Time) error {
+	return s.stdin.SetReadDeadline(t)
+}
+
+// SetWriteDeadline sets the read/write deadline.
+func (s *StdioConn) SetWriteDeadline(t time.Time) error {
+	return s.stdout.SetWriteDeadline(t)
+}

internal/backend/rclone/stdio_conn_other.go

@@ -0,0 +1,22 @@
+// +build !go1.10
+
+package rclone
+
+import "time"
+
+// On Go < 1.10, it's not possible to set read/write deadlines on files, so we just ignore that.
+
+// SetDeadline sets the read/write deadline.
+func (s *StdioConn) SetDeadline(t time.Time) error {
+	return nil
+}
+
+// SetReadDeadline sets the read/write deadline.
+func (s *StdioConn) SetReadDeadline(t time.Time) error {
+	return nil
+}
+
+// SetWriteDeadline sets the read/write deadline.
+func (s *StdioConn) SetWriteDeadline(t time.Time) error {
+	return nil
+}