From fe99340e408f55a270096f08f68a2f0e2a6629c3 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 13 Mar 2018 22:30:51 +0100 Subject: [PATCH] Add rclone backend --- cmd/restic/global.go | 13 ++ internal/backend/foreground_windows.go | 2 +- internal/backend/location/location.go | 2 + internal/backend/rclone/backend.go | 225 ++++++++++++++++++++ internal/backend/rclone/backend_test.go | 83 ++++++++ internal/backend/rclone/config.go | 36 ++++ internal/backend/rclone/config_test.go | 33 +++ internal/backend/rclone/stdio_conn.go | 72 +++++++ internal/backend/rclone/stdio_conn_go110.go | 25 +++ internal/backend/rclone/stdio_conn_other.go | 22 ++ 10 files changed, 512 insertions(+), 1 deletion(-) create mode 100644 internal/backend/rclone/backend.go create mode 100644 internal/backend/rclone/backend_test.go create mode 100644 internal/backend/rclone/config.go create mode 100644 internal/backend/rclone/config_test.go create mode 100644 internal/backend/rclone/stdio_conn.go create mode 100644 internal/backend/rclone/stdio_conn_go110.go create mode 100644 internal/backend/rclone/stdio_conn_other.go diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 5d8e5110f..0c3d805b2 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -18,6 +18,7 @@ import ( "github.com/restic/restic/internal/backend/gs" "github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/rclone" "github.com/restic/restic/internal/backend/rest" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/sftp" @@ -509,6 +510,14 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro return nil, err } + debug.Log("opening rest repository at %#v", cfg) + return cfg, nil + case "rclone": + cfg := loc.Config.(rclone.Config) + if err := opts.Apply(loc.Scheme, &cfg); err != nil { + return nil, err + } + debug.Log("opening rest repository at %#v", cfg) return cfg, nil } @@ -564,6 +573,8 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt) case "rest": be, err = rest.Open(cfg.(rest.Config), rt) + case "rclone": + be, err = rclone.Open(cfg.(rclone.Config)) default: return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) @@ -625,6 +636,8 @@ func create(s string, opts options.Options) (restic.Backend, error) { return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt) case "rest": return rest.Create(cfg.(rest.Config), rt) + case "rclone": + return rclone.Open(cfg.(rclone.Config)) } debug.Log("invalid repository scheme: %v", s) diff --git a/internal/backend/foreground_windows.go b/internal/backend/foreground_windows.go index 0e8b03a13..281d5f3f8 100644 --- a/internal/backend/foreground_windows.go +++ b/internal/backend/foreground_windows.go @@ -9,7 +9,7 @@ import ( // StartForeground runs cmd in the foreground, by temporarily switching to the // new process group created for cmd. The returned function `bg` switches back // to the previous process group. -func startForeground(cmd *exec.Cmd) (bg func() error, err error) { +func StartForeground(cmd *exec.Cmd) (bg func() error, err error) { // just start the process and hope for the best err = cmd.Start() if err != nil { diff --git a/internal/backend/location/location.go b/internal/backend/location/location.go index cc547c60b..68f4dfaff 100644 --- a/internal/backend/location/location.go +++ b/internal/backend/location/location.go @@ -8,6 +8,7 @@ import ( "github.com/restic/restic/internal/backend/b2" "github.com/restic/restic/internal/backend/gs" "github.com/restic/restic/internal/backend/local" + "github.com/restic/restic/internal/backend/rclone" "github.com/restic/restic/internal/backend/rest" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/sftp" @@ -38,6 +39,7 @@ var parsers = []parser{ {"azure", azure.ParseConfig}, {"swift", swift.ParseConfig}, {"rest", rest.ParseConfig}, + {"rclone", rclone.ParseConfig}, } func isPath(s string) bool { diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go new file mode 100644 index 000000000..e2e538cb4 --- /dev/null +++ b/internal/backend/rclone/backend.go @@ -0,0 +1,225 @@ +package rclone + +import ( + "context" + "crypto/tls" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "time" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/rest" + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "golang.org/x/net/context/ctxhttp" + "golang.org/x/net/http2" +) + +// Backend is used to access data stored somewhere via rclone. +type Backend struct { + *rest.Backend + tr *http2.Transport + cmd *exec.Cmd + waitCh <-chan struct{} + waitResult error +} + +// run starts command with args and initializes the StdioConn. +func run(command string, args ...string) (*StdioConn, *exec.Cmd, func() error, error) { + cmd := exec.Command(command, args...) + cmd.Stderr = os.Stderr + + r, stdin, err := os.Pipe() + if err != nil { + return nil, nil, nil, err + } + + stdout, w, err := os.Pipe() + if err != nil { + return nil, nil, nil, err + } + + cmd.Stdin = r + cmd.Stdout = w + + bg, err := backend.StartForeground(cmd) + if err != nil { + return nil, nil, nil, err + } + + c := &StdioConn{ + stdin: stdout, + stdout: stdin, + cmd: cmd, + } + + return c, cmd, bg, nil +} + +// New initializes a Backend and starts the process. +func New(cfg Config) (*Backend, error) { + var ( + args []string + err error + ) + + // build program args, start with the program + if cfg.Program != "" { + a, err := backend.SplitShellStrings(cfg.Program) + if err != nil { + return nil, err + } + args = append(args, a...) + } else { + args = append(args, "rclone") + } + + // then add the arguments + if cfg.Args != "" { + a, err := backend.SplitShellStrings(cfg.Args) + if err != nil { + return nil, err + } + + args = append(args, a...) + } else { + args = append(args, "serve", "restic", "--stdio") + } + + // finally, add the remote + args = append(args, cfg.Remote) + arg0, args := args[0], args[1:] + + debug.Log("running command: %v %v", arg0, args) + conn, cmd, bg, err := run(arg0, args...) + if err != nil { + return nil, err + } + + tr := &http2.Transport{ + AllowHTTP: true, // this is not really HTTP, just stdin/stdout + DialTLS: func(network, address string, cfg *tls.Config) (net.Conn, error) { + debug.Log("new connection requested, %v %v", network, address) + return conn, nil + }, + } + + waitCh := make(chan struct{}) + be := &Backend{ + tr: tr, + cmd: cmd, + waitCh: waitCh, + } + + go func() { + debug.Log("waiting for error result") + err := cmd.Wait() + debug.Log("Wait returned %v", err) + be.waitResult = err + close(waitCh) + }() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + debug.Log("monitoring command to cancel first HTTP request context") + select { + case <-ctx.Done(): + debug.Log("context has been cancelled, returning") + case <-be.waitCh: + debug.Log("command has exited, cancelling context") + cancel() + } + }() + + // send an HTTP request to the base URL, see if the server is there + client := &http.Client{ + Transport: tr, + Timeout: 5 * time.Second, + } + + req, err := http.NewRequest(http.MethodGet, "http://localhost/", nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", rest.ContentTypeV2) + + res, err := ctxhttp.Do(ctx, client, req) + if err != nil { + bg() + _ = cmd.Process.Kill() + return nil, errors.Errorf("error talking HTTP to rclone: %v", err) + } + + debug.Log("HTTP status %q returned, moving instance to background", res.Status) + bg() + + return be, nil +} + +// Open starts an rclone process with the given config. +func Open(cfg Config) (*Backend, error) { + be, err := New(cfg) + if err != nil { + return nil, err + } + + url, err := url.Parse("http://localhost/") + if err != nil { + return nil, err + } + + restConfig := rest.Config{ + Connections: 20, + URL: url, + } + + restBackend, err := rest.Open(restConfig, be.tr) + if err != nil { + return nil, err + } + + be.Backend = restBackend + return be, nil +} + +// Create initializes a new restic repo with clone. +func Create(cfg Config) (*Backend, error) { + be, err := New(cfg) + if err != nil { + return nil, err + } + + debug.Log("new backend created") + + url, err := url.Parse("http://localhost/") + if err != nil { + return nil, err + } + + restConfig := rest.Config{ + Connections: 20, + URL: url, + } + + restBackend, err := rest.Create(restConfig, be.tr) + if err != nil { + return nil, err + } + + be.Backend = restBackend + return be, nil +} + +// Close terminates the backend. +func (be *Backend) Close() error { + debug.Log("exting rclone") + be.tr.CloseIdleConnections() + <-be.waitCh + debug.Log("wait for rclone returned: %v", be.waitResult) + return be.waitResult +} diff --git a/internal/backend/rclone/backend_test.go b/internal/backend/rclone/backend_test.go new file mode 100644 index 000000000..ab8eb8376 --- /dev/null +++ b/internal/backend/rclone/backend_test.go @@ -0,0 +1,83 @@ +package rclone_test + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/restic/restic/internal/backend/rclone" + "github.com/restic/restic/internal/backend/test" + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" +) + +const rcloneConfig = ` +[local] +type = local +` + +func newTestSuite(t testing.TB) *test.Suite { + dir, cleanup := rtest.TempDir(t) + + return &test.Suite{ + // NewConfig returns a config for a new temporary backend that will be used in tests. + NewConfig: func() (interface{}, error) { + cfgfile := filepath.Join(dir, "rclone.conf") + t.Logf("write rclone config to %v", cfgfile) + err := ioutil.WriteFile(cfgfile, []byte(rcloneConfig), 0644) + if err != nil { + return nil, err + } + + t.Logf("use backend at %v", dir) + + repodir := filepath.Join(dir, "repo") + err = os.Mkdir(repodir, 0755) + if err != nil { + return nil, err + } + + cfg := rclone.NewConfig() + cfg.Program = fmt.Sprintf("rclone --config %q", cfgfile) + cfg.Remote = "local:" + repodir + return cfg, nil + }, + + // CreateFn is a function that creates a temporary repository for the tests. + Create: func(config interface{}) (restic.Backend, error) { + t.Logf("Create()") + cfg := config.(rclone.Config) + return rclone.Create(cfg) + }, + + // OpenFn is a function that opens a previously created temporary repository. + Open: func(config interface{}) (restic.Backend, error) { + t.Logf("Open()") + cfg := config.(rclone.Config) + return rclone.Open(cfg) + }, + + // CleanupFn removes data created during the tests. + Cleanup: func(config interface{}) error { + t.Logf("cleanup dir %v", dir) + cleanup() + return nil + }, + } +} + +func TestBackendRclone(t *testing.T) { + defer func() { + if t.Skipped() { + rtest.SkipDisallowed(t, "restic/backend/rclone.TestBackendRclone") + } + }() + + newTestSuite(t).RunTests(t) +} + +func BenchmarkBackendREST(t *testing.B) { + newTestSuite(t).RunBenchmarks(t) +} diff --git a/internal/backend/rclone/config.go b/internal/backend/rclone/config.go new file mode 100644 index 000000000..a833d153b --- /dev/null +++ b/internal/backend/rclone/config.go @@ -0,0 +1,36 @@ +package rclone + +import ( + "strings" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/options" +) + +// Config contains all configuration necessary to start rclone. +type Config struct { + Program string `option:"program" help:"path to rclone (default: rclone)"` + Args string `option:"args" help:"arguments for running rclone (default: restic serve --stdio)"` + Remote string +} + +func init() { + options.Register("rclone", Config{}) +} + +// NewConfig returns a new Config with the default values filled in. +func NewConfig() Config { + return Config{} +} + +// ParseConfig parses the string s and extracts the remote server URL. +func ParseConfig(s string) (interface{}, error) { + if !strings.HasPrefix(s, "rclone:") { + return nil, errors.New("invalid rclone backend specification") + } + + s = s[7:] + cfg := NewConfig() + cfg.Remote = s + return cfg, nil +} diff --git a/internal/backend/rclone/config_test.go b/internal/backend/rclone/config_test.go new file mode 100644 index 000000000..05e4b2b29 --- /dev/null +++ b/internal/backend/rclone/config_test.go @@ -0,0 +1,33 @@ +package rclone + +import ( + "reflect" + "testing" +) + +func TestParseConfig(t *testing.T) { + var tests = []struct { + s string + cfg Config + }{ + { + "rclone:local:foo:/bar", + Config{ + Remote: "local:foo:/bar", + }, + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + cfg, err := ParseConfig(test.s) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(cfg, test.cfg) { + t.Fatalf("wrong config, want:\n %v\ngot:\n %v", test.cfg, cfg) + } + }) + } +} diff --git a/internal/backend/rclone/stdio_conn.go b/internal/backend/rclone/stdio_conn.go new file mode 100644 index 000000000..4472300ec --- /dev/null +++ b/internal/backend/rclone/stdio_conn.go @@ -0,0 +1,72 @@ +package rclone + +import ( + "net" + "os" + "os/exec" + + "github.com/restic/restic/internal/debug" +) + +// StdioConn implements a net.Conn via stdin/stdout. +type StdioConn struct { + stdin *os.File + stdout *os.File + bytesWritten, bytesRead int + cmd *exec.Cmd +} + +func (s *StdioConn) Read(p []byte) (int, error) { + n, err := s.stdin.Read(p) + s.bytesRead += n + return n, err +} + +func (s *StdioConn) Write(p []byte) (int, error) { + n, err := s.stdout.Write(p) + s.bytesWritten += n + return n, err +} + +// Close closes both streams. +func (s *StdioConn) Close() error { + debug.Log("close server instance") + var errs []error + + for _, f := range []func() error{s.stdin.Close, s.stdout.Close} { + err := f() + if err != nil { + errs = append(errs, err) + } + } + + if len(errs) > 0 { + return errs[0] + } + return nil +} + +// LocalAddr returns nil. +func (s *StdioConn) LocalAddr() net.Addr { + return Addr{} +} + +// RemoteAddr returns nil. +func (s *StdioConn) RemoteAddr() net.Addr { + return Addr{} +} + +// make sure StdioConn implements net.Conn +var _ net.Conn = &StdioConn{} + +// Addr implements net.Addr for stdin/stdout. +type Addr struct{} + +// Network returns the network type as a string. +func (a Addr) Network() string { + return "stdio" +} + +func (a Addr) String() string { + return "stdio" +} diff --git a/internal/backend/rclone/stdio_conn_go110.go b/internal/backend/rclone/stdio_conn_go110.go new file mode 100644 index 000000000..b21f65f04 --- /dev/null +++ b/internal/backend/rclone/stdio_conn_go110.go @@ -0,0 +1,25 @@ +// +build go1.10 + +package rclone + +import "time" + +// SetDeadline sets the read/write deadline. +func (s *StdioConn) SetDeadline(t time.Time) error { + err1 := s.stdin.SetReadDeadline(t) + err2 := s.stdout.SetWriteDeadline(t) + if err1 != nil { + return err1 + } + return err2 +} + +// SetReadDeadline sets the read/write deadline. +func (s *StdioConn) SetReadDeadline(t time.Time) error { + return s.stdin.SetReadDeadline(t) +} + +// SetWriteDeadline sets the read/write deadline. +func (s *StdioConn) SetWriteDeadline(t time.Time) error { + return s.stdout.SetWriteDeadline(t) +} diff --git a/internal/backend/rclone/stdio_conn_other.go b/internal/backend/rclone/stdio_conn_other.go new file mode 100644 index 000000000..07f85961b --- /dev/null +++ b/internal/backend/rclone/stdio_conn_other.go @@ -0,0 +1,22 @@ +// +build !go1.10 + +package rclone + +import "time" + +// On Go < 1.10, it's not possible to set read/write deadlines on files, so we just ignore that. + +// SetDeadline sets the read/write deadline. +func (s *StdioConn) SetDeadline(t time.Time) error { + return nil +} + +// SetReadDeadline sets the read/write deadline. +func (s *StdioConn) SetReadDeadline(t time.Time) error { + return nil +} + +// SetWriteDeadline sets the read/write deadline. +func (s *StdioConn) SetWriteDeadline(t time.Time) error { + return nil +}