octoleo
/
restic
mirror of https://github.com/octoleo/restic.git
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #1657 from restic/rclone-backend 

Rclone backend
This commit is contained in:
Alexander Neumann 2018-04-01 10:56:10 +02:00
parent e4a39e02d2 3f48e0e0f4
commit b077a1227b
25 changed files with 786 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Gopkg.lock generated
View File

@ -232,6 +232,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "a7d099b3ce195ffc37adedb05a4386be38e6158925a1c0fe579efdc20fa11f6a"
inputs-digest = "d3d59414a33bb8ecc6d88a681c782a87244a565cc9d0f85615cfa0704c02800a"
solver-name = "gps-cdcl"
solver-version = 1

1
README.rst
View File

@ -57,6 +57,7 @@ Therefore, restic supports the following backends for storing backups natively:
- `BackBlaze B2 <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#backblaze-b2>`__
- `Microsoft Azure Blob Storage <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#microsoft-azure-blob-storage>`__
- `Google Cloud Storage <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#google-cloud-storage>`__
- And many other services via the `rclone <https://rclone.org>`__ `Backend <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone>`__
Design Principles
-----------------

10
changelog/unreleased/issue-1561 Normal file
View File

@ -0,0 +1,10 @@
Enhancement: Allow using rclone to access other services
We've added the ability to use rclone to store backup data on all backends that
it supports. This was done in collaboration with Nick, the author of rclone.
You can now use it to first configure a service, then restic manages the rest
(starting and stopping rclone). For details, please see the manual.
https://github.com/restic/restic/issues/1561
https://github.com/restic/restic/pull/1657
https://rclone.org

13
cmd/restic/global.go
View File

@ -18,6 +18,7 @@ import (
"github.com/restic/restic/internal/backend/gs"
"github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/rclone"
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/backend/sftp"
@ -509,6 +510,14 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
return nil, err
}
debug.Log("opening rest repository at %#v", cfg)
return cfg, nil
case "rclone":
cfg := loc.Config.(rclone.Config)
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
return nil, err
}
debug.Log("opening rest repository at %#v", cfg)
return cfg, nil
}
@ -564,6 +573,8 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt)
case "rest":
be, err = rest.Open(cfg.(rest.Config), rt)
case "rclone":
be, err = rclone.Open(cfg.(rclone.Config))
default:
return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
@ -625,6 +636,8 @@ func create(s string, opts options.Options) (restic.Backend, error) {
return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt)
case "rest":
return rest.Create(cfg.(rest.Config), rt)
case "rclone":
return rclone.Open(cfg.(rclone.Config))
}
debug.Log("invalid repository scheme: %v", s)

119
doc/030_preparing_a_new_repo.rst
View File

@ -139,7 +139,7 @@ If you use TLS, restic will use the system's CA certificates to verify the
server certificate. When the verification fails, restic refuses to proceed and
exits with an error. If you have your own self-signed certificate, or a custom
CA certificate should be used for verification, you can pass restic the
certificate filename via the `--cacert` option.
certificate filename via the ``--cacert`` option.
REST server uses exactly the same directory structure as local backend,
so you should be able to access it both locally and via HTTP, even
@ -306,8 +306,8 @@ bucket does not exist yet, it will be created:
Please note that knowledge of your password is required to access the repository.
Losing your password means that your data is irrecoverably lost.
The number of concurrent connections to the B2 service can be set with the `-o
b2.connections=10`. By default, at most five parallel connections are
The number of concurrent connections to the B2 service can be set with the ``-o
b2.connections=10``. By default, at most five parallel connections are
established.
Microsoft Azure Blob Storage
@ -321,7 +321,7 @@ account name and key as follows:
$ export AZURE_ACCOUNT_NAME=<ACCOUNT_NAME>
$ export AZURE_ACCOUNT_KEY=<SECRET_KEY>
Afterwards you can initialize a repository in a container called `foo` in the
Afterwards you can initialize a repository in a container called ``foo`` in the
root path like this:
.. code-block:: console
@ -334,7 +334,7 @@ root path like this:
[...]
The number of concurrent connections to the Azure Blob Storage service can be set with the
`-o azure.connections=10`. By default, at most five parallel connections are
``-o azure.connections=10``. By default, at most five parallel connections are
established.
Google Cloud Storage
@ -369,7 +369,7 @@ located on an instance with default service accounts then these should work out
the box.
Once authenticated, you can use the ``gs:`` backend type to create a new
repository in the bucket `foo` at the root path:
repository in the bucket ``foo`` at the root path:
.. code-block:: console
@ -381,12 +381,117 @@ repository in the bucket `foo` at the root path:
[...]
The number of concurrent connections to the GCS service can be set with the
`-o gs.connections=10`. By default, at most five parallel connections are
``-o gs.connections=10``. By default, at most five parallel connections are
established.
.. _service account: https://cloud.google.com/storage/docs/authentication#service_accounts
.. _create a service account key: https://cloud.google.com/storage/docs/authentication#generating-a-private-key
Other Services via rclone
*************************
The program `rclone`_ can be used to access many other different services and
store data there. First, you need to install and `configure`_ rclone. The
general backend specification format is ``rclone:<remote>:<path>``, the
``<remote>:<path>`` component will be directly passed to rclone. When you
configure a remote named ``foo``, you can then call restic as follows to
initiate a new repository in the path ``bar`` in the repo:
.. code-block:: console
$ restic -r rclone:foo:bar init
Restic takes care of starting and stopping rclone.
As a more concrete example, suppose you have configured a remote named
``b2prod`` for Backblaze B2 with rclone, with a bucket called ``yggdrasil``.
You can then use rclone to list files in the bucket like this:
.. code-block:: console
$ rclone ls b2prod:yggdrasil
In order to create a new repository in the root directory of the bucket, call
restic like this:
.. code-block:: console
$ restic -r rclone:b2prod:yggdrasil init
If you want to use the path ``foo/bar/baz`` in the bucket instead, pass this to
restic:
.. code-block:: console
$ restic -r rclone:b2prod:yggdrasil/foo/bar/baz init
Listing the files of an empty repository directly with rclone should return a
listing similar to the following:
.. code-block:: console
$ rclone ls b2prod:yggdrasil/foo/bar/baz
155 bar/baz/config
448 bar/baz/keys/4bf9c78049de689d73a56ed0546f83b8416795295cda12ec7fb9465af3900b44
Rclone can be `configured with environment variables`_, so for instance
configuring a bandwidth limit for rclone cat be achieve by setting the
``RCLONE_BWLIMIT`` environment variable:
.. code-block:: console
$ export RCLONE_BWLIMIT=1M
For debugging rclone, you can set the environment variable ``RCLONE_VERBOSE=2``.
The rclone backend has two additional options:
* ``-o rclone.program`` specifies the path to rclone, the default value is just ``rclone``
* ``-o rclone.args`` allows setting the arguments passed to rclone, by default this is ``serve restic --stdio --b2-hard-delete --drive-use-trash=false``
The reason why the two last parameters (``--b2-hard-delete`` and
``--drive-use-trash=false``) can be found in the corresponding GitHub `issue #1657`_.
In order to start rclone, restic will build a list of arguments by joining the
following lists (in this order): ``rclone.program``, ``rclone.args`` and as the
last parameter the value that follows the ``rclone:`` prefix of the repository
specification.
So, calling restic like this
.. code-block:: console
$ restic -o rclone.program="/path/to/rclone" \
-o rclone.args="serve restic --stdio --bwlimit 1M --b2-hard-delete --verbose" \
-r rclone:b2:foo/bar
runs rclone as follows:
.. code-block:: console
$ /path/to/rclone serve restic --stdio --bwlimit 1M --b2-hard-delete --verbose b2:foo/bar
Manually setting ``rclone.program`` also allows running a remote instance of
rclone e.g. via SSH on a server, for example:
.. code-block:: console
$ restic -o rclone.program="ssh user@host rclone" -r rclone:b2:foo/bar
The rclone command may also be hard-coded in the SSH configuration or the
user's public key, in this case it may be sufficient to just start the SSH
connection (and it's irrelevant what's passed after ``rclone:`` in the
repository specification):
.. code-block:: console
$ restic -o rclone.program="ssh user@host" -r rclone:x
.. _rclone: https://rclone.org/
.. _configure: https://rclone.org/docs/
.. _configured with environment variables: https://rclone.org/docs/#environment-variables
.. _issue #1657: https://github.com/restic/restic/pull/1657#issuecomment-377707486
Password prompt on Windows
**************************

6
doc/100_references.rst
View File

@ -672,8 +672,8 @@ The following values are valid for ``{type}``:
The API version is selected via the ``Accept`` HTTP header in the request. The
following values are defined:
* ``application/vnd.x.restic.rest.v1+json`` or empty: Select API version 1
* ``application/vnd.x.restic.rest.v2+json``: Select API version 2
* ``application/vnd.x.restic.rest.v1`` or empty: Select API version 1
* ``application/vnd.x.restic.rest.v2``: Select API version 2
The server will respond with the value of the highest version it supports in
the ``Content-Type`` HTTP response header for the HTTP requests which should
@ -681,7 +681,7 @@ return JSON. Any different value for this header means API version 1.
The placeholder ``{path}`` in this document is a path to the repository, so
that multiple different repositories can be accessed. The default path is
``/``.
``/``. The path must end with a slash.
POST {path}?create=true
=======================

7
internal/backend/sftp/foreground_solaris.go → internal/backend/foreground_solaris.go
View File

@ -1,4 +1,4 @@
package sftp
package backend
import (
"os/exec"
@ -7,7 +7,10 @@ import (
"github.com/restic/restic/internal/errors"
)
func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
// StartForeground runs cmd in the foreground, by temporarily switching to the
// new process group created for cmd. The returned function `bg` switches back
// to the previous process group.
func StartForeground(cmd *exec.Cmd) (bg func() error, err error) {
// run the command in it's own process group so that SIGINT
// is not sent to it.
cmd.SysProcAttr = &syscall.SysProcAttr{

6
internal/backend/sftp/foreground_unix.go → internal/backend/foreground_unix.go
View File

@ -1,7 +1,7 @@
// +build !solaris
// +build !windows
package sftp
package backend
import (
"os"
@ -24,10 +24,10 @@ func tcsetpgrp(fd int, pid int) error {
return errno
}
// startForeground runs cmd in the foreground, by temporarily switching to the
// StartForeground runs cmd in the foreground, by temporarily switching to the
// new process group created for cmd. The returned function `bg` switches back
// to the previous process group.
func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
func StartForeground(cmd *exec.Cmd) (bg func() error, err error) {
// open the TTY, we need the file descriptor
tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0)
if err != nil {

6
internal/backend/sftp/foreground_windows.go → internal/backend/foreground_windows.go
View File

@ -1,4 +1,4 @@
package sftp
package backend
import (
"os/exec"
@ -6,10 +6,10 @@ import (
"github.com/restic/restic/internal/errors"
)
// startForeground runs cmd in the foreground, by temporarily switching to the
// StartForeground runs cmd in the foreground, by temporarily switching to the
// new process group created for cmd. The returned function `bg` switches back
// to the previous process group.
func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
func StartForeground(cmd *exec.Cmd) (bg func() error, err error) {
// just start the process and hope for the best
err = cmd.Start()
if err != nil {

2
internal/backend/location/location.go
View File

@ -8,6 +8,7 @@ import (
"github.com/restic/restic/internal/backend/b2"
"github.com/restic/restic/internal/backend/gs"
"github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/backend/rclone"
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/backend/sftp"
@ -38,6 +39,7 @@ var parsers = []parser{
{"azure", azure.ParseConfig},
{"swift", swift.ParseConfig},
{"rest", rest.ParseConfig},
{"rclone", rclone.ParseConfig},
}
func isPath(s string) bool {

280
internal/backend/rclone/backend.go Normal file
View File

@ -0,0 +1,280 @@
package rclone
import (
"bufio"
"context"
"crypto/tls"
"fmt"
"math/rand"
"net"
"net/http"
"net/url"
"os"
"os/exec"
"sync"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"golang.org/x/net/context/ctxhttp"
"golang.org/x/net/http2"
)
// Backend is used to access data stored somewhere via rclone.
type Backend struct {
*rest.Backend
tr *http2.Transport
cmd *exec.Cmd
waitCh <-chan struct{}
waitResult error
wg *sync.WaitGroup
conn *StdioConn
}
// run starts command with args and initializes the StdioConn.
func run(command string, args ...string) (*StdioConn, *exec.Cmd, *sync.WaitGroup, func() error, error) {
cmd := exec.Command(command, args...)
p, err := cmd.StderrPipe()
if err != nil {
return nil, nil, nil, nil, err
}
var wg sync.WaitGroup
// start goroutine to add a prefix to all messages printed by to stderr by rclone
wg.Add(1)
go func() {
defer wg.Done()
sc := bufio.NewScanner(p)
for sc.Scan() {
fmt.Fprintf(os.Stderr, "rclone: %v\n", sc.Text())
}
}()
r, stdin, err := os.Pipe()
if err != nil {
return nil, nil, nil, nil, err
}
stdout, w, err := os.Pipe()
if err != nil {
return nil, nil, nil, nil, err
}
cmd.Stdin = r
cmd.Stdout = w
bg, err := backend.StartForeground(cmd)
if err != nil {
return nil, nil, nil, nil, err
}
c := &StdioConn{
stdin: stdout,
stdout: stdin,
cmd: cmd,
}
return c, cmd, &wg, bg, nil
}
// New initializes a Backend and starts the process.
func New(cfg Config) (*Backend, error) {
var (
args []string
err error
)
// build program args, start with the program
if cfg.Program != "" {
a, err := backend.SplitShellStrings(cfg.Program)
if err != nil {
return nil, err
}
args = append(args, a...)
} else {
args = append(args, "rclone")
}
// then add the arguments
if cfg.Args != "" {
a, err := backend.SplitShellStrings(cfg.Args)
if err != nil {
return nil, err
}
args = append(args, a...)
} else {
args = append(args,
"serve", "restic", "--stdio",
"--b2-hard-delete", "--drive-use-trash=false")
}
// finally, add the remote
args = append(args, cfg.Remote)
arg0, args := args[0], args[1:]
debug.Log("running command: %v %v", arg0, args)
conn, cmd, wg, bg, err := run(arg0, args...)
if err != nil {
return nil, err
}
dialCount := 0
tr := &http2.Transport{
AllowHTTP: true, // this is not really HTTP, just stdin/stdout
DialTLS: func(network, address string, cfg *tls.Config) (net.Conn, error) {
debug.Log("new connection requested, %v %v", network, address)
if dialCount > 0 {
panic("dial count > 0")
}
dialCount++
return conn, nil
},
}
waitCh := make(chan struct{})
be := &Backend{
tr: tr,
cmd: cmd,
waitCh: waitCh,
conn: conn,
wg: wg,
}
wg.Add(1)
go func() {
defer wg.Done()
debug.Log("waiting for error result")
err := cmd.Wait()
debug.Log("Wait returned %v", err)
be.waitResult = err
close(waitCh)
}()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
wg.Add(1)
go func() {
defer wg.Done()
debug.Log("monitoring command to cancel first HTTP request context")
select {
case <-ctx.Done():
debug.Log("context has been cancelled, returning")
case <-be.waitCh:
debug.Log("command has exited, cancelling context")
cancel()
}
}()
// send an HTTP request to the base URL, see if the server is there
client := &http.Client{
Transport: tr,
Timeout: 60 * time.Second,
}
// request a random file which does not exist. we just want to test when
// rclone is able to accept HTTP requests.
url := fmt.Sprintf("http://localhost/file-%d", rand.Uint64())
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", rest.ContentTypeV2)
req.Cancel = ctx.Done()
res, err := ctxhttp.Do(ctx, client, req)
if err != nil {
bg()
_ = cmd.Process.Kill()
return nil, errors.Errorf("error talking HTTP to rclone: %v", err)
}
debug.Log("HTTP status %q returned, moving instance to background", res.Status)
bg()
return be, nil
}
// Open starts an rclone process with the given config.
func Open(cfg Config) (*Backend, error) {
be, err := New(cfg)
if err != nil {
return nil, err
}
url, err := url.Parse("http://localhost/")
if err != nil {
return nil, err
}
restConfig := rest.Config{
Connections: cfg.Connections,
URL: url,
}
restBackend, err := rest.Open(restConfig, be.tr)
if err != nil {
return nil, err
}
be.Backend = restBackend
return be, nil
}
// Create initializes a new restic repo with clone.
func Create(cfg Config) (*Backend, error) {
be, err := New(cfg)
if err != nil {
return nil, err
}
debug.Log("new backend created")
url, err := url.Parse("http://localhost/")
if err != nil {
return nil, err
}
restConfig := rest.Config{
Connections: 20,
URL: url,
}
restBackend, err := rest.Create(restConfig, be.tr)
if err != nil {
_ = be.Close()
return nil, err
}
be.Backend = restBackend
return be, nil
}
const waitForExit = 5 * time.Second
// Close terminates the backend.
func (be *Backend) Close() error {
debug.Log("exiting rclone")
be.tr.CloseIdleConnections()
select {
case <-be.waitCh:
debug.Log("rclone exited")
case <-time.After(waitForExit):
debug.Log("timeout, closing file descriptors")
err := be.conn.Close()
if err != nil {
return err
}
}
be.wg.Wait()
debug.Log("wait for rclone returned: %v", be.waitResult)
return be.waitResult
}

66
internal/backend/rclone/backend_test.go Normal file
View File

@ -0,0 +1,66 @@
package rclone_test
import (
"os/exec"
"testing"
"github.com/restic/restic/internal/backend/rclone"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func newTestSuite(t testing.TB) *test.Suite {
dir, cleanup := rtest.TempDir(t)
return &test.Suite{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
t.Logf("use backend at %v", dir)
cfg := rclone.NewConfig()
cfg.Remote = dir
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
t.Logf("Create()")
cfg := config.(rclone.Config)
be, err := rclone.Create(cfg)
if e, ok := errors.Cause(err).(*exec.Error); ok && e.Err == exec.ErrNotFound {
t.Skipf("program %q not found", e.Name)
return nil, nil
}
return be, err
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
t.Logf("Open()")
cfg := config.(rclone.Config)
return rclone.Open(cfg)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
t.Logf("cleanup dir %v", dir)
cleanup()
return nil
},
}
}
func TestBackendRclone(t *testing.T) {
defer func() {
if t.Skipped() {
rtest.SkipDisallowed(t, "restic/backend/rclone.TestBackendRclone")
}
}()
newTestSuite(t).RunTests(t)
}
func BenchmarkBackendREST(t *testing.B) {
newTestSuite(t).RunBenchmarks(t)
}

39
internal/backend/rclone/config.go Normal file
View File

@ -0,0 +1,39 @@
package rclone
import (
"strings"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
)
// Config contains all configuration necessary to start rclone.
type Config struct {
Program string `option:"program" help:"path to rclone (default: rclone)"`
Args string `option:"args" help:"arguments for running rclone (default: serve restic --stdio --b2-hard-delete --drive-use-trash=false)"`
Remote string
Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
}
func init() {
options.Register("rclone", Config{})
}
// NewConfig returns a new Config with the default values filled in.
func NewConfig() Config {
return Config{
Connections: 5,
}
}
// ParseConfig parses the string s and extracts the remote server URL.
func ParseConfig(s string) (interface{}, error) {
if !strings.HasPrefix(s, "rclone:") {
return nil, errors.New("invalid rclone backend specification")
}
s = s[7:]
cfg := NewConfig()
cfg.Remote = s
return cfg, nil
}

34
internal/backend/rclone/config_test.go Normal file
View File

@ -0,0 +1,34 @@
package rclone
import (
"reflect"
"testing"
)
func TestParseConfig(t *testing.T) {
var tests = []struct {
s string
cfg Config
}{
{
"rclone:local:foo:/bar",
Config{
Remote: "local:foo:/bar",
Connections: 5,
},
},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
cfg, err := ParseConfig(test.s)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(cfg, test.cfg) {
t.Fatalf("wrong config, want:\n %v\ngot:\n %v", test.cfg, cfg)
}
})
}
}

74
internal/backend/rclone/stdio_conn.go Normal file
View File

@ -0,0 +1,74 @@
package rclone
import (
"net"
"os"
"os/exec"
"sync"
"github.com/restic/restic/internal/debug"
)
// StdioConn implements a net.Conn via stdin/stdout.
type StdioConn struct {
stdin *os.File
stdout *os.File
cmd *exec.Cmd
close sync.Once
}
func (s *StdioConn) Read(p []byte) (int, error) {
n, err := s.stdin.Read(p)
return n, err
}
func (s *StdioConn) Write(p []byte) (int, error) {
n, err := s.stdout.Write(p)
return n, err
}
// Close closes both streams.
func (s *StdioConn) Close() (err error) {
s.close.Do(func() {
debug.Log("close stdio connection")
var errs []error
for _, f := range []func() error{s.stdin.Close, s.stdout.Close} {
err := f()
if err != nil {
errs = append(errs, err)
}
}
if len(errs) > 0 {
err = errs[0]
}
})
return err
}
// LocalAddr returns nil.
func (s *StdioConn) LocalAddr() net.Addr {
return Addr{}
}
// RemoteAddr returns nil.
func (s *StdioConn) RemoteAddr() net.Addr {
return Addr{}
}
// make sure StdioConn implements net.Conn
var _ net.Conn = &StdioConn{}
// Addr implements net.Addr for stdin/stdout.
type Addr struct{}
// Network returns the network type as a string.
func (a Addr) Network() string {
return "stdio"
}
func (a Addr) String() string {
return "stdio"
}

25
internal/backend/rclone/stdio_conn_go110.go Normal file
View File

@ -0,0 +1,25 @@
// +build go1.10
package rclone
import "time"
// SetDeadline sets the read/write deadline.
func (s *StdioConn) SetDeadline(t time.Time) error {
err1 := s.stdin.SetReadDeadline(t)
err2 := s.stdout.SetWriteDeadline(t)
if err1 != nil {
return err1
}
return err2
}
// SetReadDeadline sets the read/write deadline.
func (s *StdioConn) SetReadDeadline(t time.Time) error {
return s.stdin.SetReadDeadline(t)
}
// SetWriteDeadline sets the read/write deadline.
func (s *StdioConn) SetWriteDeadline(t time.Time) error {
return s.stdout.SetWriteDeadline(t)
}

22
internal/backend/rclone/stdio_conn_other.go Normal file
View File

@ -0,0 +1,22 @@
// +build !go1.10
package rclone
import "time"
// On Go < 1.10, it's not possible to set read/write deadlines on files, so we just ignore that.
// SetDeadline sets the read/write deadline.
func (s *StdioConn) SetDeadline(t time.Time) error {
return nil
}
// SetReadDeadline sets the read/write deadline.
func (s *StdioConn) SetReadDeadline(t time.Time) error {
return nil
}
// SetWriteDeadline sets the read/write deadline.
func (s *StdioConn) SetWriteDeadline(t time.Time) error {
return nil
}

4
internal/backend/rest/config.go
View File

@ -32,6 +32,10 @@ func ParseConfig(s string) (interface{}, error) {
}
s = s[5:]
if !strings.HasSuffix(s, "/") {
s += "/"
}
u, err := url.Parse(s)
if err != nil {

40
internal/backend/rest/config_test.go
View File

@ -19,24 +19,34 @@ var configTests = []struct {
s string
cfg Config
}{
{"rest:http://localhost:1234", Config{
URL: parseURL("http://localhost:1234"),
Connections: 5,
}},
{
s: "rest:http://localhost:1234",
cfg: Config{
URL: parseURL("http://localhost:1234/"),
Connections: 5,
},
},
{
s: "rest:http://localhost:1234/",
cfg: Config{
URL: parseURL("http://localhost:1234/"),
Connections: 5,
},
},
}
func TestParseConfig(t *testing.T) {
for i, test := range configTests {
cfg, err := ParseConfig(test.s)
if err != nil {
t.Errorf("test %d:%s failed: %v", i, test.s, err)
continue
}
for _, test := range configTests {
t.Run("", func(t *testing.T) {
cfg, err := ParseConfig(test.s)
if err != nil {
t.Fatalf("%s failed: %v", test.s, err)
}
if !reflect.DeepEqual(cfg, test.cfg) {
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
i, test.s, test.cfg, cfg)
continue
}
if !reflect.DeepEqual(cfg, test.cfg) {
t.Fatalf("\ninput: %s\n wrong config, want:\n %v\ngot:\n %v",
test.s, test.cfg, cfg)
}
})
}
}

56
internal/backend/rest/rest.go
View File

@ -21,22 +21,24 @@ import (
)
// make sure the rest backend implements restic.Backend
var _ restic.Backend = &restBackend{}
var _ restic.Backend = &Backend{}
type restBackend struct {
// Backend uses the REST protocol to access data stored on a server.
type Backend struct {
url *url.URL
sem *backend.Semaphore
client *http.Client
backend.Layout
}
// the REST API protocol version is decided by HTTP request headers, these are the constants.
const (
contentTypeV1 = "application/vnd.x.restic.rest.v1"
contentTypeV2 = "application/vnd.x.restic.rest.v2"
ContentTypeV1 = "application/vnd.x.restic.rest.v1"
ContentTypeV2 = "application/vnd.x.restic.rest.v2"
)
// Open opens the REST backend with the given config.
func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) {
func Open(cfg Config, rt http.RoundTripper) (*Backend, error) {
client := &http.Client{Transport: rt}
sem, err := backend.NewSemaphore(cfg.Connections)
@ -50,7 +52,7 @@ func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) {
url = url[:len(url)-1]
}
be := &restBackend{
be := &Backend{
url: cfg.URL,
client: client,
Layout: &backend.RESTLayout{URL: url, Join: path.Join},
@ -61,7 +63,7 @@ func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) {
}
// Create creates a new REST on server configured in config.
func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Create(cfg Config, rt http.RoundTripper) (*Backend, error) {
be, err := Open(cfg, rt)
if err != nil {
return nil, err
@ -100,12 +102,12 @@ func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
}
// Location returns this backend's location (the server's URL).
func (b *restBackend) Location() string {
func (b *Backend) Location() string {
return b.url.String()
}
// Save stores data in the backend at the handle.
func (b *restBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
if err := h.Valid(); err != nil {
return err
}
@ -119,7 +121,7 @@ func (b *restBackend) Save(ctx context.Context, h restic.Handle, rd restic.Rewin
return errors.Wrap(err, "NewRequest")
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Accept", contentTypeV2)
req.Header.Set("Accept", ContentTypeV2)
// explicitly set the content length, this prevents chunked encoding and
// let's the server know what's coming.
@ -162,7 +164,7 @@ func (e ErrIsNotExist) Error() string {
}
// IsNotExist returns true if the error was caused by a non-existing file.
func (b *restBackend) IsNotExist(err error) bool {
func (b *Backend) IsNotExist(err error) bool {
err = errors.Cause(err)
_, ok := err.(ErrIsNotExist)
return ok
@ -170,11 +172,11 @@ func (b *restBackend) IsNotExist(err error) bool {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (b *restBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return backend.DefaultLoad(ctx, h, length, offset, b.openReader, fn)
}
func (b *restBackend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
debug.Log("Load %v, length %v, offset %v", h, length, offset)
if err := h.Valid(); err != nil {
return nil, err
@ -198,7 +200,7 @@ func (b *restBackend) openReader(ctx context.Context, h restic.Handle, length in
byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
}
req.Header.Set("Range", byteRange)
req.Header.Set("Accept", contentTypeV2)
req.Header.Set("Accept", ContentTypeV2)
debug.Log("Load(%v) send range %v", h, byteRange)
b.sem.GetToken()
@ -227,7 +229,7 @@ func (b *restBackend) openReader(ctx context.Context, h restic.Handle, length in
}
// Stat returns information about a blob.
func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
if err := h.Valid(); err != nil {
return restic.FileInfo{}, err
}
@ -236,7 +238,7 @@ func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInf
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "NewRequest")
}
req.Header.Set("Accept", contentTypeV2)
req.Header.Set("Accept", ContentTypeV2)
b.sem.GetToken()
resp, err := ctxhttp.Do(ctx, b.client, req)
@ -272,7 +274,7 @@ func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInf
}
// Test returns true if a blob of the given type and name exists in the backend.
func (b *restBackend) Test(ctx context.Context, h restic.Handle) (bool, error) {
func (b *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
_, err := b.Stat(ctx, h)
if err != nil {
return false, nil
@ -282,7 +284,7 @@ func (b *restBackend) Test(ctx context.Context, h restic.Handle) (bool, error) {
}
// Remove removes the blob with the given name and type.
func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
if err := h.Valid(); err != nil {
return err
}
@ -291,7 +293,7 @@ func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
if err != nil {
return errors.Wrap(err, "http.NewRequest")
}
req.Header.Set("Accept", contentTypeV2)
req.Header.Set("Accept", ContentTypeV2)
b.sem.GetToken()
resp, err := ctxhttp.Do(ctx, b.client, req)
@ -320,7 +322,7 @@ func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
url := b.Dirname(restic.Handle{Type: t})
if !strings.HasSuffix(url, "/") {
url += "/"
@ -330,7 +332,7 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
if err != nil {
return errors.Wrap(err, "NewRequest")
}
req.Header.Set("Accept", contentTypeV2)
req.Header.Set("Accept", ContentTypeV2)
b.sem.GetToken()
resp, err := ctxhttp.Do(ctx, b.client, req)
@ -344,7 +346,7 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
return errors.Errorf("List failed, server response: %v (%v)", resp.Status, resp.StatusCode)
}
if resp.Header.Get("Content-Type") == contentTypeV2 {
if resp.Header.Get("Content-Type") == ContentTypeV2 {
return b.listv2(ctx, t, resp, fn)
}
@ -354,7 +356,7 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
// listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET
// /data/`) only returns the names of the files, so we need to issue an HTTP
// HEAD request for each file.
func (b *restBackend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
debug.Log("parsing API v1 response")
dec := json.NewDecoder(resp.Body)
var list []string
@ -388,7 +390,7 @@ func (b *restBackend) listv1(ctx context.Context, t restic.FileType, resp *http.
// listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET
// /data/`) returns the names and sizes of all files.
func (b *restBackend) listv2(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
func (b *Backend) listv2(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
debug.Log("parsing API v2 response")
dec := json.NewDecoder(resp.Body)
@ -424,21 +426,21 @@ func (b *restBackend) listv2(ctx context.Context, t restic.FileType, resp *http.
}
// Close closes all open files.
func (b *restBackend) Close() error {
func (b *Backend) Close() error {
// this does not need to do anything, all open files are closed within the
// same function.
return nil
}
// Remove keys for a specified backend type.
func (b *restBackend) removeKeys(ctx context.Context, t restic.FileType) error {
func (b *Backend) removeKeys(ctx context.Context, t restic.FileType) error {
return b.List(ctx, t, func(fi restic.FileInfo) error {
return b.Remove(ctx, restic.Handle{Type: t, Name: fi.Name})
})
}
// Delete removes all data in the backend.
func (b *restBackend) Delete(ctx context.Context) error {
func (b *Backend) Delete(ctx context.Context) error {
alltypes := []restic.FileType{
restic.DataFile,
restic.KeyFile,

9
internal/backend/sftp/sftp.go
View File

@ -65,7 +65,7 @@ func startClient(program string, args ...string) (*SFTP, error) {
return nil, errors.Wrap(err, "cmd.StdoutPipe")
}
bg, err := startForeground(cmd)
bg, err := backend.StartForeground(cmd)
if err != nil {
return nil, errors.Wrap(err, "cmd.Start")
}
@ -179,7 +179,12 @@ func (r *SFTP) IsNotExist(err error) bool {
func buildSSHCommand(cfg Config) (cmd string, args []string, err error) {
if cfg.Command != "" {
return SplitShellArgs(cfg.Command)
args, err := backend.SplitShellStrings(cfg.Command)
if err != nil {
return "", nil, err
}
return args[0], args[1:], nil
}
cmd = "ssh"

22
internal/backend/sftp/split.go → internal/backend/shell_split.go
View File

@ -1,4 +1,4 @@
package sftp
package backend
import (
"unicode"
@ -41,8 +41,8 @@ func (s *shellSplitter) isSplitChar(c rune) bool {
return c == '\\' || unicode.IsSpace(c)
}
// SplitShellArgs returns the list of arguments from a shell command string.
func SplitShellArgs(data string) (cmd string, args []string, err error) {
// SplitShellStrings returns the list of shell strings from a shell command string.
func SplitShellStrings(data string) (strs []string, err error) {
s := &shellSplitter{}
// derived from strings.SplitFunc
@ -50,7 +50,7 @@ func SplitShellArgs(data string) (cmd string, args []string, err error) {
for i, rune := range data {
if s.isSplitChar(rune) {
if fieldStart >= 0 {
args = append(args, data[fieldStart:i])
strs = append(strs, data[fieldStart:i])
fieldStart = -1
}
} else if fieldStart == -1 {
@ -58,21 +58,19 @@ func SplitShellArgs(data string) (cmd string, args []string, err error) {
}
}
if fieldStart >= 0 { // Last field might end at EOF.
args = append(args, data[fieldStart:])
strs = append(strs, data[fieldStart:])
}
switch s.quote {
case '\'':
return "", nil, errors.New("single-quoted string not terminated")
return nil, errors.New("single-quoted string not terminated")
case '"':
return "", nil, errors.New("double-quoted string not terminated")
return nil, errors.New("double-quoted string not terminated")
}
if len(args) == 0 {
return "", nil, errors.New("command string is empty")
if len(strs) == 0 {
return nil, errors.New("command string is empty")
}
cmd, args = args[0], args[1:]
return cmd, args, nil
return strs, nil
}

34
internal/backend/sftp/split_test.go → internal/backend/shell_split_test.go
View File

@ -1,4 +1,4 @@
package sftp
package backend
import (
"reflect"
@ -8,59 +8,53 @@ import (
func TestShellSplitter(t *testing.T) {
var tests = []struct {
data string
cmd string
args []string
}{
{
`foo`,
"foo", []string{},
[]string{"foo"},
},
{
`'foo'`,
"foo", []string{},
[]string{"foo"},
},
{
`foo bar baz`,
"foo", []string{"bar", "baz"},
[]string{"foo", "bar", "baz"},
},
{
`foo 'bar' baz`,
"foo", []string{"bar", "baz"},
[]string{"foo", "bar", "baz"},
},
{
`'bar box' baz`,
"bar box", []string{"baz"},
[]string{"bar box", "baz"},
},
{
`"bar 'box'" baz`,
"bar 'box'", []string{"baz"},
[]string{"bar 'box'", "baz"},
},
{
`'bar "box"' baz`,
`bar "box"`, []string{"baz"},
[]string{`bar "box"`, "baz"},
},
{
`\"bar box baz`,
`"bar`, []string{"box", "baz"},
[]string{`"bar`, "box", "baz"},
},
{
`"bar/foo/x" "box baz"`,
"bar/foo/x", []string{"box baz"},
[]string{"bar/foo/x", "box baz"},
},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
cmd, args, err := SplitShellArgs(test.data)
args, err := SplitShellStrings(test.data)
if err != nil {
t.Fatal(err)
}
if cmd != test.cmd {
t.Fatalf("wrong cmd returned, want:\n %#v\ngot:\n %#v",
test.cmd, cmd)
}
if !reflect.DeepEqual(args, test.args) {
t.Fatalf("wrong args returned, want:\n %#v\ngot:\n %#v",
test.args, args)
@ -94,7 +88,7 @@ func TestShellSplitterInvalid(t *testing.T) {
for _, test := range tests {
t.Run("", func(t *testing.T) {
cmd, args, err := SplitShellArgs(test.data)
args, err := SplitShellStrings(test.data)
if err == nil {
t.Fatalf("expected error not found: %v", test.err)
}
@ -103,10 +97,6 @@ func TestShellSplitterInvalid(t *testing.T) {
t.Fatalf("expected error not found, want:\n %q\ngot:\n %q", test.err, err.Error())
}
if cmd != "" {
t.Fatalf("splitter returned cmd from invalid data: %v", cmd)
}
if len(args) > 0 {
t.Fatalf("splitter returned fields from invalid data: %v", args)
}

4
internal/backend/test/tests.go
View File

@ -147,6 +147,10 @@ func (s *Suite) TestLoad(t *testing.T) {
}
err = b.Load(context.TODO(), handle, 0, 0, func(rd io.Reader) error {
_, err := io.Copy(ioutil.Discard, rd)
if err != nil {
t.Fatal(err)
}
return errors.Errorf("deliberate error")
})
if err == nil {

2
run_integration_tests.go
View File

@ -97,6 +97,7 @@ func (env *TravisEnvironment) Prepare() error {
"github.com/golang/dep/cmd/dep",
"github.com/restic/rest-server/cmd/rest-server",
"github.com/restic/calens",
"github.com/ncw/rclone",
}
for _, pkg := range pkgs {
@ -191,6 +192,7 @@ func (env *TravisEnvironment) RunTests() error {
"restic/backend/rest.TestBackendREST",
"restic/backend/sftp.TestBackendSFTP",
"restic/backend/s3.TestBackendMinio",
"restic/backend/rclone.TestBackendRclone",
}
// if the test s3 repository is available, make sure that the test is not skipped