mirror of
https://github.com/octoleo/restic.git
synced 2024-11-05 12:57:53 +00:00
commit
b077a1227b
2
Gopkg.lock
generated
2
Gopkg.lock
generated
@ -232,6 +232,6 @@
|
||||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
analyzer-version = 1
|
||||
inputs-digest = "a7d099b3ce195ffc37adedb05a4386be38e6158925a1c0fe579efdc20fa11f6a"
|
||||
inputs-digest = "d3d59414a33bb8ecc6d88a681c782a87244a565cc9d0f85615cfa0704c02800a"
|
||||
solver-name = "gps-cdcl"
|
||||
solver-version = 1
|
||||
|
@ -57,6 +57,7 @@ Therefore, restic supports the following backends for storing backups natively:
|
||||
- `BackBlaze B2 <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#backblaze-b2>`__
|
||||
- `Microsoft Azure Blob Storage <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#microsoft-azure-blob-storage>`__
|
||||
- `Google Cloud Storage <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#google-cloud-storage>`__
|
||||
- And many other services via the `rclone <https://rclone.org>`__ `Backend <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone>`__
|
||||
|
||||
Design Principles
|
||||
-----------------
|
||||
|
10
changelog/unreleased/issue-1561
Normal file
10
changelog/unreleased/issue-1561
Normal file
@ -0,0 +1,10 @@
|
||||
Enhancement: Allow using rclone to access other services
|
||||
|
||||
We've added the ability to use rclone to store backup data on all backends that
|
||||
it supports. This was done in collaboration with Nick, the author of rclone.
|
||||
You can now use it to first configure a service, then restic manages the rest
|
||||
(starting and stopping rclone). For details, please see the manual.
|
||||
|
||||
https://github.com/restic/restic/issues/1561
|
||||
https://github.com/restic/restic/pull/1657
|
||||
https://rclone.org
|
@ -18,6 +18,7 @@ import (
|
||||
"github.com/restic/restic/internal/backend/gs"
|
||||
"github.com/restic/restic/internal/backend/local"
|
||||
"github.com/restic/restic/internal/backend/location"
|
||||
"github.com/restic/restic/internal/backend/rclone"
|
||||
"github.com/restic/restic/internal/backend/rest"
|
||||
"github.com/restic/restic/internal/backend/s3"
|
||||
"github.com/restic/restic/internal/backend/sftp"
|
||||
@ -509,6 +510,14 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debug.Log("opening rest repository at %#v", cfg)
|
||||
return cfg, nil
|
||||
case "rclone":
|
||||
cfg := loc.Config.(rclone.Config)
|
||||
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debug.Log("opening rest repository at %#v", cfg)
|
||||
return cfg, nil
|
||||
}
|
||||
@ -564,6 +573,8 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
|
||||
be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt)
|
||||
case "rest":
|
||||
be, err = rest.Open(cfg.(rest.Config), rt)
|
||||
case "rclone":
|
||||
be, err = rclone.Open(cfg.(rclone.Config))
|
||||
|
||||
default:
|
||||
return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
|
||||
@ -625,6 +636,8 @@ func create(s string, opts options.Options) (restic.Backend, error) {
|
||||
return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt)
|
||||
case "rest":
|
||||
return rest.Create(cfg.(rest.Config), rt)
|
||||
case "rclone":
|
||||
return rclone.Open(cfg.(rclone.Config))
|
||||
}
|
||||
|
||||
debug.Log("invalid repository scheme: %v", s)
|
||||
|
@ -139,7 +139,7 @@ If you use TLS, restic will use the system's CA certificates to verify the
|
||||
server certificate. When the verification fails, restic refuses to proceed and
|
||||
exits with an error. If you have your own self-signed certificate, or a custom
|
||||
CA certificate should be used for verification, you can pass restic the
|
||||
certificate filename via the `--cacert` option.
|
||||
certificate filename via the ``--cacert`` option.
|
||||
|
||||
REST server uses exactly the same directory structure as local backend,
|
||||
so you should be able to access it both locally and via HTTP, even
|
||||
@ -306,8 +306,8 @@ bucket does not exist yet, it will be created:
|
||||
Please note that knowledge of your password is required to access the repository.
|
||||
Losing your password means that your data is irrecoverably lost.
|
||||
|
||||
The number of concurrent connections to the B2 service can be set with the `-o
|
||||
b2.connections=10`. By default, at most five parallel connections are
|
||||
The number of concurrent connections to the B2 service can be set with the ``-o
|
||||
b2.connections=10``. By default, at most five parallel connections are
|
||||
established.
|
||||
|
||||
Microsoft Azure Blob Storage
|
||||
@ -321,7 +321,7 @@ account name and key as follows:
|
||||
$ export AZURE_ACCOUNT_NAME=<ACCOUNT_NAME>
|
||||
$ export AZURE_ACCOUNT_KEY=<SECRET_KEY>
|
||||
|
||||
Afterwards you can initialize a repository in a container called `foo` in the
|
||||
Afterwards you can initialize a repository in a container called ``foo`` in the
|
||||
root path like this:
|
||||
|
||||
.. code-block:: console
|
||||
@ -334,7 +334,7 @@ root path like this:
|
||||
[...]
|
||||
|
||||
The number of concurrent connections to the Azure Blob Storage service can be set with the
|
||||
`-o azure.connections=10`. By default, at most five parallel connections are
|
||||
``-o azure.connections=10``. By default, at most five parallel connections are
|
||||
established.
|
||||
|
||||
Google Cloud Storage
|
||||
@ -369,7 +369,7 @@ located on an instance with default service accounts then these should work out
|
||||
the box.
|
||||
|
||||
Once authenticated, you can use the ``gs:`` backend type to create a new
|
||||
repository in the bucket `foo` at the root path:
|
||||
repository in the bucket ``foo`` at the root path:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
@ -381,12 +381,117 @@ repository in the bucket `foo` at the root path:
|
||||
[...]
|
||||
|
||||
The number of concurrent connections to the GCS service can be set with the
|
||||
`-o gs.connections=10`. By default, at most five parallel connections are
|
||||
``-o gs.connections=10``. By default, at most five parallel connections are
|
||||
established.
|
||||
|
||||
.. _service account: https://cloud.google.com/storage/docs/authentication#service_accounts
|
||||
.. _create a service account key: https://cloud.google.com/storage/docs/authentication#generating-a-private-key
|
||||
|
||||
Other Services via rclone
|
||||
*************************
|
||||
|
||||
The program `rclone`_ can be used to access many other different services and
|
||||
store data there. First, you need to install and `configure`_ rclone. The
|
||||
general backend specification format is ``rclone:<remote>:<path>``, the
|
||||
``<remote>:<path>`` component will be directly passed to rclone. When you
|
||||
configure a remote named ``foo``, you can then call restic as follows to
|
||||
initiate a new repository in the path ``bar`` in the repo:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ restic -r rclone:foo:bar init
|
||||
|
||||
Restic takes care of starting and stopping rclone.
|
||||
|
||||
As a more concrete example, suppose you have configured a remote named
|
||||
``b2prod`` for Backblaze B2 with rclone, with a bucket called ``yggdrasil``.
|
||||
You can then use rclone to list files in the bucket like this:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ rclone ls b2prod:yggdrasil
|
||||
|
||||
In order to create a new repository in the root directory of the bucket, call
|
||||
restic like this:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ restic -r rclone:b2prod:yggdrasil init
|
||||
|
||||
If you want to use the path ``foo/bar/baz`` in the bucket instead, pass this to
|
||||
restic:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ restic -r rclone:b2prod:yggdrasil/foo/bar/baz init
|
||||
|
||||
Listing the files of an empty repository directly with rclone should return a
|
||||
listing similar to the following:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ rclone ls b2prod:yggdrasil/foo/bar/baz
|
||||
155 bar/baz/config
|
||||
448 bar/baz/keys/4bf9c78049de689d73a56ed0546f83b8416795295cda12ec7fb9465af3900b44
|
||||
|
||||
Rclone can be `configured with environment variables`_, so for instance
|
||||
configuring a bandwidth limit for rclone cat be achieve by setting the
|
||||
``RCLONE_BWLIMIT`` environment variable:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ export RCLONE_BWLIMIT=1M
|
||||
|
||||
For debugging rclone, you can set the environment variable ``RCLONE_VERBOSE=2``.
|
||||
|
||||
The rclone backend has two additional options:
|
||||
|
||||
* ``-o rclone.program`` specifies the path to rclone, the default value is just ``rclone``
|
||||
* ``-o rclone.args`` allows setting the arguments passed to rclone, by default this is ``serve restic --stdio --b2-hard-delete --drive-use-trash=false``
|
||||
|
||||
The reason why the two last parameters (``--b2-hard-delete`` and
|
||||
``--drive-use-trash=false``) can be found in the corresponding GitHub `issue #1657`_.
|
||||
|
||||
In order to start rclone, restic will build a list of arguments by joining the
|
||||
following lists (in this order): ``rclone.program``, ``rclone.args`` and as the
|
||||
last parameter the value that follows the ``rclone:`` prefix of the repository
|
||||
specification.
|
||||
|
||||
So, calling restic like this
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ restic -o rclone.program="/path/to/rclone" \
|
||||
-o rclone.args="serve restic --stdio --bwlimit 1M --b2-hard-delete --verbose" \
|
||||
-r rclone:b2:foo/bar
|
||||
|
||||
runs rclone as follows:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ /path/to/rclone serve restic --stdio --bwlimit 1M --b2-hard-delete --verbose b2:foo/bar
|
||||
|
||||
Manually setting ``rclone.program`` also allows running a remote instance of
|
||||
rclone e.g. via SSH on a server, for example:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ restic -o rclone.program="ssh user@host rclone" -r rclone:b2:foo/bar
|
||||
|
||||
The rclone command may also be hard-coded in the SSH configuration or the
|
||||
user's public key, in this case it may be sufficient to just start the SSH
|
||||
connection (and it's irrelevant what's passed after ``rclone:`` in the
|
||||
repository specification):
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ restic -o rclone.program="ssh user@host" -r rclone:x
|
||||
|
||||
.. _rclone: https://rclone.org/
|
||||
.. _configure: https://rclone.org/docs/
|
||||
.. _configured with environment variables: https://rclone.org/docs/#environment-variables
|
||||
.. _issue #1657: https://github.com/restic/restic/pull/1657#issuecomment-377707486
|
||||
|
||||
Password prompt on Windows
|
||||
**************************
|
||||
|
||||
|
@ -672,8 +672,8 @@ The following values are valid for ``{type}``:
|
||||
The API version is selected via the ``Accept`` HTTP header in the request. The
|
||||
following values are defined:
|
||||
|
||||
* ``application/vnd.x.restic.rest.v1+json`` or empty: Select API version 1
|
||||
* ``application/vnd.x.restic.rest.v2+json``: Select API version 2
|
||||
* ``application/vnd.x.restic.rest.v1`` or empty: Select API version 1
|
||||
* ``application/vnd.x.restic.rest.v2``: Select API version 2
|
||||
|
||||
The server will respond with the value of the highest version it supports in
|
||||
the ``Content-Type`` HTTP response header for the HTTP requests which should
|
||||
@ -681,7 +681,7 @@ return JSON. Any different value for this header means API version 1.
|
||||
|
||||
The placeholder ``{path}`` in this document is a path to the repository, so
|
||||
that multiple different repositories can be accessed. The default path is
|
||||
``/``.
|
||||
``/``. The path must end with a slash.
|
||||
|
||||
POST {path}?create=true
|
||||
=======================
|
||||
|
@ -1,4 +1,4 @@
|
||||
package sftp
|
||||
package backend
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
@ -7,7 +7,10 @@ import (
|
||||
"github.com/restic/restic/internal/errors"
|
||||
)
|
||||
|
||||
func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
|
||||
// StartForeground runs cmd in the foreground, by temporarily switching to the
|
||||
// new process group created for cmd. The returned function `bg` switches back
|
||||
// to the previous process group.
|
||||
func StartForeground(cmd *exec.Cmd) (bg func() error, err error) {
|
||||
// run the command in it's own process group so that SIGINT
|
||||
// is not sent to it.
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{
|
@ -1,7 +1,7 @@
|
||||
// +build !solaris
|
||||
// +build !windows
|
||||
|
||||
package sftp
|
||||
package backend
|
||||
|
||||
import (
|
||||
"os"
|
||||
@ -24,10 +24,10 @@ func tcsetpgrp(fd int, pid int) error {
|
||||
return errno
|
||||
}
|
||||
|
||||
// startForeground runs cmd in the foreground, by temporarily switching to the
|
||||
// StartForeground runs cmd in the foreground, by temporarily switching to the
|
||||
// new process group created for cmd. The returned function `bg` switches back
|
||||
// to the previous process group.
|
||||
func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
|
||||
func StartForeground(cmd *exec.Cmd) (bg func() error, err error) {
|
||||
// open the TTY, we need the file descriptor
|
||||
tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0)
|
||||
if err != nil {
|
@ -1,4 +1,4 @@
|
||||
package sftp
|
||||
package backend
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
@ -6,10 +6,10 @@ import (
|
||||
"github.com/restic/restic/internal/errors"
|
||||
)
|
||||
|
||||
// startForeground runs cmd in the foreground, by temporarily switching to the
|
||||
// StartForeground runs cmd in the foreground, by temporarily switching to the
|
||||
// new process group created for cmd. The returned function `bg` switches back
|
||||
// to the previous process group.
|
||||
func startForeground(cmd *exec.Cmd) (bg func() error, err error) {
|
||||
func StartForeground(cmd *exec.Cmd) (bg func() error, err error) {
|
||||
// just start the process and hope for the best
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
@ -8,6 +8,7 @@ import (
|
||||
"github.com/restic/restic/internal/backend/b2"
|
||||
"github.com/restic/restic/internal/backend/gs"
|
||||
"github.com/restic/restic/internal/backend/local"
|
||||
"github.com/restic/restic/internal/backend/rclone"
|
||||
"github.com/restic/restic/internal/backend/rest"
|
||||
"github.com/restic/restic/internal/backend/s3"
|
||||
"github.com/restic/restic/internal/backend/sftp"
|
||||
@ -38,6 +39,7 @@ var parsers = []parser{
|
||||
{"azure", azure.ParseConfig},
|
||||
{"swift", swift.ParseConfig},
|
||||
{"rest", rest.ParseConfig},
|
||||
{"rclone", rclone.ParseConfig},
|
||||
}
|
||||
|
||||
func isPath(s string) bool {
|
||||
|
280
internal/backend/rclone/backend.go
Normal file
280
internal/backend/rclone/backend.go
Normal file
@ -0,0 +1,280 @@
|
||||
package rclone
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/backend/rest"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"golang.org/x/net/context/ctxhttp"
|
||||
"golang.org/x/net/http2"
|
||||
)
|
||||
|
||||
// Backend is used to access data stored somewhere via rclone.
|
||||
type Backend struct {
|
||||
*rest.Backend
|
||||
tr *http2.Transport
|
||||
cmd *exec.Cmd
|
||||
waitCh <-chan struct{}
|
||||
waitResult error
|
||||
wg *sync.WaitGroup
|
||||
conn *StdioConn
|
||||
}
|
||||
|
||||
// run starts command with args and initializes the StdioConn.
|
||||
func run(command string, args ...string) (*StdioConn, *exec.Cmd, *sync.WaitGroup, func() error, error) {
|
||||
cmd := exec.Command(command, args...)
|
||||
|
||||
p, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// start goroutine to add a prefix to all messages printed by to stderr by rclone
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
sc := bufio.NewScanner(p)
|
||||
for sc.Scan() {
|
||||
fmt.Fprintf(os.Stderr, "rclone: %v\n", sc.Text())
|
||||
}
|
||||
}()
|
||||
|
||||
r, stdin, err := os.Pipe()
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
stdout, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
cmd.Stdin = r
|
||||
cmd.Stdout = w
|
||||
|
||||
bg, err := backend.StartForeground(cmd)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
c := &StdioConn{
|
||||
stdin: stdout,
|
||||
stdout: stdin,
|
||||
cmd: cmd,
|
||||
}
|
||||
|
||||
return c, cmd, &wg, bg, nil
|
||||
}
|
||||
|
||||
// New initializes a Backend and starts the process.
|
||||
func New(cfg Config) (*Backend, error) {
|
||||
var (
|
||||
args []string
|
||||
err error
|
||||
)
|
||||
|
||||
// build program args, start with the program
|
||||
if cfg.Program != "" {
|
||||
a, err := backend.SplitShellStrings(cfg.Program)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
args = append(args, a...)
|
||||
} else {
|
||||
args = append(args, "rclone")
|
||||
}
|
||||
|
||||
// then add the arguments
|
||||
if cfg.Args != "" {
|
||||
a, err := backend.SplitShellStrings(cfg.Args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
args = append(args, a...)
|
||||
} else {
|
||||
args = append(args,
|
||||
"serve", "restic", "--stdio",
|
||||
"--b2-hard-delete", "--drive-use-trash=false")
|
||||
}
|
||||
|
||||
// finally, add the remote
|
||||
args = append(args, cfg.Remote)
|
||||
arg0, args := args[0], args[1:]
|
||||
|
||||
debug.Log("running command: %v %v", arg0, args)
|
||||
conn, cmd, wg, bg, err := run(arg0, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dialCount := 0
|
||||
tr := &http2.Transport{
|
||||
AllowHTTP: true, // this is not really HTTP, just stdin/stdout
|
||||
DialTLS: func(network, address string, cfg *tls.Config) (net.Conn, error) {
|
||||
debug.Log("new connection requested, %v %v", network, address)
|
||||
if dialCount > 0 {
|
||||
panic("dial count > 0")
|
||||
}
|
||||
dialCount++
|
||||
return conn, nil
|
||||
},
|
||||
}
|
||||
|
||||
waitCh := make(chan struct{})
|
||||
be := &Backend{
|
||||
tr: tr,
|
||||
cmd: cmd,
|
||||
waitCh: waitCh,
|
||||
conn: conn,
|
||||
wg: wg,
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
debug.Log("waiting for error result")
|
||||
err := cmd.Wait()
|
||||
debug.Log("Wait returned %v", err)
|
||||
be.waitResult = err
|
||||
close(waitCh)
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
debug.Log("monitoring command to cancel first HTTP request context")
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
debug.Log("context has been cancelled, returning")
|
||||
case <-be.waitCh:
|
||||
debug.Log("command has exited, cancelling context")
|
||||
cancel()
|
||||
}
|
||||
}()
|
||||
|
||||
// send an HTTP request to the base URL, see if the server is there
|
||||
client := &http.Client{
|
||||
Transport: tr,
|
||||
Timeout: 60 * time.Second,
|
||||
}
|
||||
|
||||
// request a random file which does not exist. we just want to test when
|
||||
// rclone is able to accept HTTP requests.
|
||||
url := fmt.Sprintf("http://localhost/file-%d", rand.Uint64())
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Accept", rest.ContentTypeV2)
|
||||
req.Cancel = ctx.Done()
|
||||
|
||||
res, err := ctxhttp.Do(ctx, client, req)
|
||||
if err != nil {
|
||||
bg()
|
||||
_ = cmd.Process.Kill()
|
||||
return nil, errors.Errorf("error talking HTTP to rclone: %v", err)
|
||||
}
|
||||
|
||||
debug.Log("HTTP status %q returned, moving instance to background", res.Status)
|
||||
bg()
|
||||
|
||||
return be, nil
|
||||
}
|
||||
|
||||
// Open starts an rclone process with the given config.
|
||||
func Open(cfg Config) (*Backend, error) {
|
||||
be, err := New(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
url, err := url.Parse("http://localhost/")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
restConfig := rest.Config{
|
||||
Connections: cfg.Connections,
|
||||
URL: url,
|
||||
}
|
||||
|
||||
restBackend, err := rest.Open(restConfig, be.tr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
be.Backend = restBackend
|
||||
return be, nil
|
||||
}
|
||||
|
||||
// Create initializes a new restic repo with clone.
|
||||
func Create(cfg Config) (*Backend, error) {
|
||||
be, err := New(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debug.Log("new backend created")
|
||||
|
||||
url, err := url.Parse("http://localhost/")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
restConfig := rest.Config{
|
||||
Connections: 20,
|
||||
URL: url,
|
||||
}
|
||||
|
||||
restBackend, err := rest.Create(restConfig, be.tr)
|
||||
if err != nil {
|
||||
_ = be.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
be.Backend = restBackend
|
||||
return be, nil
|
||||
}
|
||||
|
||||
const waitForExit = 5 * time.Second
|
||||
|
||||
// Close terminates the backend.
|
||||
func (be *Backend) Close() error {
|
||||
debug.Log("exiting rclone")
|
||||
be.tr.CloseIdleConnections()
|
||||
|
||||
select {
|
||||
case <-be.waitCh:
|
||||
debug.Log("rclone exited")
|
||||
case <-time.After(waitForExit):
|
||||
debug.Log("timeout, closing file descriptors")
|
||||
err := be.conn.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
be.wg.Wait()
|
||||
debug.Log("wait for rclone returned: %v", be.waitResult)
|
||||
return be.waitResult
|
||||
}
|
66
internal/backend/rclone/backend_test.go
Normal file
66
internal/backend/rclone/backend_test.go
Normal file
@ -0,0 +1,66 @@
|
||||
package rclone_test
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/backend/rclone"
|
||||
"github.com/restic/restic/internal/backend/test"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func newTestSuite(t testing.TB) *test.Suite {
|
||||
dir, cleanup := rtest.TempDir(t)
|
||||
|
||||
return &test.Suite{
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig: func() (interface{}, error) {
|
||||
t.Logf("use backend at %v", dir)
|
||||
cfg := rclone.NewConfig()
|
||||
cfg.Remote = dir
|
||||
return cfg, nil
|
||||
},
|
||||
|
||||
// CreateFn is a function that creates a temporary repository for the tests.
|
||||
Create: func(config interface{}) (restic.Backend, error) {
|
||||
t.Logf("Create()")
|
||||
cfg := config.(rclone.Config)
|
||||
be, err := rclone.Create(cfg)
|
||||
if e, ok := errors.Cause(err).(*exec.Error); ok && e.Err == exec.ErrNotFound {
|
||||
t.Skipf("program %q not found", e.Name)
|
||||
return nil, nil
|
||||
}
|
||||
return be, err
|
||||
},
|
||||
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
Open: func(config interface{}) (restic.Backend, error) {
|
||||
t.Logf("Open()")
|
||||
cfg := config.(rclone.Config)
|
||||
return rclone.Open(cfg)
|
||||
},
|
||||
|
||||
// CleanupFn removes data created during the tests.
|
||||
Cleanup: func(config interface{}) error {
|
||||
t.Logf("cleanup dir %v", dir)
|
||||
cleanup()
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendRclone(t *testing.T) {
|
||||
defer func() {
|
||||
if t.Skipped() {
|
||||
rtest.SkipDisallowed(t, "restic/backend/rclone.TestBackendRclone")
|
||||
}
|
||||
}()
|
||||
|
||||
newTestSuite(t).RunTests(t)
|
||||
}
|
||||
|
||||
func BenchmarkBackendREST(t *testing.B) {
|
||||
newTestSuite(t).RunBenchmarks(t)
|
||||
}
|
39
internal/backend/rclone/config.go
Normal file
39
internal/backend/rclone/config.go
Normal file
@ -0,0 +1,39 @@
|
||||
package rclone
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/options"
|
||||
)
|
||||
|
||||
// Config contains all configuration necessary to start rclone.
|
||||
type Config struct {
|
||||
Program string `option:"program" help:"path to rclone (default: rclone)"`
|
||||
Args string `option:"args" help:"arguments for running rclone (default: serve restic --stdio --b2-hard-delete --drive-use-trash=false)"`
|
||||
Remote string
|
||||
Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
|
||||
}
|
||||
|
||||
func init() {
|
||||
options.Register("rclone", Config{})
|
||||
}
|
||||
|
||||
// NewConfig returns a new Config with the default values filled in.
|
||||
func NewConfig() Config {
|
||||
return Config{
|
||||
Connections: 5,
|
||||
}
|
||||
}
|
||||
|
||||
// ParseConfig parses the string s and extracts the remote server URL.
|
||||
func ParseConfig(s string) (interface{}, error) {
|
||||
if !strings.HasPrefix(s, "rclone:") {
|
||||
return nil, errors.New("invalid rclone backend specification")
|
||||
}
|
||||
|
||||
s = s[7:]
|
||||
cfg := NewConfig()
|
||||
cfg.Remote = s
|
||||
return cfg, nil
|
||||
}
|
34
internal/backend/rclone/config_test.go
Normal file
34
internal/backend/rclone/config_test.go
Normal file
@ -0,0 +1,34 @@
|
||||
package rclone
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
var tests = []struct {
|
||||
s string
|
||||
cfg Config
|
||||
}{
|
||||
{
|
||||
"rclone:local:foo:/bar",
|
||||
Config{
|
||||
Remote: "local:foo:/bar",
|
||||
Connections: 5,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
cfg, err := ParseConfig(test.s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(cfg, test.cfg) {
|
||||
t.Fatalf("wrong config, want:\n %v\ngot:\n %v", test.cfg, cfg)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
74
internal/backend/rclone/stdio_conn.go
Normal file
74
internal/backend/rclone/stdio_conn.go
Normal file
@ -0,0 +1,74 @@
|
||||
package rclone
|
||||
|
||||
import (
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sync"
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
)
|
||||
|
||||
// StdioConn implements a net.Conn via stdin/stdout.
|
||||
type StdioConn struct {
|
||||
stdin *os.File
|
||||
stdout *os.File
|
||||
cmd *exec.Cmd
|
||||
close sync.Once
|
||||
}
|
||||
|
||||
func (s *StdioConn) Read(p []byte) (int, error) {
|
||||
n, err := s.stdin.Read(p)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (s *StdioConn) Write(p []byte) (int, error) {
|
||||
n, err := s.stdout.Write(p)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close closes both streams.
|
||||
func (s *StdioConn) Close() (err error) {
|
||||
s.close.Do(func() {
|
||||
debug.Log("close stdio connection")
|
||||
var errs []error
|
||||
|
||||
for _, f := range []func() error{s.stdin.Close, s.stdout.Close} {
|
||||
err := f()
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
err = errs[0]
|
||||
}
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// LocalAddr returns nil.
|
||||
func (s *StdioConn) LocalAddr() net.Addr {
|
||||
return Addr{}
|
||||
}
|
||||
|
||||
// RemoteAddr returns nil.
|
||||
func (s *StdioConn) RemoteAddr() net.Addr {
|
||||
return Addr{}
|
||||
}
|
||||
|
||||
// make sure StdioConn implements net.Conn
|
||||
var _ net.Conn = &StdioConn{}
|
||||
|
||||
// Addr implements net.Addr for stdin/stdout.
|
||||
type Addr struct{}
|
||||
|
||||
// Network returns the network type as a string.
|
||||
func (a Addr) Network() string {
|
||||
return "stdio"
|
||||
}
|
||||
|
||||
func (a Addr) String() string {
|
||||
return "stdio"
|
||||
}
|
25
internal/backend/rclone/stdio_conn_go110.go
Normal file
25
internal/backend/rclone/stdio_conn_go110.go
Normal file
@ -0,0 +1,25 @@
|
||||
// +build go1.10
|
||||
|
||||
package rclone
|
||||
|
||||
import "time"
|
||||
|
||||
// SetDeadline sets the read/write deadline.
|
||||
func (s *StdioConn) SetDeadline(t time.Time) error {
|
||||
err1 := s.stdin.SetReadDeadline(t)
|
||||
err2 := s.stdout.SetWriteDeadline(t)
|
||||
if err1 != nil {
|
||||
return err1
|
||||
}
|
||||
return err2
|
||||
}
|
||||
|
||||
// SetReadDeadline sets the read/write deadline.
|
||||
func (s *StdioConn) SetReadDeadline(t time.Time) error {
|
||||
return s.stdin.SetReadDeadline(t)
|
||||
}
|
||||
|
||||
// SetWriteDeadline sets the read/write deadline.
|
||||
func (s *StdioConn) SetWriteDeadline(t time.Time) error {
|
||||
return s.stdout.SetWriteDeadline(t)
|
||||
}
|
22
internal/backend/rclone/stdio_conn_other.go
Normal file
22
internal/backend/rclone/stdio_conn_other.go
Normal file
@ -0,0 +1,22 @@
|
||||
// +build !go1.10
|
||||
|
||||
package rclone
|
||||
|
||||
import "time"
|
||||
|
||||
// On Go < 1.10, it's not possible to set read/write deadlines on files, so we just ignore that.
|
||||
|
||||
// SetDeadline sets the read/write deadline.
|
||||
func (s *StdioConn) SetDeadline(t time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetReadDeadline sets the read/write deadline.
|
||||
func (s *StdioConn) SetReadDeadline(t time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetWriteDeadline sets the read/write deadline.
|
||||
func (s *StdioConn) SetWriteDeadline(t time.Time) error {
|
||||
return nil
|
||||
}
|
@ -32,6 +32,10 @@ func ParseConfig(s string) (interface{}, error) {
|
||||
}
|
||||
|
||||
s = s[5:]
|
||||
if !strings.HasSuffix(s, "/") {
|
||||
s += "/"
|
||||
}
|
||||
|
||||
u, err := url.Parse(s)
|
||||
|
||||
if err != nil {
|
||||
|
@ -19,24 +19,34 @@ var configTests = []struct {
|
||||
s string
|
||||
cfg Config
|
||||
}{
|
||||
{"rest:http://localhost:1234", Config{
|
||||
URL: parseURL("http://localhost:1234"),
|
||||
Connections: 5,
|
||||
}},
|
||||
{
|
||||
s: "rest:http://localhost:1234",
|
||||
cfg: Config{
|
||||
URL: parseURL("http://localhost:1234/"),
|
||||
Connections: 5,
|
||||
},
|
||||
},
|
||||
{
|
||||
s: "rest:http://localhost:1234/",
|
||||
cfg: Config{
|
||||
URL: parseURL("http://localhost:1234/"),
|
||||
Connections: 5,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
for i, test := range configTests {
|
||||
cfg, err := ParseConfig(test.s)
|
||||
if err != nil {
|
||||
t.Errorf("test %d:%s failed: %v", i, test.s, err)
|
||||
continue
|
||||
}
|
||||
for _, test := range configTests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
cfg, err := ParseConfig(test.s)
|
||||
if err != nil {
|
||||
t.Fatalf("%s failed: %v", test.s, err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(cfg, test.cfg) {
|
||||
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
|
||||
i, test.s, test.cfg, cfg)
|
||||
continue
|
||||
}
|
||||
if !reflect.DeepEqual(cfg, test.cfg) {
|
||||
t.Fatalf("\ninput: %s\n wrong config, want:\n %v\ngot:\n %v",
|
||||
test.s, test.cfg, cfg)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
@ -21,22 +21,24 @@ import (
|
||||
)
|
||||
|
||||
// make sure the rest backend implements restic.Backend
|
||||
var _ restic.Backend = &restBackend{}
|
||||
var _ restic.Backend = &Backend{}
|
||||
|
||||
type restBackend struct {
|
||||
// Backend uses the REST protocol to access data stored on a server.
|
||||
type Backend struct {
|
||||
url *url.URL
|
||||
sem *backend.Semaphore
|
||||
client *http.Client
|
||||
backend.Layout
|
||||
}
|
||||
|
||||
// the REST API protocol version is decided by HTTP request headers, these are the constants.
|
||||
const (
|
||||
contentTypeV1 = "application/vnd.x.restic.rest.v1"
|
||||
contentTypeV2 = "application/vnd.x.restic.rest.v2"
|
||||
ContentTypeV1 = "application/vnd.x.restic.rest.v1"
|
||||
ContentTypeV2 = "application/vnd.x.restic.rest.v2"
|
||||
)
|
||||
|
||||
// Open opens the REST backend with the given config.
|
||||
func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) {
|
||||
func Open(cfg Config, rt http.RoundTripper) (*Backend, error) {
|
||||
client := &http.Client{Transport: rt}
|
||||
|
||||
sem, err := backend.NewSemaphore(cfg.Connections)
|
||||
@ -50,7 +52,7 @@ func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) {
|
||||
url = url[:len(url)-1]
|
||||
}
|
||||
|
||||
be := &restBackend{
|
||||
be := &Backend{
|
||||
url: cfg.URL,
|
||||
client: client,
|
||||
Layout: &backend.RESTLayout{URL: url, Join: path.Join},
|
||||
@ -61,7 +63,7 @@ func Open(cfg Config, rt http.RoundTripper) (*restBackend, error) {
|
||||
}
|
||||
|
||||
// Create creates a new REST on server configured in config.
|
||||
func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
|
||||
func Create(cfg Config, rt http.RoundTripper) (*Backend, error) {
|
||||
be, err := Open(cfg, rt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -100,12 +102,12 @@ func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
|
||||
}
|
||||
|
||||
// Location returns this backend's location (the server's URL).
|
||||
func (b *restBackend) Location() string {
|
||||
func (b *Backend) Location() string {
|
||||
return b.url.String()
|
||||
}
|
||||
|
||||
// Save stores data in the backend at the handle.
|
||||
func (b *restBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
|
||||
func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
|
||||
if err := h.Valid(); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -119,7 +121,7 @@ func (b *restBackend) Save(ctx context.Context, h restic.Handle, rd restic.Rewin
|
||||
return errors.Wrap(err, "NewRequest")
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
req.Header.Set("Accept", contentTypeV2)
|
||||
req.Header.Set("Accept", ContentTypeV2)
|
||||
|
||||
// explicitly set the content length, this prevents chunked encoding and
|
||||
// let's the server know what's coming.
|
||||
@ -162,7 +164,7 @@ func (e ErrIsNotExist) Error() string {
|
||||
}
|
||||
|
||||
// IsNotExist returns true if the error was caused by a non-existing file.
|
||||
func (b *restBackend) IsNotExist(err error) bool {
|
||||
func (b *Backend) IsNotExist(err error) bool {
|
||||
err = errors.Cause(err)
|
||||
_, ok := err.(ErrIsNotExist)
|
||||
return ok
|
||||
@ -170,11 +172,11 @@ func (b *restBackend) IsNotExist(err error) bool {
|
||||
|
||||
// Load runs fn with a reader that yields the contents of the file at h at the
|
||||
// given offset.
|
||||
func (b *restBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
|
||||
func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
|
||||
return backend.DefaultLoad(ctx, h, length, offset, b.openReader, fn)
|
||||
}
|
||||
|
||||
func (b *restBackend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
|
||||
func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
|
||||
debug.Log("Load %v, length %v, offset %v", h, length, offset)
|
||||
if err := h.Valid(); err != nil {
|
||||
return nil, err
|
||||
@ -198,7 +200,7 @@ func (b *restBackend) openReader(ctx context.Context, h restic.Handle, length in
|
||||
byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
|
||||
}
|
||||
req.Header.Set("Range", byteRange)
|
||||
req.Header.Set("Accept", contentTypeV2)
|
||||
req.Header.Set("Accept", ContentTypeV2)
|
||||
debug.Log("Load(%v) send range %v", h, byteRange)
|
||||
|
||||
b.sem.GetToken()
|
||||
@ -227,7 +229,7 @@ func (b *restBackend) openReader(ctx context.Context, h restic.Handle, length in
|
||||
}
|
||||
|
||||
// Stat returns information about a blob.
|
||||
func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
|
||||
func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
|
||||
if err := h.Valid(); err != nil {
|
||||
return restic.FileInfo{}, err
|
||||
}
|
||||
@ -236,7 +238,7 @@ func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInf
|
||||
if err != nil {
|
||||
return restic.FileInfo{}, errors.Wrap(err, "NewRequest")
|
||||
}
|
||||
req.Header.Set("Accept", contentTypeV2)
|
||||
req.Header.Set("Accept", ContentTypeV2)
|
||||
|
||||
b.sem.GetToken()
|
||||
resp, err := ctxhttp.Do(ctx, b.client, req)
|
||||
@ -272,7 +274,7 @@ func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInf
|
||||
}
|
||||
|
||||
// Test returns true if a blob of the given type and name exists in the backend.
|
||||
func (b *restBackend) Test(ctx context.Context, h restic.Handle) (bool, error) {
|
||||
func (b *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
|
||||
_, err := b.Stat(ctx, h)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
@ -282,7 +284,7 @@ func (b *restBackend) Test(ctx context.Context, h restic.Handle) (bool, error) {
|
||||
}
|
||||
|
||||
// Remove removes the blob with the given name and type.
|
||||
func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
|
||||
func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
|
||||
if err := h.Valid(); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -291,7 +293,7 @@ func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "http.NewRequest")
|
||||
}
|
||||
req.Header.Set("Accept", contentTypeV2)
|
||||
req.Header.Set("Accept", ContentTypeV2)
|
||||
|
||||
b.sem.GetToken()
|
||||
resp, err := ctxhttp.Do(ctx, b.client, req)
|
||||
@ -320,7 +322,7 @@ func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error {
|
||||
|
||||
// List runs fn for each file in the backend which has the type t. When an
|
||||
// error occurs (or fn returns an error), List stops and returns it.
|
||||
func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
|
||||
func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
|
||||
url := b.Dirname(restic.Handle{Type: t})
|
||||
if !strings.HasSuffix(url, "/") {
|
||||
url += "/"
|
||||
@ -330,7 +332,7 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "NewRequest")
|
||||
}
|
||||
req.Header.Set("Accept", contentTypeV2)
|
||||
req.Header.Set("Accept", ContentTypeV2)
|
||||
|
||||
b.sem.GetToken()
|
||||
resp, err := ctxhttp.Do(ctx, b.client, req)
|
||||
@ -344,7 +346,7 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
|
||||
return errors.Errorf("List failed, server response: %v (%v)", resp.Status, resp.StatusCode)
|
||||
}
|
||||
|
||||
if resp.Header.Get("Content-Type") == contentTypeV2 {
|
||||
if resp.Header.Get("Content-Type") == ContentTypeV2 {
|
||||
return b.listv2(ctx, t, resp, fn)
|
||||
}
|
||||
|
||||
@ -354,7 +356,7 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(resti
|
||||
// listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET
|
||||
// /data/`) only returns the names of the files, so we need to issue an HTTP
|
||||
// HEAD request for each file.
|
||||
func (b *restBackend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
|
||||
func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
|
||||
debug.Log("parsing API v1 response")
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
var list []string
|
||||
@ -388,7 +390,7 @@ func (b *restBackend) listv1(ctx context.Context, t restic.FileType, resp *http.
|
||||
|
||||
// listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET
|
||||
// /data/`) returns the names and sizes of all files.
|
||||
func (b *restBackend) listv2(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
|
||||
func (b *Backend) listv2(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
|
||||
debug.Log("parsing API v2 response")
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
|
||||
@ -424,21 +426,21 @@ func (b *restBackend) listv2(ctx context.Context, t restic.FileType, resp *http.
|
||||
}
|
||||
|
||||
// Close closes all open files.
|
||||
func (b *restBackend) Close() error {
|
||||
func (b *Backend) Close() error {
|
||||
// this does not need to do anything, all open files are closed within the
|
||||
// same function.
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove keys for a specified backend type.
|
||||
func (b *restBackend) removeKeys(ctx context.Context, t restic.FileType) error {
|
||||
func (b *Backend) removeKeys(ctx context.Context, t restic.FileType) error {
|
||||
return b.List(ctx, t, func(fi restic.FileInfo) error {
|
||||
return b.Remove(ctx, restic.Handle{Type: t, Name: fi.Name})
|
||||
})
|
||||
}
|
||||
|
||||
// Delete removes all data in the backend.
|
||||
func (b *restBackend) Delete(ctx context.Context) error {
|
||||
func (b *Backend) Delete(ctx context.Context) error {
|
||||
alltypes := []restic.FileType{
|
||||
restic.DataFile,
|
||||
restic.KeyFile,
|
||||
|
@ -65,7 +65,7 @@ func startClient(program string, args ...string) (*SFTP, error) {
|
||||
return nil, errors.Wrap(err, "cmd.StdoutPipe")
|
||||
}
|
||||
|
||||
bg, err := startForeground(cmd)
|
||||
bg, err := backend.StartForeground(cmd)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cmd.Start")
|
||||
}
|
||||
@ -179,7 +179,12 @@ func (r *SFTP) IsNotExist(err error) bool {
|
||||
|
||||
func buildSSHCommand(cfg Config) (cmd string, args []string, err error) {
|
||||
if cfg.Command != "" {
|
||||
return SplitShellArgs(cfg.Command)
|
||||
args, err := backend.SplitShellStrings(cfg.Command)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
return args[0], args[1:], nil
|
||||
}
|
||||
|
||||
cmd = "ssh"
|
||||
|
@ -1,4 +1,4 @@
|
||||
package sftp
|
||||
package backend
|
||||
|
||||
import (
|
||||
"unicode"
|
||||
@ -41,8 +41,8 @@ func (s *shellSplitter) isSplitChar(c rune) bool {
|
||||
return c == '\\' || unicode.IsSpace(c)
|
||||
}
|
||||
|
||||
// SplitShellArgs returns the list of arguments from a shell command string.
|
||||
func SplitShellArgs(data string) (cmd string, args []string, err error) {
|
||||
// SplitShellStrings returns the list of shell strings from a shell command string.
|
||||
func SplitShellStrings(data string) (strs []string, err error) {
|
||||
s := &shellSplitter{}
|
||||
|
||||
// derived from strings.SplitFunc
|
||||
@ -50,7 +50,7 @@ func SplitShellArgs(data string) (cmd string, args []string, err error) {
|
||||
for i, rune := range data {
|
||||
if s.isSplitChar(rune) {
|
||||
if fieldStart >= 0 {
|
||||
args = append(args, data[fieldStart:i])
|
||||
strs = append(strs, data[fieldStart:i])
|
||||
fieldStart = -1
|
||||
}
|
||||
} else if fieldStart == -1 {
|
||||
@ -58,21 +58,19 @@ func SplitShellArgs(data string) (cmd string, args []string, err error) {
|
||||
}
|
||||
}
|
||||
if fieldStart >= 0 { // Last field might end at EOF.
|
||||
args = append(args, data[fieldStart:])
|
||||
strs = append(strs, data[fieldStart:])
|
||||
}
|
||||
|
||||
switch s.quote {
|
||||
case '\'':
|
||||
return "", nil, errors.New("single-quoted string not terminated")
|
||||
return nil, errors.New("single-quoted string not terminated")
|
||||
case '"':
|
||||
return "", nil, errors.New("double-quoted string not terminated")
|
||||
return nil, errors.New("double-quoted string not terminated")
|
||||
}
|
||||
|
||||
if len(args) == 0 {
|
||||
return "", nil, errors.New("command string is empty")
|
||||
if len(strs) == 0 {
|
||||
return nil, errors.New("command string is empty")
|
||||
}
|
||||
|
||||
cmd, args = args[0], args[1:]
|
||||
|
||||
return cmd, args, nil
|
||||
return strs, nil
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
package sftp
|
||||
package backend
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
@ -8,59 +8,53 @@ import (
|
||||
func TestShellSplitter(t *testing.T) {
|
||||
var tests = []struct {
|
||||
data string
|
||||
cmd string
|
||||
args []string
|
||||
}{
|
||||
{
|
||||
`foo`,
|
||||
"foo", []string{},
|
||||
[]string{"foo"},
|
||||
},
|
||||
{
|
||||
`'foo'`,
|
||||
"foo", []string{},
|
||||
[]string{"foo"},
|
||||
},
|
||||
{
|
||||
`foo bar baz`,
|
||||
"foo", []string{"bar", "baz"},
|
||||
[]string{"foo", "bar", "baz"},
|
||||
},
|
||||
{
|
||||
`foo 'bar' baz`,
|
||||
"foo", []string{"bar", "baz"},
|
||||
[]string{"foo", "bar", "baz"},
|
||||
},
|
||||
{
|
||||
`'bar box' baz`,
|
||||
"bar box", []string{"baz"},
|
||||
[]string{"bar box", "baz"},
|
||||
},
|
||||
{
|
||||
`"bar 'box'" baz`,
|
||||
"bar 'box'", []string{"baz"},
|
||||
[]string{"bar 'box'", "baz"},
|
||||
},
|
||||
{
|
||||
`'bar "box"' baz`,
|
||||
`bar "box"`, []string{"baz"},
|
||||
[]string{`bar "box"`, "baz"},
|
||||
},
|
||||
{
|
||||
`\"bar box baz`,
|
||||
`"bar`, []string{"box", "baz"},
|
||||
[]string{`"bar`, "box", "baz"},
|
||||
},
|
||||
{
|
||||
`"bar/foo/x" "box baz"`,
|
||||
"bar/foo/x", []string{"box baz"},
|
||||
[]string{"bar/foo/x", "box baz"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
cmd, args, err := SplitShellArgs(test.data)
|
||||
args, err := SplitShellStrings(test.data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if cmd != test.cmd {
|
||||
t.Fatalf("wrong cmd returned, want:\n %#v\ngot:\n %#v",
|
||||
test.cmd, cmd)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(args, test.args) {
|
||||
t.Fatalf("wrong args returned, want:\n %#v\ngot:\n %#v",
|
||||
test.args, args)
|
||||
@ -94,7 +88,7 @@ func TestShellSplitterInvalid(t *testing.T) {
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
cmd, args, err := SplitShellArgs(test.data)
|
||||
args, err := SplitShellStrings(test.data)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error not found: %v", test.err)
|
||||
}
|
||||
@ -103,10 +97,6 @@ func TestShellSplitterInvalid(t *testing.T) {
|
||||
t.Fatalf("expected error not found, want:\n %q\ngot:\n %q", test.err, err.Error())
|
||||
}
|
||||
|
||||
if cmd != "" {
|
||||
t.Fatalf("splitter returned cmd from invalid data: %v", cmd)
|
||||
}
|
||||
|
||||
if len(args) > 0 {
|
||||
t.Fatalf("splitter returned fields from invalid data: %v", args)
|
||||
}
|
@ -147,6 +147,10 @@ func (s *Suite) TestLoad(t *testing.T) {
|
||||
}
|
||||
|
||||
err = b.Load(context.TODO(), handle, 0, 0, func(rd io.Reader) error {
|
||||
_, err := io.Copy(ioutil.Discard, rd)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return errors.Errorf("deliberate error")
|
||||
})
|
||||
if err == nil {
|
||||
|
@ -97,6 +97,7 @@ func (env *TravisEnvironment) Prepare() error {
|
||||
"github.com/golang/dep/cmd/dep",
|
||||
"github.com/restic/rest-server/cmd/rest-server",
|
||||
"github.com/restic/calens",
|
||||
"github.com/ncw/rclone",
|
||||
}
|
||||
|
||||
for _, pkg := range pkgs {
|
||||
@ -191,6 +192,7 @@ func (env *TravisEnvironment) RunTests() error {
|
||||
"restic/backend/rest.TestBackendREST",
|
||||
"restic/backend/sftp.TestBackendSFTP",
|
||||
"restic/backend/s3.TestBackendMinio",
|
||||
"restic/backend/rclone.TestBackendRclone",
|
||||
}
|
||||
|
||||
// if the test s3 repository is available, make sure that the test is not skipped
|
||||
|
Loading…
Reference in New Issue
Block a user