octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-12-30 21:41:54 +00:00
Code Issues Releases Wiki Activity

backend: move backend implementation helpers to util package

Browse Source 
This removes code that is only used within a backend implementation from
the backend package. The latter now only contains code that also has
external users.
This commit is contained in:
Michael Eischer 2023-10-01 10:24:33 +02:00
parent 8e6fdf5edf
commit 7881309d63
23 changed files with 158 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
internal/backend/azure/azure.go
View File

@ -12,9 +12,9 @@ import (
"path"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -295,7 +295,7 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
@ -407,7 +407,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *Backend) Delete(ctx context.Context) error {
return backend.DefaultDelete(ctx, be)
return util.DefaultDelete(ctx, be)
}
// Close does nothing

6
internal/backend/b2/b2.go
View File

@ -9,9 +9,9 @@ import (
"sync"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -192,7 +192,7 @@ func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offs
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
@ -313,7 +313,7 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *b2Backend) Delete(ctx context.Context) error {
return backend.DefaultDelete(ctx, be)
return util.DefaultDelete(ctx, be)
}
// Close does nothing

6
internal/backend/gs/gs.go
View File

@ -13,9 +13,9 @@ import (
"cloud.google.com/go/storage"
"github.com/pkg/errors"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
@ -257,7 +257,7 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
@ -350,7 +350,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *Backend) Delete(ctx context.Context) error {
return backend.DefaultDelete(ctx, be)
return util.DefaultDelete(ctx, be)
}
// Close does nothing.

7
internal/backend/local/local.go
View File

@ -12,6 +12,7 @@ import (
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
@ -24,7 +25,7 @@ import (
type Local struct {
Config
layout.Layout
backend.Modes
util.Modes
}
// ensure statically that *Local implements restic.Backend.
@ -43,7 +44,7 @@ func open(ctx context.Context, cfg Config) (*Local, error) {
}
fi, err := fs.Stat(l.Filename(restic.Handle{Type: restic.ConfigFile}))
m := backend.DeriveModesFromFileInfo(fi, err)
m := util.DeriveModesFromFileInfo(fi, err)
debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir)
return &Local{
@ -210,7 +211,7 @@ var tempFile = os.CreateTemp // Overridden by test.
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return backend.DefaultLoad(ctx, h, length, offset, b.openReader, fn)
return util.DefaultLoad(ctx, h, length, offset, b.openReader, fn)
}
func (b *Local) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {

4
internal/backend/mem/mem_backend.go
View File

@ -10,8 +10,8 @@ import (
"sync"
"github.com/cespare/xxhash/v2"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -113,7 +113,7 @@ func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *MemoryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {

5
internal/backend/rclone/backend.go
View File

@ -21,6 +21,7 @@ import (
"github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"golang.org/x/net/http2"
@ -81,7 +82,7 @@ func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, chan stru
cmd.Stdin = r
cmd.Stdout = w
bg, err := backend.StartForeground(cmd)
bg, err := util.StartForeground(cmd)
// close rclone side of pipes
errR := r.Close()
errW := w.Close()
@ -93,7 +94,7 @@ func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, chan stru
err = errW
}
if err != nil {
if backend.IsErrDot(err) {
if util.IsErrDot(err) {
return nil, nil, nil, nil, errors.Errorf("cannot implicitly run relative executable %v found in current directory, use -o rclone.program=./<program> to override", cmd.Path)
}
return nil, nil, nil, nil, err

4
internal/backend/rest/rest.go
View File

@ -11,9 +11,9 @@ import (
"path"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -424,5 +424,5 @@ func (b *Backend) Close() error {
// Delete removes all data in the backend.
func (b *Backend) Delete(ctx context.Context) error {
return backend.DefaultDelete(ctx, b)
return util.DefaultDelete(ctx, b)
}

6
internal/backend/s3/s3.go
View File

@ -11,9 +11,9 @@ import (
"strings"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -298,7 +298,7 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
@ -424,7 +424,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *Backend) Delete(ctx context.Context) error {
return backend.DefaultDelete(ctx, be)
return util.DefaultDelete(ctx, be)
}
// Close does nothing

13
internal/backend/sftp/sftp.go
View File

@ -17,6 +17,7 @@ import (
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -38,7 +39,7 @@ type SFTP struct {
layout.Layout
Config
backend.Modes
util.Modes
}
var _ restic.Backend = &SFTP{}
@ -83,9 +84,9 @@ func startClient(cfg Config) (*SFTP, error) {
return nil, errors.Wrap(err, "cmd.StdoutPipe")
}
bg, err := backend.StartForeground(cmd)
bg, err := util.StartForeground(cmd)
if err != nil {
if backend.IsErrDot(err) {
if util.IsErrDot(err) {
return nil, errors.Errorf("cannot implicitly run relative executable %v found in current directory, use -o sftp.command=./<command> to override", cmd.Path)
}
return nil, err
@ -153,7 +154,7 @@ func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) {
debug.Log("layout: %v\n", sftp.Layout)
fi, err := sftp.c.Stat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile}))
m := backend.DeriveModesFromFileInfo(fi, err)
m := util.DeriveModesFromFileInfo(fi, err)
debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir)
sftp.Config = cfg
@ -259,7 +260,7 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) {
return nil, err
}
sftp.Modes = backend.DefaultModes
sftp.Modes = util.DefaultModes
// test if config file already exists
_, err = sftp.c.Lstat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile}))
@ -414,7 +415,7 @@ func (r *SFTP) checkNoSpace(dir string, size int64, origErr error) error {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (r *SFTP) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return backend.DefaultLoad(ctx, h, length, offset, r.openReader, fn)
return util.DefaultLoad(ctx, h, length, offset, r.openReader, fn)
}
func (r *SFTP) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {

6
internal/backend/swift/swift.go
View File

@ -13,9 +13,9 @@ import (
"strings"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -135,7 +135,7 @@ func (be *beSwift) HasAtomicReplace() bool {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
@ -245,7 +245,7 @@ func (be *beSwift) IsNotExist(err error) bool {
// Delete removes all restic objects in the container.
// It will not remove the container itself.
func (be *beSwift) Delete(ctx context.Context) error {
return backend.DefaultDelete(ctx, be)
return util.DefaultDelete(ctx, be)
}
// Close does nothing

50
internal/backend/util/defaults.go Normal file
View File

@ -0,0 +1,50 @@
package util
import (
"context"
"io"
"github.com/restic/restic/internal/restic"
)
// DefaultLoad implements Backend.Load using lower-level openReader func
func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64,
openReader func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error),
fn func(rd io.Reader) error) error {
rd, err := openReader(ctx, h, length, offset)
if err != nil {
return err
}
err = fn(rd)
if err != nil {
_ = rd.Close() // ignore secondary errors closing the reader
return err
}
return rd.Close()
}
// DefaultDelete removes all restic keys in the bucket. It will not remove the bucket itself.
func DefaultDelete(ctx context.Context, be restic.Backend) error {
alltypes := []restic.FileType{
restic.PackFile,
restic.KeyFile,
restic.LockFile,
restic.SnapshotFile,
restic.IndexFile}
for _, t := range alltypes {
err := be.List(ctx, t, func(fi restic.FileInfo) error {
return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name})
})
if err != nil {
return nil
}
}
err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
if err != nil && be.IsNotExist(err) {
err = nil
}
return err
}

64
internal/backend/util/defaults_test.go Normal file
View File

@ -0,0 +1,64 @@
package util_test
import (
"context"
"io"
"testing"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
type mockReader struct {
closed bool
}
func (rd *mockReader) Read(_ []byte) (n int, err error) {
return 0, nil
}
func (rd *mockReader) Close() error {
rd.closed = true
return nil
}
func TestDefaultLoad(t *testing.T) {
h := restic.Handle{Name: "id", Type: restic.PackFile}
rd := &mockReader{}
// happy case, assert correct parameters are passed around and content stream is closed
err := util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) {
rtest.Equals(t, h, ih)
rtest.Equals(t, int(10), length)
rtest.Equals(t, int64(11), offset)
return rd, nil
}, func(ird io.Reader) error {
rtest.Equals(t, rd, ird)
return nil
})
rtest.OK(t, err)
rtest.Equals(t, true, rd.closed)
// unhappy case, assert producer errors are handled correctly
err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) {
return nil, errors.Errorf("producer error")
}, func(ird io.Reader) error {
t.Fatalf("unexpected consumer invocation")
return nil
})
rtest.Equals(t, "producer error", err.Error())
// unhappy case, assert consumer errors are handled correctly
rd = &mockReader{}
err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) {
return rd, nil
}, func(ird io.Reader) error {
return errors.Errorf("consumer error")
})
rtest.Equals(t, true, rd.closed)
rtest.Equals(t, "consumer error", err.Error())
}

2
internal/backend/errdot_119.go → internal/backend/util/errdot_119.go
View File

@ -8,7 +8,7 @@
// Once the minimum Go version restic supports is 1.19, remove this file and
// replace any calls to it with the corresponding code as per below.
package backend
package util
import (
"errors"

2
internal/backend/errdot_old.go → internal/backend/util/errdot_old.go
View File

@ -6,7 +6,7 @@
// Once the minimum Go version restic supports is 1.19, remove this file
// and perform the actions listed in errdot_119.go.
package backend
package util
func IsErrDot(err error) bool {
return false

2
internal/backend/foreground.go → internal/backend/util/foreground.go
View File

@ -1,4 +1,4 @@
package backend
package util
import (
"os"

2
internal/backend/foreground_sysv.go → internal/backend/util/foreground_sysv.go
View File

@ -1,7 +1,7 @@
//go:build aix || solaris
// +build aix solaris
package backend
package util
import (
"os/exec"

6
internal/backend/foreground_test.go → internal/backend/util/foreground_test.go
View File

@ -1,7 +1,7 @@
//go:build !windows
// +build !windows
package backend_test
package util_test
import (
"bufio"
@ -10,7 +10,7 @@ import (
"strings"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/util"
rtest "github.com/restic/restic/internal/test"
)
@ -22,7 +22,7 @@ func TestForeground(t *testing.T) {
stdout, err := cmd.StdoutPipe()
rtest.OK(t, err)
bg, err := backend.StartForeground(cmd)
bg, err := util.StartForeground(cmd)
rtest.OK(t, err)
defer func() {
rtest.OK(t, cmd.Wait())

2
internal/backend/foreground_unix.go → internal/backend/util/foreground_unix.go
View File

@ -1,7 +1,7 @@
//go:build !aix && !solaris && !windows
// +build !aix,!solaris,!windows
package backend
package util
import (
"os"

2
internal/backend/foreground_windows.go → internal/backend/util/foreground_windows.go
View File

@ -1,4 +1,4 @@
package backend
package util
import (
"os/exec"

2
internal/backend/paths.go → internal/backend/util/paths.go
View File

@ -1,4 +1,4 @@
package backend
package util
import "os"

42
internal/backend/utils.go
View File

@ -58,48 +58,6 @@ func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
return &LimitedReadCloser{Closer: r, LimitedReader: io.LimitedReader{R: r, N: n}}
}
// DefaultLoad implements Backend.Load using lower-level openReader func
func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64,
openReader func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error),
fn func(rd io.Reader) error) error {
rd, err := openReader(ctx, h, length, offset)
if err != nil {
return err
}
err = fn(rd)
if err != nil {
_ = rd.Close() // ignore secondary errors closing the reader
return err
}
return rd.Close()
}
// DefaultDelete removes all restic keys in the bucket. It will not remove the bucket itself.
func DefaultDelete(ctx context.Context, be restic.Backend) error {
alltypes := []restic.FileType{
restic.PackFile,
restic.KeyFile,
restic.LockFile,
restic.SnapshotFile,
restic.IndexFile}
for _, t := range alltypes {
err := be.List(ctx, t, func(fi restic.FileInfo) error {
return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name})
})
if err != nil {
return nil
}
}
err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
if err != nil && be.IsNotExist(err) {
err = nil
}
return err
}
type memorizedLister struct {
fileInfos []restic.FileInfo
tpe restic.FileType

52
internal/backend/utils_test.go
View File

@ -11,7 +11,6 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -150,57 +149,6 @@ func TestLoadAllAppend(t *testing.T) {
}
}
type mockReader struct {
closed bool
}
func (rd *mockReader) Read(_ []byte) (n int, err error) {
return 0, nil
}
func (rd *mockReader) Close() error {
rd.closed = true
return nil
}
func TestDefaultLoad(t *testing.T) {
h := restic.Handle{Name: "id", Type: restic.PackFile}
rd := &mockReader{}
// happy case, assert correct parameters are passed around and content stream is closed
err := backend.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) {
rtest.Equals(t, h, ih)
rtest.Equals(t, int(10), length)
rtest.Equals(t, int64(11), offset)
return rd, nil
}, func(ird io.Reader) error {
rtest.Equals(t, rd, ird)
return nil
})
rtest.OK(t, err)
rtest.Equals(t, true, rd.closed)
// unhappy case, assert producer errors are handled correctly
err = backend.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) {
return nil, errors.Errorf("producer error")
}, func(ird io.Reader) error {
t.Fatalf("unexpected consumer invocation")
return nil
})
rtest.Equals(t, "producer error", err.Error())
// unhappy case, assert consumer errors are handled correctly
rd = &mockReader{}
err = backend.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) {
return rd, nil
}, func(ird io.Reader) error {
return errors.Errorf("consumer error")
})
rtest.Equals(t, true, rd.closed)
rtest.Equals(t, "consumer error", err.Error())
}
func TestMemoizeList(t *testing.T) {
// setup backend to serve as data source for memoized list
be := mock.NewBackend()

2
internal/restic/backend.go
View File

@ -43,7 +43,7 @@ type Backend interface {
// The function fn may be called multiple times during the same Load invocation
// and therefore must be idempotent.
//
// Implementations are encouraged to use backend.DefaultLoad
// Implementations are encouraged to use util.DefaultLoad
Load(ctx context.Context, h Handle, length int, offset int64, fn func(rd io.Reader) error) error
// Stat returns information about the File identified by h.