octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-22 04:45:15 +00:00
Code Issues Releases Wiki Activity

move Backend interface to backend package

Browse Source
This commit is contained in:
Michael Eischer 2023-10-01 11:40:12 +02:00
parent ceb0774af1
commit 1b8a67fe76
105 changed files with 822 additions and 775 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/restic/cmd_backup_integration_test.go
View File

@ -9,6 +9,7 @@ import (
"runtime" "runtime"
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
@ -265,7 +266,7 @@ func TestBackupTreeLoadError(t *testing.T) {
// delete the subdirectory pack first // delete the subdirectory pack first
for id := range treePacks { for id := range treePacks {
rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})) rtest.OK(t, r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()}))
} }
testRunRebuildIndex(t, env.gopts) testRunRebuildIndex(t, env.gopts)
// now the repo is missing the tree blob in the index; check should report this // now the repo is missing the tree blob in the index; check should report this

2
cmd/restic/cmd_cat.go
View File

@ -154,7 +154,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
return nil return nil
case "pack": case "pack":
h := restic.Handle{Type: restic.PackFile, Name: id.String()} h := backend.Handle{Type: restic.PackFile, Name: id.String()}
buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h) buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
if err != nil { if err != nil {
return err return err

4
cmd/restic/cmd_debug.go
View File

@ -321,7 +321,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li
panic(err) panic(err)
} }
be := repo.Backend() be := repo.Backend()
h := restic.Handle{ h := backend.Handle{
Name: packID.String(), Name: packID.String(),
Type: restic.PackFile, Type: restic.PackFile,
} }
@ -490,7 +490,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error { func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error {
Printf("examine %v\n", id) Printf("examine %v\n", id)
h := restic.Handle{ h := backend.Handle{
Type: restic.PackFile, Type: restic.PackFile,
Name: id.String(), Name: id.String(),
} }

2
cmd/restic/cmd_diff.go
View File

@ -58,7 +58,7 @@ func init() {
f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata") f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
} }
func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) { func loadSnapshot(ctx context.Context, be backend.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) {
sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc) sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc)
if err != nil { if err != nil {
return nil, "", errors.Fatal(err.Error()) return nil, "", errors.Fatal(err.Error())

7
cmd/restic/cmd_key.go
View File

@ -7,6 +7,7 @@ import (
"strings" "strings"
"sync" "sync"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -149,7 +150,7 @@ func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) e
return errors.Fatal("refusing to remove key currently used to access repository") return errors.Fatal("refusing to remove key currently used to access repository")
} }
h := restic.Handle{Type: restic.KeyFile, Name: id.String()} h := backend.Handle{Type: restic.KeyFile, Name: id.String()}
err := repo.Backend().Remove(ctx, h) err := repo.Backend().Remove(ctx, h)
if err != nil { if err != nil {
return err return err
@ -176,7 +177,7 @@ func changePassword(ctx context.Context, repo *repository.Repository, gopts Glob
return err return err
} }
h := restic.Handle{Type: restic.KeyFile, Name: oldID.String()} h := backend.Handle{Type: restic.KeyFile, Name: oldID.String()}
err = repo.Backend().Remove(ctx, h) err = repo.Backend().Remove(ctx, h)
if err != nil { if err != nil {
return err return err
@ -193,7 +194,7 @@ func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repos
err := repo.SearchKey(ctx, pw, 0, key.ID().String()) err := repo.SearchKey(ctx, pw, 0, key.ID().String())
if err != nil { if err != nil {
// the key is invalid, try to remove it // the key is invalid, try to remove it
h := restic.Handle{Type: restic.KeyFile, Name: key.ID().String()} h := backend.Handle{Type: restic.KeyFile, Name: key.ID().String()}
_ = repo.Backend().Remove(ctx, h) _ = repo.Backend().Remove(ctx, h)
return errors.Fatalf("failed to access repository with new key: %v", err) return errors.Fatalf("failed to access repository with new key: %v", err)
} }

10
cmd/restic/cmd_key_integration_test.go
View File

@ -6,8 +6,8 @@ import (
"regexp" "regexp"
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
@ -110,11 +110,11 @@ func TestKeyAddRemove(t *testing.T) {
} }
type emptySaveBackend struct { type emptySaveBackend struct {
restic.Backend backend.Backend
} }
func (b *emptySaveBackend) Save(ctx context.Context, h restic.Handle, _ restic.RewindReader) error { func (b *emptySaveBackend) Save(ctx context.Context, h backend.Handle, _ backend.RewindReader) error {
return b.Backend.Save(ctx, h, restic.NewByteReader([]byte{}, nil)) return b.Backend.Save(ctx, h, backend.NewByteReader([]byte{}, nil))
} }
func TestKeyProblems(t *testing.T) { func TestKeyProblems(t *testing.T) {
@ -122,7 +122,7 @@ func TestKeyProblems(t *testing.T) {
defer cleanup() defer cleanup()
testRunInit(t, env.gopts) testRunInit(t, env.gopts)
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return &emptySaveBackend{r}, nil return &emptySaveBackend{r}, nil
} }

6
cmd/restic/cmd_prune_integration_test.go
View File

@ -6,13 +6,13 @@ import (
"path/filepath" "path/filepath"
"testing" "testing"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) { func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
oldHook := gopts.backendTestHook oldHook := gopts.backendTestHook
gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil } gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
defer func() { defer func() {
gopts.backendTestHook = oldHook gopts.backendTestHook = oldHook
}() }()
@ -130,7 +130,7 @@ func TestPruneWithDamagedRepository(t *testing.T) {
removePacksExcept(env.gopts, t, oldPacks, false) removePacksExcept(env.gopts, t, oldPacks, false)
oldHook := env.gopts.backendTestHook oldHook := env.gopts.backendTestHook
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil } env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
defer func() { defer func() {
env.gopts.backendTestHook = oldHook env.gopts.backendTestHook = oldHook
}() }()

13
cmd/restic/cmd_repair_index_integration_test.go
View File

@ -8,6 +8,7 @@ import (
"sync" "sync"
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/index" "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -70,12 +71,12 @@ func TestRebuildIndexAlwaysFull(t *testing.T) {
// indexErrorBackend modifies the first index after reading. // indexErrorBackend modifies the first index after reading.
type indexErrorBackend struct { type indexErrorBackend struct {
restic.Backend backend.Backend
lock sync.Mutex lock sync.Mutex
hasErred bool hasErred bool
} }
func (b *indexErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { func (b *indexErrorBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
// protect hasErred // protect hasErred
b.lock.Lock() b.lock.Lock()
@ -101,7 +102,7 @@ func (erd errorReadCloser) Read(p []byte) (int, error) {
} }
func TestRebuildIndexDamage(t *testing.T) { func TestRebuildIndexDamage(t *testing.T) {
testRebuildIndex(t, func(r restic.Backend) (restic.Backend, error) { testRebuildIndex(t, func(r backend.Backend) (backend.Backend, error) {
return &indexErrorBackend{ return &indexErrorBackend{
Backend: r, Backend: r,
}, nil }, nil
@ -109,11 +110,11 @@ func TestRebuildIndexDamage(t *testing.T) {
} }
type appendOnlyBackend struct { type appendOnlyBackend struct {
restic.Backend backend.Backend
} }
// called via repo.Backend().Remove() // called via repo.Backend().Remove()
func (b *appendOnlyBackend) Remove(_ context.Context, h restic.Handle) error { func (b *appendOnlyBackend) Remove(_ context.Context, h backend.Handle) error {
return errors.Errorf("Failed to remove %v", h) return errors.Errorf("Failed to remove %v", h)
} }
@ -127,7 +128,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
err := withRestoreGlobalOptions(func() error { err := withRestoreGlobalOptions(func() error {
globalOptions.stdout = io.Discard globalOptions.stdout = io.Discard
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return &appendOnlyBackend{r}, nil return &appendOnlyBackend{r}, nil
} }
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts) return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts)

3
cmd/restic/cmd_repair_packs.go
View File

@ -5,6 +5,7 @@ import (
"io" "io"
"os" "os"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -84,7 +85,7 @@ func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repo
return errors.Fatalf("%s", err) return errors.Fatalf("%s", err)
} }
err = repo.Backend().Load(ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error { err = repo.Backend().Load(ctx, backend.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error {
_, err := f.Seek(0, 0) _, err := f.Seek(0, 0)
if err != nil { if err != nil {
return err return err

4
cmd/restic/cmd_rewrite.go
View File

@ -128,7 +128,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
if dryRun { if dryRun {
Verbosef("would delete empty snapshot\n") Verbosef("would delete empty snapshot\n")
} else { } else {
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
if err = repo.Backend().Remove(ctx, h); err != nil { if err = repo.Backend().Remove(ctx, h); err != nil {
return false, err return false, err
} }
@ -170,7 +170,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
Verbosef("saved new snapshot %v\n", id.Str()) Verbosef("saved new snapshot %v\n", id.Str())
if forget { if forget {
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
if err = repo.Backend().Remove(ctx, h); err != nil { if err = repo.Backend().Remove(ctx, h); err != nil {
return false, err return false, err
} }

3
cmd/restic/cmd_tag.go
View File

@ -5,6 +5,7 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
@ -85,7 +86,7 @@ func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Sna
debug.Log("new snapshot saved as %v", id) debug.Log("new snapshot saved as %v", id)
// Remove the old snapshot. // Remove the old snapshot.
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
if err = repo.Backend().Remove(ctx, h); err != nil { if err = repo.Backend().Remove(ctx, h); err != nil {
return false, err return false, err
} }

3
cmd/restic/delete.go
View File

@ -5,6 +5,7 @@ import (
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
) )
@ -45,7 +46,7 @@ func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, rep
for i := 0; i < int(workerCount); i++ { for i := 0; i < int(workerCount); i++ {
wg.Go(func() error { wg.Go(func() error {
for id := range fileChan { for id := range fileChan {
h := restic.Handle{Type: fileType, Name: id.String()} h := backend.Handle{Type: fileType, Name: id.String()}
err := repo.Backend().Remove(ctx, h) err := repo.Backend().Remove(ctx, h)
if err != nil { if err != nil {
if !gopts.JSON { if !gopts.JSON {

2
cmd/restic/find.go
View File

@ -29,7 +29,7 @@ func initSingleSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter)
} }
// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. // FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, f *restic.SnapshotFilter, snapshotIDs []string) <-chan *restic.Snapshot { func FindFilteredSnapshots(ctx context.Context, be backend.Lister, loader restic.LoaderUnpacked, f *restic.SnapshotFilter, snapshotIDs []string) <-chan *restic.Snapshot {
out := make(chan *restic.Snapshot) out := make(chan *restic.Snapshot)
go func() { go func() {
defer close(out) defer close(out)

12
cmd/restic/global.go
View File

@ -48,7 +48,7 @@ var version = "0.16.1-dev (compiled manually)"
// TimeFormat is the format used for all timestamps printed by restic. // TimeFormat is the format used for all timestamps printed by restic.
const TimeFormat = "2006-01-02 15:04:05" const TimeFormat = "2006-01-02 15:04:05"
type backendWrapper func(r restic.Backend) (restic.Backend, error) type backendWrapper func(r backend.Backend) (backend.Backend, error)
// GlobalOptions hold all global options for restic. // GlobalOptions hold all global options for restic.
type GlobalOptions struct { type GlobalOptions struct {
@ -553,7 +553,7 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi
func parseConfig(loc location.Location, opts options.Options) (interface{}, error) { func parseConfig(loc location.Location, opts options.Options) (interface{}, error) {
cfg := loc.Config cfg := loc.Config
if cfg, ok := cfg.(restic.ApplyEnvironmenter); ok { if cfg, ok := cfg.(backend.ApplyEnvironmenter); ok {
cfg.ApplyEnvironment("") cfg.ApplyEnvironment("")
} }
@ -568,14 +568,14 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
} }
// Open the backend specified by a location config. // Open the backend specified by a location config.
func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) {
debug.Log("parsing location %v", location.StripPassword(gopts.backends, s)) debug.Log("parsing location %v", location.StripPassword(gopts.backends, s))
loc, err := location.Parse(gopts.backends, s) loc, err := location.Parse(gopts.backends, s)
if err != nil { if err != nil {
return nil, errors.Fatalf("parsing repository location failed: %v", err) return nil, errors.Fatalf("parsing repository location failed: %v", err)
} }
var be restic.Backend var be backend.Backend
cfg, err := parseConfig(loc, opts) cfg, err := parseConfig(loc, opts)
if err != nil { if err != nil {
@ -613,7 +613,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
} }
// check if config is there // check if config is there
fi, err := be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) fi, err := be.Stat(ctx, backend.Handle{Type: restic.ConfigFile})
if err != nil { if err != nil {
return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(gopts.backends, s)) return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(gopts.backends, s))
} }
@ -626,7 +626,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
} }
// Create the backend specified by URI. // Create the backend specified by URI.
func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) {
debug.Log("parsing location %v", location.StripPassword(gopts.backends, s)) debug.Log("parsing location %v", location.StripPassword(gopts.backends, s))
loc, err := location.Parse(gopts.backends, s) loc, err := location.Parse(gopts.backends, s)
if err != nil { if err != nil {

7
cmd/restic/integration_helpers_test.go
View File

@ -12,6 +12,7 @@ import (
"sync" "sync"
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/retry" "github.com/restic/restic/internal/backend/retry"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options" "github.com/restic/restic/internal/options"
@ -205,7 +206,7 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
extended: make(options.Options), extended: make(options.Options),
// replace this hook with "nil" if listing a filetype more than once is necessary // replace this hook with "nil" if listing a filetype more than once is necessary
backendTestHook: func(r restic.Backend) (restic.Backend, error) { return newOrderedListOnceBackend(r), nil }, backendTestHook: func(r backend.Backend) (backend.Backend, error) { return newOrderedListOnceBackend(r), nil },
// start with default set of backends // start with default set of backends
backends: globalOptions.backends, backends: globalOptions.backends,
} }
@ -249,7 +250,7 @@ func removePacks(gopts GlobalOptions, t testing.TB, remove restic.IDSet) {
rtest.OK(t, err) rtest.OK(t, err)
for id := range remove { for id := range remove {
rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})) rtest.OK(t, r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()}))
} }
} }
@ -272,7 +273,7 @@ func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, rem
if treePacks.Has(id) != removeTreePacks || keep.Has(id) { if treePacks.Has(id) != removeTreePacks || keep.Has(id) {
return nil return nil
} }
return r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}) return r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()})
})) }))
} }

19
cmd/restic/integration_test.go
View File

@ -8,6 +8,7 @@ import (
"path/filepath" "path/filepath"
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
@ -41,12 +42,12 @@ func TestCheckRestoreNoLock(t *testing.T) {
// backends (like e.g. Amazon S3) as the second listing may be inconsistent to what // backends (like e.g. Amazon S3) as the second listing may be inconsistent to what
// is expected by the first listing + some operations. // is expected by the first listing + some operations.
type listOnceBackend struct { type listOnceBackend struct {
restic.Backend backend.Backend
listedFileType map[restic.FileType]bool listedFileType map[restic.FileType]bool
strictOrder bool strictOrder bool
} }
func newListOnceBackend(be restic.Backend) *listOnceBackend { func newListOnceBackend(be backend.Backend) *listOnceBackend {
return &listOnceBackend{ return &listOnceBackend{
Backend: be, Backend: be,
listedFileType: make(map[restic.FileType]bool), listedFileType: make(map[restic.FileType]bool),
@ -54,7 +55,7 @@ func newListOnceBackend(be restic.Backend) *listOnceBackend {
} }
} }
func newOrderedListOnceBackend(be restic.Backend) *listOnceBackend { func newOrderedListOnceBackend(be backend.Backend) *listOnceBackend {
return &listOnceBackend{ return &listOnceBackend{
Backend: be, Backend: be,
listedFileType: make(map[restic.FileType]bool), listedFileType: make(map[restic.FileType]bool),
@ -62,7 +63,7 @@ func newOrderedListOnceBackend(be restic.Backend) *listOnceBackend {
} }
} }
func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func(backend.FileInfo) error) error {
if t != restic.LockFile && be.listedFileType[t] { if t != restic.LockFile && be.listedFileType[t] {
return errors.Errorf("tried listing type %v the second time", t) return errors.Errorf("tried listing type %v the second time", t)
} }
@ -77,7 +78,7 @@ func TestListOnce(t *testing.T) {
env, cleanup := withTestEnvironment(t) env, cleanup := withTestEnvironment(t)
defer cleanup() defer cleanup()
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return newListOnceBackend(r), nil return newListOnceBackend(r), nil
} }
pruneOpts := PruneOptions{MaxUnused: "0"} pruneOpts := PruneOptions{MaxUnused: "0"}
@ -104,10 +105,10 @@ func (r *writeToOnly) WriteTo(w io.Writer) (int64, error) {
} }
type onlyLoadWithWriteToBackend struct { type onlyLoadWithWriteToBackend struct {
restic.Backend backend.Backend
} }
func (be *onlyLoadWithWriteToBackend) Load(ctx context.Context, h restic.Handle, func (be *onlyLoadWithWriteToBackend) Load(ctx context.Context, h backend.Handle,
length int, offset int64, fn func(rd io.Reader) error) error { length int, offset int64, fn func(rd io.Reader) error) error {
return be.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { return be.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
@ -120,7 +121,7 @@ func TestBackendLoadWriteTo(t *testing.T) {
defer cleanup() defer cleanup()
// setup backend which only works if it's WriteTo method is correctly propagated upwards // setup backend which only works if it's WriteTo method is correctly propagated upwards
env.gopts.backendInnerTestHook = func(r restic.Backend) (restic.Backend, error) { env.gopts.backendInnerTestHook = func(r backend.Backend) (backend.Backend, error) {
return &onlyLoadWithWriteToBackend{Backend: r}, nil return &onlyLoadWithWriteToBackend{Backend: r}, nil
} }
@ -140,7 +141,7 @@ func TestFindListOnce(t *testing.T) {
env, cleanup := withTestEnvironment(t) env, cleanup := withTestEnvironment(t)
defer cleanup() defer cleanup()
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return newListOnceBackend(r), nil return newListOnceBackend(r), nil
} }

7
cmd/restic/lock.go
View File

@ -6,6 +6,7 @@ import (
"sync" "sync"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -131,7 +132,7 @@ type refreshLockRequest struct {
result chan bool result chan bool
} }
func refreshLocks(ctx context.Context, backend restic.Backend, lockInfo *lockContext, refreshed chan<- struct{}, forceRefresh <-chan refreshLockRequest) { func refreshLocks(ctx context.Context, backend backend.Backend, lockInfo *lockContext, refreshed chan<- struct{}, forceRefresh <-chan refreshLockRequest) {
debug.Log("start") debug.Log("start")
lock := lockInfo.lock lock := lockInfo.lock
ticker := time.NewTicker(refreshInterval) ticker := time.NewTicker(refreshInterval)
@ -257,8 +258,8 @@ func monitorLockRefresh(ctx context.Context, lockInfo *lockContext, refreshed <-
} }
} }
func tryRefreshStaleLock(ctx context.Context, backend restic.Backend, lock *restic.Lock, cancel context.CancelFunc) bool { func tryRefreshStaleLock(ctx context.Context, be backend.Backend, lock *restic.Lock, cancel context.CancelFunc) bool {
freeze := restic.AsBackend[restic.FreezeBackend](backend) freeze := backend.AsBackend[backend.FreezeBackend](be)
if freeze != nil { if freeze != nil {
debug.Log("freezing backend") debug.Log("freezing backend")
freeze.Freeze() freeze.Freeze()

19
cmd/restic/lock_test.go
View File

@ -9,6 +9,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
@ -104,11 +105,11 @@ func TestLockConflict(t *testing.T) {
} }
type writeOnceBackend struct { type writeOnceBackend struct {
restic.Backend backend.Backend
written bool written bool
} }
func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (b *writeOnceBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if b.written { if b.written {
return fmt.Errorf("fail after first write") return fmt.Errorf("fail after first write")
} }
@ -117,7 +118,7 @@ func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic.
} }
func TestLockFailedRefresh(t *testing.T) { func TestLockFailedRefresh(t *testing.T) {
repo, cleanup, env := openLockTestRepo(t, func(r restic.Backend) (restic.Backend, error) { repo, cleanup, env := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) {
return &writeOnceBackend{Backend: r}, nil return &writeOnceBackend{Backend: r}, nil
}) })
defer cleanup() defer cleanup()
@ -143,11 +144,11 @@ func TestLockFailedRefresh(t *testing.T) {
} }
type loggingBackend struct { type loggingBackend struct {
restic.Backend backend.Backend
t *testing.T t *testing.T
} }
func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (b *loggingBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
b.t.Logf("save %v @ %v", h, time.Now()) b.t.Logf("save %v @ %v", h, time.Now())
err := b.Backend.Save(ctx, h, rd) err := b.Backend.Save(ctx, h, rd)
b.t.Logf("save finished %v @ %v", h, time.Now()) b.t.Logf("save finished %v @ %v", h, time.Now())
@ -155,7 +156,7 @@ func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re
} }
func TestLockSuccessfulRefresh(t *testing.T) { func TestLockSuccessfulRefresh(t *testing.T) {
repo, cleanup, env := openLockTestRepo(t, func(r restic.Backend) (restic.Backend, error) { repo, cleanup, env := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) {
return &loggingBackend{ return &loggingBackend{
Backend: r, Backend: r,
t: t, t: t,
@ -193,12 +194,12 @@ func TestLockSuccessfulRefresh(t *testing.T) {
} }
type slowBackend struct { type slowBackend struct {
restic.Backend backend.Backend
m sync.Mutex m sync.Mutex
sleep time.Duration sleep time.Duration
} }
func (b *slowBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (b *slowBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
b.m.Lock() b.m.Lock()
sleep := b.sleep sleep := b.sleep
b.m.Unlock() b.m.Unlock()
@ -208,7 +209,7 @@ func (b *slowBackend) Save(ctx context.Context, h restic.Handle, rd restic.Rewin
func TestLockSuccessfulStaleRefresh(t *testing.T) { func TestLockSuccessfulStaleRefresh(t *testing.T) {
var sb *slowBackend var sb *slowBackend
repo, cleanup, env := openLockTestRepo(t, func(r restic.Backend) (restic.Backend, error) { repo, cleanup, env := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) {
sb = &slowBackend{Backend: r} sb = &slowBackend{Backend: r}
return sb, nil return sb, nil
}) })

13
internal/archiver/archiver_test.go
View File

@ -15,6 +15,7 @@ import (
"time" "time"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
@ -1842,26 +1843,26 @@ func TestArchiverErrorReporting(t *testing.T) {
} }
type noCancelBackend struct { type noCancelBackend struct {
restic.Backend backend.Backend
} }
func (c *noCancelBackend) Remove(_ context.Context, h restic.Handle) error { func (c *noCancelBackend) Remove(_ context.Context, h backend.Handle) error {
return c.Backend.Remove(context.Background(), h) return c.Backend.Remove(context.Background(), h)
} }
func (c *noCancelBackend) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) error { func (c *noCancelBackend) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) error {
return c.Backend.Save(context.Background(), h, rd) return c.Backend.Save(context.Background(), h, rd)
} }
func (c *noCancelBackend) Load(_ context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { func (c *noCancelBackend) Load(_ context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return c.Backend.Load(context.Background(), h, length, offset, fn) return c.Backend.Load(context.Background(), h, length, offset, fn)
} }
func (c *noCancelBackend) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) { func (c *noCancelBackend) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) {
return c.Backend.Stat(context.Background(), h) return c.Backend.Stat(context.Background(), h)
} }
func (c *noCancelBackend) List(_ context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { func (c *noCancelBackend) List(_ context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
return c.Backend.List(context.Background(), t, fn) return c.Backend.List(context.Background(), t, fn)
} }

26
internal/backend/azure/azure.go
View File

@ -12,12 +12,12 @@ import (
"path" "path"
"strings" "strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
@ -43,7 +43,7 @@ const saveLargeSize = 256 * 1024 * 1024
const defaultListMaxItems = 5000 const defaultListMaxItems = 5000
// make sure that *Backend implements backend.Backend // make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{} var _ backend.Backend = &Backend{}
func NewFactory() location.Factory { func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("azure", ParseConfig, location.NoPassword, Create, Open) return location.NewHTTPBackendFactory("azure", ParseConfig, location.NoPassword, Create, Open)
@ -197,7 +197,7 @@ func (be *Backend) Path() string {
} }
// Save stores data in the backend at the handle. // Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h) objName := be.Filename(h)
debug.Log("InsertObject(%v, %v)", be.cfg.AccountName, objName) debug.Log("InsertObject(%v, %v)", be.cfg.AccountName, objName)
@ -214,7 +214,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
return err return err
} }
func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.RewindReader) error { func (be *Backend) saveSmall(ctx context.Context, objName string, rd backend.RewindReader) error {
blockBlobClient := be.container.NewBlockBlobClient(objName) blockBlobClient := be.container.NewBlockBlobClient(objName)
// upload it as a new "block", use the base64 hash for the ID // upload it as a new "block", use the base64 hash for the ID
@ -239,7 +239,7 @@ func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.Rewi
return errors.Wrap(err, "CommitBlockList") return errors.Wrap(err, "CommitBlockList")
} }
func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.RewindReader) error { func (be *Backend) saveLarge(ctx context.Context, objName string, rd backend.RewindReader) error {
blockBlobClient := be.container.NewBlockBlobClient(objName) blockBlobClient := be.container.NewBlockBlobClient(objName)
buf := make([]byte, 100*1024*1024) buf := make([]byte, 100*1024*1024)
@ -294,11 +294,11 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
// Load runs fn with a reader that yields the contents of the file at h at the // Load runs fn with a reader that yields the contents of the file at h at the
// given offset. // given offset.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
} }
func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
objName := be.Filename(h) objName := be.Filename(h)
blockBlobClient := be.container.NewBlobClient(objName) blockBlobClient := be.container.NewBlobClient(objName)
@ -317,17 +317,17 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int,
} }
// Stat returns information about a blob. // Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
objName := be.Filename(h) objName := be.Filename(h)
blobClient := be.container.NewBlobClient(objName) blobClient := be.container.NewBlobClient(objName)
props, err := blobClient.GetProperties(ctx, nil) props, err := blobClient.GetProperties(ctx, nil)
if err != nil { if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "blob.GetProperties") return backend.FileInfo{}, errors.Wrap(err, "blob.GetProperties")
} }
fi := restic.FileInfo{ fi := backend.FileInfo{
Size: *props.ContentLength, Size: *props.ContentLength,
Name: h.Name, Name: h.Name,
} }
@ -335,7 +335,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo,
} }
// Remove removes the blob with the given name and type. // Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h) objName := be.Filename(h)
blob := be.container.NewBlobClient(objName) blob := be.container.NewBlobClient(objName)
@ -350,7 +350,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an // List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it. // error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, _ := be.Basedir(t) prefix, _ := be.Basedir(t)
// make sure prefix ends with a slash // make sure prefix ends with a slash
@ -381,7 +381,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue continue
} }
fi := restic.FileInfo{ fi := backend.FileInfo{
Name: path.Base(m), Name: path.Base(m),
Size: *item.Properties.ContentLength, Size: *item.Properties.ContentLength,
} }

4
internal/backend/azure/azure_test.go
View File

@ -122,11 +122,11 @@ func TestUploadLargeFile(t *testing.T) {
data := rtest.Random(23, 300*1024*1024) data := rtest.Random(23, 300*1024*1024)
id := restic.Hash(data) id := restic.Hash(data)
h := restic.Handle{Name: id.String(), Type: restic.PackFile} h := backend.Handle{Name: id.String(), Type: backend.PackFile}
t.Logf("hash of %d bytes: %v", len(data), id) t.Logf("hash of %d bytes: %v", len(data), id)
err = be.Save(ctx, h, restic.NewByteReader(data, be.Hasher())) err = be.Save(ctx, h, backend.NewByteReader(data, be.Hasher()))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

4
internal/backend/azure/config.go
View File

@ -5,9 +5,9 @@ import (
"path" "path"
"strings" "strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options" "github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
) )
// Config contains all configuration necessary to connect to an azure compatible // Config contains all configuration necessary to connect to an azure compatible
@ -57,7 +57,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil return &cfg, nil
} }
var _ restic.ApplyEnvironmenter = &Config{} var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config. // ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) { func (cfg *Config) ApplyEnvironment(prefix string) {

28
internal/backend/b2/b2.go
View File

@ -9,12 +9,12 @@ import (
"sync" "sync"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/Backblaze/blazer/b2" "github.com/Backblaze/blazer/b2"
"github.com/Backblaze/blazer/base" "github.com/Backblaze/blazer/base"
@ -34,8 +34,8 @@ type b2Backend struct {
// Billing happens in 1000 item granlarity, but we are more interested in reducing the number of network round trips // Billing happens in 1000 item granlarity, but we are more interested in reducing the number of network round trips
const defaultListMaxItems = 10 * 1000 const defaultListMaxItems = 10 * 1000
// ensure statically that *b2Backend implements restic.Backend. // ensure statically that *b2Backend implements backend.Backend.
var _ restic.Backend = &b2Backend{} var _ backend.Backend = &b2Backend{}
func NewFactory() location.Factory { func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("b2", ParseConfig, location.NoPassword, Create, Open) return location.NewHTTPBackendFactory("b2", ParseConfig, location.NoPassword, Create, Open)
@ -85,7 +85,7 @@ func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Clien
} }
// Open opens a connection to the B2 service. // Open opens a connection to the B2 service.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
debug.Log("cfg %#v", cfg) debug.Log("cfg %#v", cfg)
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
@ -118,7 +118,7 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend
// Create opens a connection to the B2 service. If the bucket does not exist yet, // Create opens a connection to the B2 service. If the bucket does not exist yet,
// it is created. // it is created.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
debug.Log("cfg %#v", cfg) debug.Log("cfg %#v", cfg)
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
@ -188,14 +188,14 @@ func (be *b2Backend) IsNotExist(err error) bool {
// Load runs fn with a reader that yields the contents of the file at h at the // Load runs fn with a reader that yields the contents of the file at h at the
// given offset. // given offset.
func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { func (be *b2Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
} }
func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { func (be *b2Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
name := be.Layout.Filename(h) name := be.Layout.Filename(h)
obj := be.bucket.Object(name) obj := be.bucket.Object(name)
@ -213,7 +213,7 @@ func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int
} }
// Save stores data in the backend at the handle. // Save stores data in the backend at the handle.
func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (be *b2Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
@ -237,18 +237,18 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind
} }
// Stat returns information about a blob. // Stat returns information about a blob.
func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { func (be *b2Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
name := be.Filename(h) name := be.Filename(h)
obj := be.bucket.Object(name) obj := be.bucket.Object(name)
info, err := obj.Attrs(ctx) info, err := obj.Attrs(ctx)
if err != nil { if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Stat") return backend.FileInfo{}, errors.Wrap(err, "Stat")
} }
return restic.FileInfo{Size: info.Size, Name: h.Name}, nil return backend.FileInfo{Size: info.Size, Name: h.Name}, nil
} }
// Remove removes the blob with the given name and type. // Remove removes the blob with the given name and type.
func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error { func (be *b2Backend) Remove(ctx context.Context, h backend.Handle) error {
// the retry backend will also repeat the remove method up to 10 times // the retry backend will also repeat the remove method up to 10 times
for i := 0; i < 3; i++ { for i := 0; i < 3; i++ {
obj := be.bucket.Object(be.Filename(h)) obj := be.bucket.Object(be.Filename(h))
@ -284,7 +284,7 @@ func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
} }
// List returns a channel that yields all names of blobs of type t. // List returns a channel that yields all names of blobs of type t.
func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { func (be *b2Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
@ -299,7 +299,7 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic
return err return err
} }
fi := restic.FileInfo{ fi := backend.FileInfo{
Name: path.Base(obj.Name()), Name: path.Base(obj.Name()),
Size: attrs.Size, Size: attrs.Size,
} }

4
internal/backend/b2/config.go
View File

@ -6,9 +6,9 @@ import (
"regexp" "regexp"
"strings" "strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options" "github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
) )
// Config contains all configuration necessary to connect to an b2 compatible // Config contains all configuration necessary to connect to an b2 compatible
@ -82,7 +82,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil return &cfg, nil
} }
var _ restic.ApplyEnvironmenter = &Config{} var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config. // ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) { func (cfg *Config) ApplyEnvironment(prefix string) {

11
internal/restic/backend.go → internal/backend/backend.go
View File

@ -1,4 +1,4 @@
package restic package backend
import ( import (
"context" "context"
@ -70,7 +70,7 @@ type Backend interface {
Delete(ctx context.Context) error Delete(ctx context.Context) error
} }
type BackendUnwrapper interface { type Unwrapper interface {
// Unwrap returns the underlying backend or nil if there is none. // Unwrap returns the underlying backend or nil if there is none.
Unwrap() Backend Unwrap() Backend
} }
@ -81,7 +81,7 @@ func AsBackend[B Backend](b Backend) B {
return be return be
} }
if be, ok := b.(BackendUnwrapper); ok { if be, ok := b.(Unwrapper); ok {
b = be.Unwrap() b = be.Unwrap()
} else { } else {
// not the backend we're looking for // not the backend we're looking for
@ -110,3 +110,8 @@ type FileInfo struct {
type ApplyEnvironmenter interface { type ApplyEnvironmenter interface {
ApplyEnvironment(prefix string) ApplyEnvironment(prefix string)
} }
// Lister allows listing files in a backend.
type Lister interface {
List(context.Context, FileType, func(FileInfo) error) error
}

38
internal/backend/backend_test.go Normal file
View File

@ -0,0 +1,38 @@
package backend_test
import (
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/test"
)
type testBackend struct {
backend.Backend
}
func (t *testBackend) Unwrap() backend.Backend {
return nil
}
type otherTestBackend struct {
backend.Backend
}
func (t *otherTestBackend) Unwrap() backend.Backend {
return t.Backend
}
func TestAsBackend(t *testing.T) {
other := otherTestBackend{}
test.Assert(t, backend.AsBackend[*testBackend](other) == nil, "otherTestBackend is not a testBackend backend")
testBe := &testBackend{}
test.Assert(t, backend.AsBackend[*testBackend](testBe) == testBe, "testBackend was not returned")
wrapper := &otherTestBackend{Backend: testBe}
test.Assert(t, backend.AsBackend[*testBackend](wrapper) == testBe, "failed to unwrap testBackend backend")
wrapper.Backend = other
test.Assert(t, backend.AsBackend[*testBackend](wrapper) == nil, "a wrapped otherTestBackend is not a testBackend")
}

20
internal/backend/dryrun/dry_backend.go
View File

@ -5,8 +5,8 @@ import (
"hash" "hash"
"io" "io"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
) )
// Backend passes reads through to an underlying layer and accepts writes, but // Backend passes reads through to an underlying layer and accepts writes, but
@ -15,20 +15,20 @@ import (
// the repo and does normal operations else. // the repo and does normal operations else.
// This is used for `backup --dry-run`. // This is used for `backup --dry-run`.
type Backend struct { type Backend struct {
b restic.Backend b backend.Backend
} }
// statically ensure that Backend implements restic.Backend. // statically ensure that Backend implements backend.Backend.
var _ restic.Backend = &Backend{} var _ backend.Backend = &Backend{}
func New(be restic.Backend) *Backend { func New(be backend.Backend) *Backend {
b := &Backend{b: be} b := &Backend{b: be}
debug.Log("created new dry backend") debug.Log("created new dry backend")
return b return b
} }
// Save adds new Data to the backend. // Save adds new Data to the backend.
func (be *Backend) Save(_ context.Context, h restic.Handle, _ restic.RewindReader) error { func (be *Backend) Save(_ context.Context, h backend.Handle, _ backend.RewindReader) error {
if err := h.Valid(); err != nil { if err := h.Valid(); err != nil {
return err return err
} }
@ -38,7 +38,7 @@ func (be *Backend) Save(_ context.Context, h restic.Handle, _ restic.RewindReade
} }
// Remove deletes a file from the backend. // Remove deletes a file from the backend.
func (be *Backend) Remove(_ context.Context, _ restic.Handle) error { func (be *Backend) Remove(_ context.Context, _ backend.Handle) error {
return nil return nil
} }
@ -72,14 +72,14 @@ func (be *Backend) IsNotExist(err error) bool {
return be.b.IsNotExist(err) return be.b.IsNotExist(err)
} }
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
return be.b.List(ctx, t, fn) return be.b.List(ctx, t, fn)
} }
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error { func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(io.Reader) error) error {
return be.b.Load(ctx, h, length, offset, fn) return be.b.Load(ctx, h, length, offset, fn)
} }
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
return be.b.Stat(ctx, h) return be.b.Stat(ctx, h)
} }

16
internal/backend/dryrun/dry_backend_test.go
View File

@ -8,16 +8,16 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/dryrun" "github.com/restic/restic/internal/backend/dryrun"
"github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/backend/mem"
) )
// make sure that Backend implements backend.Backend // make sure that Backend implements backend.Backend
var _ restic.Backend = &dryrun.Backend{} var _ backend.Backend = &dryrun.Backend{}
func newBackends() (*dryrun.Backend, restic.Backend) { func newBackends() (*dryrun.Backend, backend.Backend) {
m := mem.New() m := mem.New()
return dryrun.New(m), m return dryrun.New(m), m
} }
@ -30,7 +30,7 @@ func TestDry(t *testing.T) {
// won't pass. Instead, perform a series of operations over the backend, testing the state // won't pass. Instead, perform a series of operations over the backend, testing the state
// at each step. // at each step.
steps := []struct { steps := []struct {
be restic.Backend be backend.Backend
op string op string
fname string fname string
content string content string
@ -61,13 +61,13 @@ func TestDry(t *testing.T) {
for i, step := range steps { for i, step := range steps {
var err error var err error
handle := restic.Handle{Type: restic.PackFile, Name: step.fname} handle := backend.Handle{Type: backend.PackFile, Name: step.fname}
switch step.op { switch step.op {
case "save": case "save":
err = step.be.Save(ctx, handle, restic.NewByteReader([]byte(step.content), step.be.Hasher())) err = step.be.Save(ctx, handle, backend.NewByteReader([]byte(step.content), step.be.Hasher()))
case "list": case "list":
fileList := []string{} fileList := []string{}
err = step.be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { err = step.be.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
fileList = append(fileList, fi.Name) fileList = append(fileList, fi.Name)
return nil return nil
}) })
@ -86,7 +86,7 @@ func TestDry(t *testing.T) {
case "remove": case "remove":
err = step.be.Remove(ctx, handle) err = step.be.Remove(ctx, handle)
case "stat": case "stat":
var fi restic.FileInfo var fi backend.FileInfo
fi, err = step.be.Stat(ctx, handle) fi, err = step.be.Stat(ctx, handle)
if err == nil { if err == nil {
fis := fmt.Sprintf("%s %d", fi.Name, fi.Size) fis := fmt.Sprintf("%s %d", fi.Name, fi.Size)

2
internal/restic/file.go → internal/backend/file.go
View File

@ -1,4 +1,4 @@
package restic package backend
import ( import (
"fmt" "fmt"

2
internal/restic/file_test.go → internal/backend/file_test.go
View File

@ -1,4 +1,4 @@
package restic package backend
import ( import (
"testing" "testing"

4
internal/backend/gs/config.go
View File

@ -5,9 +5,9 @@ import (
"path" "path"
"strings" "strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options" "github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
) )
// Config contains all configuration necessary to connect to a Google Cloud Storage // Config contains all configuration necessary to connect to a Google Cloud Storage
@ -59,7 +59,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil return &cfg, nil
} }
var _ restic.ApplyEnvironmenter = &Config{} var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config. // ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) { func (cfg *Config) ApplyEnvironment(prefix string) {

28
internal/backend/gs/gs.go
View File

@ -13,11 +13,11 @@ import (
"cloud.google.com/go/storage" "cloud.google.com/go/storage"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
"golang.org/x/oauth2" "golang.org/x/oauth2"
"golang.org/x/oauth2/google" "golang.org/x/oauth2/google"
@ -45,8 +45,8 @@ type Backend struct {
layout.Layout layout.Layout
} }
// Ensure that *Backend implements restic.Backend. // Ensure that *Backend implements backend.Backend.
var _ restic.Backend = &Backend{} var _ backend.Backend = &Backend{}
func NewFactory() location.Factory { func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open) return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open)
@ -122,7 +122,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
} }
// Open opens the gs backend at the specified bucket. // Open opens the gs backend at the specified bucket.
func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { func Open(_ context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
return open(cfg, rt) return open(cfg, rt)
} }
@ -131,7 +131,7 @@ func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend,
// //
// The service account must have the "storage.buckets.create" permission to // The service account must have the "storage.buckets.create" permission to
// create a bucket the does not yet exist. // create a bucket the does not yet exist.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
be, err := open(cfg, rt) be, err := open(cfg, rt)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "open") return nil, errors.Wrap(err, "open")
@ -203,7 +203,7 @@ func (be *Backend) Path() string {
} }
// Save stores data in the backend at the handle. // Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h) objName := be.Filename(h)
// Set chunk size to zero to disable resumable uploads. // Set chunk size to zero to disable resumable uploads.
@ -253,14 +253,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
// Load runs fn with a reader that yields the contents of the file at h at the // Load runs fn with a reader that yields the contents of the file at h at the
// given offset. // given offset.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
} }
func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
if length == 0 { if length == 0 {
// negative length indicates read till end to GCS lib // negative length indicates read till end to GCS lib
length = -1 length = -1
@ -277,20 +277,20 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int,
} }
// Stat returns information about a blob. // Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h) objName := be.Filename(h)
attr, err := be.bucket.Object(objName).Attrs(ctx) attr, err := be.bucket.Object(objName).Attrs(ctx)
if err != nil { if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "service.Objects.Get") return backend.FileInfo{}, errors.Wrap(err, "service.Objects.Get")
} }
return restic.FileInfo{Size: attr.Size, Name: h.Name}, nil return backend.FileInfo{Size: attr.Size, Name: h.Name}, nil
} }
// Remove removes the blob with the given name and type. // Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h) objName := be.Filename(h)
err := be.bucket.Object(objName).Delete(ctx) err := be.bucket.Object(objName).Delete(ctx)
@ -304,7 +304,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an // List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it. // error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, _ := be.Basedir(t) prefix, _ := be.Basedir(t)
// make sure prefix ends with a slash // make sure prefix ends with a slash
@ -330,7 +330,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue continue
} }
fi := restic.FileInfo{ fi := backend.FileInfo{
Name: path.Base(m), Name: path.Base(m),
Size: int64(attrs.Size), Size: int64(attrs.Size),
} }

11
internal/backend/layout/layout.go
View File

@ -7,6 +7,7 @@ import (
"path/filepath" "path/filepath"
"regexp" "regexp"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/fs"
@ -15,9 +16,9 @@ import (
// Layout computes paths for file name storage. // Layout computes paths for file name storage.
type Layout interface { type Layout interface {
Filename(restic.Handle) string Filename(backend.Handle) string
Dirname(restic.Handle) string Dirname(backend.Handle) string
Basedir(restic.FileType) (dir string, subdirs bool) Basedir(backend.FileType) (dir string, subdirs bool)
Paths() []string Paths() []string
Name() string Name() string
} }
@ -102,13 +103,13 @@ func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, err
} }
// key file in the "keys" dir (DefaultLayout) // key file in the "keys" dir (DefaultLayout)
foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile])) foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[backend.KeyFile]))
if err != nil { if err != nil {
return nil, err return nil, err
} }
// key file in the "key" dir (S3LegacyLayout) // key file in the "key" dir (S3LegacyLayout)
foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile])) foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[backend.KeyFile]))
if err != nil { if err != nil {
return nil, err return nil, err
} }

28
internal/backend/layout/layout_default.go
View File

@ -3,7 +3,7 @@ package layout
import ( import (
"encoding/hex" "encoding/hex"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/backend"
) )
// DefaultLayout implements the default layout for local and sftp backends, as // DefaultLayout implements the default layout for local and sftp backends, as
@ -15,12 +15,12 @@ type DefaultLayout struct {
Join func(...string) string Join func(...string) string
} }
var defaultLayoutPaths = map[restic.FileType]string{ var defaultLayoutPaths = map[backend.FileType]string{
restic.PackFile: "data", backend.PackFile: "data",
restic.SnapshotFile: "snapshots", backend.SnapshotFile: "snapshots",
restic.IndexFile: "index", backend.IndexFile: "index",
restic.LockFile: "locks", backend.LockFile: "locks",
restic.KeyFile: "keys", backend.KeyFile: "keys",
} }
func (l *DefaultLayout) String() string { func (l *DefaultLayout) String() string {
@ -33,10 +33,10 @@ func (l *DefaultLayout) Name() string {
} }
// Dirname returns the directory path for a given file type and name. // Dirname returns the directory path for a given file type and name.
func (l *DefaultLayout) Dirname(h restic.Handle) string { func (l *DefaultLayout) Dirname(h backend.Handle) string {
p := defaultLayoutPaths[h.Type] p := defaultLayoutPaths[h.Type]
if h.Type == restic.PackFile && len(h.Name) > 2 { if h.Type == backend.PackFile && len(h.Name) > 2 {
p = l.Join(p, h.Name[:2]) + "/" p = l.Join(p, h.Name[:2]) + "/"
} }
@ -44,9 +44,9 @@ func (l *DefaultLayout) Dirname(h restic.Handle) string {
} }
// Filename returns a path to a file, including its name. // Filename returns a path to a file, including its name.
func (l *DefaultLayout) Filename(h restic.Handle) string { func (l *DefaultLayout) Filename(h backend.Handle) string {
name := h.Name name := h.Name
if h.Type == restic.ConfigFile { if h.Type == backend.ConfigFile {
return l.Join(l.Path, "config") return l.Join(l.Path, "config")
} }
@ -62,15 +62,15 @@ func (l *DefaultLayout) Paths() (dirs []string) {
// also add subdirs // also add subdirs
for i := 0; i < 256; i++ { for i := 0; i < 256; i++ {
subdir := hex.EncodeToString([]byte{byte(i)}) subdir := hex.EncodeToString([]byte{byte(i)})
dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[restic.PackFile], subdir)) dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[backend.PackFile], subdir))
} }
return dirs return dirs
} }
// Basedir returns the base dir name for type t. // Basedir returns the base dir name for type t.
func (l *DefaultLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) { func (l *DefaultLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) {
if t == restic.PackFile { if t == backend.PackFile {
subdirs = true subdirs = true
} }

14
internal/backend/layout/layout_rest.go
View File

@ -1,6 +1,8 @@
package layout package layout
import "github.com/restic/restic/internal/restic" import (
"github.com/restic/restic/internal/backend"
)
// RESTLayout implements the default layout for the REST protocol. // RESTLayout implements the default layout for the REST protocol.
type RESTLayout struct { type RESTLayout struct {
@ -21,8 +23,8 @@ func (l *RESTLayout) Name() string {
} }
// Dirname returns the directory path for a given file type and name. // Dirname returns the directory path for a given file type and name.
func (l *RESTLayout) Dirname(h restic.Handle) string { func (l *RESTLayout) Dirname(h backend.Handle) string {
if h.Type == restic.ConfigFile { if h.Type == backend.ConfigFile {
return l.URL + l.Join(l.Path, "/") return l.URL + l.Join(l.Path, "/")
} }
@ -30,10 +32,10 @@ func (l *RESTLayout) Dirname(h restic.Handle) string {
} }
// Filename returns a path to a file, including its name. // Filename returns a path to a file, including its name.
func (l *RESTLayout) Filename(h restic.Handle) string { func (l *RESTLayout) Filename(h backend.Handle) string {
name := h.Name name := h.Name
if h.Type == restic.ConfigFile { if h.Type == backend.ConfigFile {
name = "config" name = "config"
} }
@ -49,6 +51,6 @@ func (l *RESTLayout) Paths() (dirs []string) {
} }
// Basedir returns the base dir name for files of type t. // Basedir returns the base dir name for files of type t.
func (l *RESTLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) { func (l *RESTLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) {
return l.URL + l.Join(l.Path, restLayoutPaths[t]), false return l.URL + l.Join(l.Path, restLayoutPaths[t]), false
} }

26
internal/backend/layout/layout_s3legacy.go
View File

@ -1,6 +1,8 @@
package layout package layout
import "github.com/restic/restic/internal/restic" import (
"github.com/restic/restic/internal/backend"
)
// S3LegacyLayout implements the old layout used for s3 cloud storage backends, as // S3LegacyLayout implements the old layout used for s3 cloud storage backends, as
// described in the Design document. // described in the Design document.
@ -10,12 +12,12 @@ type S3LegacyLayout struct {
Join func(...string) string Join func(...string) string
} }
var s3LayoutPaths = map[restic.FileType]string{ var s3LayoutPaths = map[backend.FileType]string{
restic.PackFile: "data", backend.PackFile: "data",
restic.SnapshotFile: "snapshot", backend.SnapshotFile: "snapshot",
restic.IndexFile: "index", backend.IndexFile: "index",
restic.LockFile: "lock", backend.LockFile: "lock",
restic.KeyFile: "key", backend.KeyFile: "key",
} }
func (l *S3LegacyLayout) String() string { func (l *S3LegacyLayout) String() string {
@ -44,8 +46,8 @@ func (l *S3LegacyLayout) join(url string, items ...string) string {
} }
// Dirname returns the directory path for a given file type and name. // Dirname returns the directory path for a given file type and name.
func (l *S3LegacyLayout) Dirname(h restic.Handle) string { func (l *S3LegacyLayout) Dirname(h backend.Handle) string {
if h.Type == restic.ConfigFile { if h.Type == backend.ConfigFile {
return l.URL + l.Join(l.Path, "/") return l.URL + l.Join(l.Path, "/")
} }
@ -53,10 +55,10 @@ func (l *S3LegacyLayout) Dirname(h restic.Handle) string {
} }
// Filename returns a path to a file, including its name. // Filename returns a path to a file, including its name.
func (l *S3LegacyLayout) Filename(h restic.Handle) string { func (l *S3LegacyLayout) Filename(h backend.Handle) string {
name := h.Name name := h.Name
if h.Type == restic.ConfigFile { if h.Type == backend.ConfigFile {
name = "config" name = "config"
} }
@ -72,6 +74,6 @@ func (l *S3LegacyLayout) Paths() (dirs []string) {
} }
// Basedir returns the base dir name for type t. // Basedir returns the base dir name for type t.
func (l *S3LegacyLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) { func (l *S3LegacyLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) {
return l.Join(l.Path, s3LayoutPaths[t]), false return l.Join(l.Path, s3LayoutPaths[t]), false
} }

80
internal/backend/layout/layout_test.go
View File

@ -9,7 +9,7 @@ import (
"sort" "sort"
"testing" "testing"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
@ -19,79 +19,79 @@ func TestDefaultLayout(t *testing.T) {
var tests = []struct { var tests = []struct {
path string path string
join func(...string) string join func(...string) string
restic.Handle backend.Handle
filename string filename string
}{ }{
{ {
tempdir, tempdir,
filepath.Join, filepath.Join,
restic.Handle{Type: restic.PackFile, Name: "0123456"}, backend.Handle{Type: backend.PackFile, Name: "0123456"},
filepath.Join(tempdir, "data", "01", "0123456"), filepath.Join(tempdir, "data", "01", "0123456"),
}, },
{ {
tempdir, tempdir,
filepath.Join, filepath.Join,
restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
filepath.Join(tempdir, "config"), filepath.Join(tempdir, "config"),
}, },
{ {
tempdir, tempdir,
filepath.Join, filepath.Join,
restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
filepath.Join(tempdir, "snapshots", "123456"), filepath.Join(tempdir, "snapshots", "123456"),
}, },
{ {
tempdir, tempdir,
filepath.Join, filepath.Join,
restic.Handle{Type: restic.IndexFile, Name: "123456"}, backend.Handle{Type: backend.IndexFile, Name: "123456"},
filepath.Join(tempdir, "index", "123456"), filepath.Join(tempdir, "index", "123456"),
}, },
{ {
tempdir, tempdir,
filepath.Join, filepath.Join,
restic.Handle{Type: restic.LockFile, Name: "123456"}, backend.Handle{Type: backend.LockFile, Name: "123456"},
filepath.Join(tempdir, "locks", "123456"), filepath.Join(tempdir, "locks", "123456"),
}, },
{ {
tempdir, tempdir,
filepath.Join, filepath.Join,
restic.Handle{Type: restic.KeyFile, Name: "123456"}, backend.Handle{Type: backend.KeyFile, Name: "123456"},
filepath.Join(tempdir, "keys", "123456"), filepath.Join(tempdir, "keys", "123456"),
}, },
{ {
"", "",
path.Join, path.Join,
restic.Handle{Type: restic.PackFile, Name: "0123456"}, backend.Handle{Type: backend.PackFile, Name: "0123456"},
"data/01/0123456", "data/01/0123456",
}, },
{ {
"", "",
path.Join, path.Join,
restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
"config", "config",
}, },
{ {
"", "",
path.Join, path.Join,
restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
"snapshots/123456", "snapshots/123456",
}, },
{ {
"", "",
path.Join, path.Join,
restic.Handle{Type: restic.IndexFile, Name: "123456"}, backend.Handle{Type: backend.IndexFile, Name: "123456"},
"index/123456", "index/123456",
}, },
{ {
"", "",
path.Join, path.Join,
restic.Handle{Type: restic.LockFile, Name: "123456"}, backend.Handle{Type: backend.LockFile, Name: "123456"},
"locks/123456", "locks/123456",
}, },
{ {
"", "",
path.Join, path.Join,
restic.Handle{Type: restic.KeyFile, Name: "123456"}, backend.Handle{Type: backend.KeyFile, Name: "123456"},
"keys/123456", "keys/123456",
}, },
} }
@ -143,31 +143,31 @@ func TestRESTLayout(t *testing.T) {
path := rtest.TempDir(t) path := rtest.TempDir(t)
var tests = []struct { var tests = []struct {
restic.Handle backend.Handle
filename string filename string
}{ }{
{ {
restic.Handle{Type: restic.PackFile, Name: "0123456"}, backend.Handle{Type: backend.PackFile, Name: "0123456"},
filepath.Join(path, "data", "0123456"), filepath.Join(path, "data", "0123456"),
}, },
{ {
restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
filepath.Join(path, "config"), filepath.Join(path, "config"),
}, },
{ {
restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
filepath.Join(path, "snapshots", "123456"), filepath.Join(path, "snapshots", "123456"),
}, },
{ {
restic.Handle{Type: restic.IndexFile, Name: "123456"}, backend.Handle{Type: backend.IndexFile, Name: "123456"},
filepath.Join(path, "index", "123456"), filepath.Join(path, "index", "123456"),
}, },
{ {
restic.Handle{Type: restic.LockFile, Name: "123456"}, backend.Handle{Type: backend.LockFile, Name: "123456"},
filepath.Join(path, "locks", "123456"), filepath.Join(path, "locks", "123456"),
}, },
{ {
restic.Handle{Type: restic.KeyFile, Name: "123456"}, backend.Handle{Type: backend.KeyFile, Name: "123456"},
filepath.Join(path, "keys", "123456"), filepath.Join(path, "keys", "123456"),
}, },
} }
@ -209,61 +209,61 @@ func TestRESTLayout(t *testing.T) {
func TestRESTLayoutURLs(t *testing.T) { func TestRESTLayoutURLs(t *testing.T) {
var tests = []struct { var tests = []struct {
l Layout l Layout
h restic.Handle h backend.Handle
fn string fn string
dir string dir string
}{ }{
{ {
&RESTLayout{URL: "https://hostname.foo", Path: "", Join: path.Join}, &RESTLayout{URL: "https://hostname.foo", Path: "", Join: path.Join},
restic.Handle{Type: restic.PackFile, Name: "foobar"}, backend.Handle{Type: backend.PackFile, Name: "foobar"},
"https://hostname.foo/data/foobar", "https://hostname.foo/data/foobar",
"https://hostname.foo/data/", "https://hostname.foo/data/",
}, },
{ {
&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, &RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"}, backend.Handle{Type: backend.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/locks/foobar", "https://hostname.foo:1234/prefix/repo/locks/foobar",
"https://hostname.foo:1234/prefix/repo/locks/", "https://hostname.foo:1234/prefix/repo/locks/",
}, },
{ {
&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, &RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config", "https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/", "https://hostname.foo:1234/prefix/repo/",
}, },
{ {
&S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join}, &S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.PackFile, Name: "foobar"}, backend.Handle{Type: backend.PackFile, Name: "foobar"},
"https://hostname.foo/data/foobar", "https://hostname.foo/data/foobar",
"https://hostname.foo/data/", "https://hostname.foo/data/",
}, },
{ {
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join}, &S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"}, backend.Handle{Type: backend.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/lock/foobar", "https://hostname.foo:1234/prefix/repo/lock/foobar",
"https://hostname.foo:1234/prefix/repo/lock/", "https://hostname.foo:1234/prefix/repo/lock/",
}, },
{ {
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, &S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config", "https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/", "https://hostname.foo:1234/prefix/repo/",
}, },
{ {
&S3LegacyLayout{URL: "", Path: "", Join: path.Join}, &S3LegacyLayout{URL: "", Path: "", Join: path.Join},
restic.Handle{Type: restic.PackFile, Name: "foobar"}, backend.Handle{Type: backend.PackFile, Name: "foobar"},
"data/foobar", "data/foobar",
"data/", "data/",
}, },
{ {
&S3LegacyLayout{URL: "", Path: "", Join: path.Join}, &S3LegacyLayout{URL: "", Path: "", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"}, backend.Handle{Type: backend.LockFile, Name: "foobar"},
"lock/foobar", "lock/foobar",
"lock/", "lock/",
}, },
{ {
&S3LegacyLayout{URL: "", Path: "/", Join: path.Join}, &S3LegacyLayout{URL: "", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"/config", "/config",
"/", "/",
}, },
@ -288,31 +288,31 @@ func TestS3LegacyLayout(t *testing.T) {
path := rtest.TempDir(t) path := rtest.TempDir(t)
var tests = []struct { var tests = []struct {
restic.Handle backend.Handle
filename string filename string
}{ }{
{ {
restic.Handle{Type: restic.PackFile, Name: "0123456"}, backend.Handle{Type: backend.PackFile, Name: "0123456"},
filepath.Join(path, "data", "0123456"), filepath.Join(path, "data", "0123456"),
}, },
{ {
restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
filepath.Join(path, "config"), filepath.Join(path, "config"),
}, },
{ {
restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
filepath.Join(path, "snapshot", "123456"), filepath.Join(path, "snapshot", "123456"),
}, },
{ {
restic.Handle{Type: restic.IndexFile, Name: "123456"}, backend.Handle{Type: backend.IndexFile, Name: "123456"},
filepath.Join(path, "index", "123456"), filepath.Join(path, "index", "123456"),
}, },
{ {
restic.Handle{Type: restic.LockFile, Name: "123456"}, backend.Handle{Type: backend.LockFile, Name: "123456"},
filepath.Join(path, "lock", "123456"), filepath.Join(path, "lock", "123456"),
}, },
{ {
restic.Handle{Type: restic.KeyFile, Name: "123456"}, backend.Handle{Type: backend.KeyFile, Name: "123456"},
filepath.Join(path, "key", "123456"), filepath.Join(path, "key", "123456"),
}, },
} }
@ -415,8 +415,8 @@ func TestParseLayout(t *testing.T) {
} }
// test that the functions work (and don't panic) // test that the functions work (and don't panic)
_ = layout.Dirname(restic.Handle{Type: restic.PackFile}) _ = layout.Dirname(backend.Handle{Type: backend.PackFile})
_ = layout.Filename(restic.Handle{Type: restic.PackFile, Name: "1234"}) _ = layout.Filename(backend.Handle{Type: backend.PackFile, Name: "1234"})
_ = layout.Paths() _ = layout.Paths()
layoutName := fmt.Sprintf("%T", layout) layoutName := fmt.Sprintf("%T", layout)

22
internal/backend/limiter/limiter_backend.go
View File

@ -4,12 +4,12 @@ import (
"context" "context"
"io" "io"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/backend"
) )
func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) { func WrapBackendConstructor[B backend.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (backend.Backend, error) {
return func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) { return func(ctx context.Context, cfg C, lim Limiter) (backend.Backend, error) {
var be restic.Backend var be backend.Backend
be, err := constructor(ctx, cfg) be, err := constructor(ctx, cfg)
if err != nil { if err != nil {
return nil, err return nil, err
@ -24,7 +24,7 @@ func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx contex
// LimitBackend wraps a Backend and applies rate limiting to Load() and Save() // LimitBackend wraps a Backend and applies rate limiting to Load() and Save()
// calls on the backend. // calls on the backend.
func LimitBackend(be restic.Backend, l Limiter) restic.Backend { func LimitBackend(be backend.Backend, l Limiter) backend.Backend {
return rateLimitedBackend{ return rateLimitedBackend{
Backend: be, Backend: be,
limiter: l, limiter: l,
@ -32,11 +32,11 @@ func LimitBackend(be restic.Backend, l Limiter) restic.Backend {
} }
type rateLimitedBackend struct { type rateLimitedBackend struct {
restic.Backend backend.Backend
limiter Limiter limiter Limiter
} }
func (r rateLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (r rateLimitedBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
limited := limitedRewindReader{ limited := limitedRewindReader{
RewindReader: rd, RewindReader: rd,
limited: r.limiter.Upstream(rd), limited: r.limiter.Upstream(rd),
@ -46,7 +46,7 @@ func (r rateLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic
} }
type limitedRewindReader struct { type limitedRewindReader struct {
restic.RewindReader backend.RewindReader
limited io.Reader limited io.Reader
} }
@ -55,13 +55,13 @@ func (l limitedRewindReader) Read(b []byte) (int, error) {
return l.limited.Read(b) return l.limited.Read(b)
} }
func (r rateLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { func (r rateLimitedBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
return r.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { return r.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
return consumer(newDownstreamLimitedReader(rd, r.limiter)) return consumer(newDownstreamLimitedReader(rd, r.limiter))
}) })
} }
func (r rateLimitedBackend) Unwrap() restic.Backend { return r.Backend } func (r rateLimitedBackend) Unwrap() backend.Backend { return r.Backend }
type limitedReader struct { type limitedReader struct {
io.Reader io.Reader
@ -85,4 +85,4 @@ func (l *limitedReader) WriteTo(w io.Writer) (int64, error) {
return l.writerTo.WriteTo(l.limiter.DownstreamWriter(w)) return l.writerTo.WriteTo(l.limiter.DownstreamWriter(w))
} }
var _ restic.Backend = (*rateLimitedBackend)(nil) var _ backend.Backend = (*rateLimitedBackend)(nil)

12
internal/backend/limiter/limiter_backend_test.go
View File

@ -8,8 +8,8 @@ import (
"io" "io"
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
@ -21,11 +21,11 @@ func randomBytes(t *testing.T, size int) []byte {
} }
func TestLimitBackendSave(t *testing.T) { func TestLimitBackendSave(t *testing.T) {
testHandle := restic.Handle{Type: restic.PackFile, Name: "test"} testHandle := backend.Handle{Type: backend.PackFile, Name: "test"}
data := randomBytes(t, 1234) data := randomBytes(t, 1234)
be := mock.NewBackend() be := mock.NewBackend()
be.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { be.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
_, err := io.Copy(buf, rd) _, err := io.Copy(buf, rd)
if err != nil { if err != nil {
@ -39,7 +39,7 @@ func TestLimitBackendSave(t *testing.T) {
limiter := NewStaticLimiter(Limits{42 * 1024, 42 * 1024}) limiter := NewStaticLimiter(Limits{42 * 1024, 42 * 1024})
limbe := LimitBackend(be, limiter) limbe := LimitBackend(be, limiter)
rd := restic.NewByteReader(data, nil) rd := backend.NewByteReader(data, nil)
err := limbe.Save(context.TODO(), testHandle, rd) err := limbe.Save(context.TODO(), testHandle, rd)
rtest.OK(t, err) rtest.OK(t, err)
} }
@ -64,7 +64,7 @@ func (r *tracedReadWriteToCloser) Close() error {
} }
func TestLimitBackendLoad(t *testing.T) { func TestLimitBackendLoad(t *testing.T) {
testHandle := restic.Handle{Type: restic.PackFile, Name: "test"} testHandle := backend.Handle{Type: backend.PackFile, Name: "test"}
data := randomBytes(t, 1234) data := randomBytes(t, 1234)
for _, test := range []struct { for _, test := range []struct {
@ -72,7 +72,7 @@ func TestLimitBackendLoad(t *testing.T) {
}{{false, false}, {false, true}, {true, false}, {true, true}} { }{{false, false}, {false, true}, {true, false}, {true, true}} {
be := mock.NewBackend() be := mock.NewBackend()
src := newTracedReadWriteToCloser(bytes.NewReader(data)) src := newTracedReadWriteToCloser(bytes.NewReader(data))
be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
if length != 0 || offset != 0 { if length != 0 || offset != 0 {
return nil, fmt.Errorf("Not supported") return nil, fmt.Errorf("Not supported")
} }

4
internal/backend/local/layout_test.go
View File

@ -5,7 +5,7 @@ import (
"path/filepath" "path/filepath"
"testing" "testing"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
@ -49,7 +49,7 @@ func TestLayout(t *testing.T) {
} }
packs := make(map[string]bool) packs := make(map[string]bool)
err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
packs[fi.Name] = false packs[fi.Name] = false
return nil return nil
}) })

31
internal/backend/local/local.go
View File

@ -16,7 +16,6 @@ import (
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
"github.com/cenkalti/backoff/v4" "github.com/cenkalti/backoff/v4"
) )
@ -28,8 +27,8 @@ type Local struct {
util.Modes util.Modes
} }
// ensure statically that *Local implements restic.Backend. // ensure statically that *Local implements backend.Backend.
var _ restic.Backend = &Local{} var _ backend.Backend = &Local{}
func NewFactory() location.Factory { func NewFactory() location.Factory {
return location.NewLimitedBackendFactory("local", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) return location.NewLimitedBackendFactory("local", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open))
@ -43,7 +42,7 @@ func open(ctx context.Context, cfg Config) (*Local, error) {
return nil, err return nil, err
} }
fi, err := fs.Stat(l.Filename(restic.Handle{Type: restic.ConfigFile})) fi, err := fs.Stat(l.Filename(backend.Handle{Type: backend.ConfigFile}))
m := util.DeriveModesFromFileInfo(fi, err) m := util.DeriveModesFromFileInfo(fi, err)
debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir)
@ -71,7 +70,7 @@ func Create(ctx context.Context, cfg Config) (*Local, error) {
} }
// test if config file already exists // test if config file already exists
_, err = fs.Lstat(be.Filename(restic.Handle{Type: restic.ConfigFile})) _, err = fs.Lstat(be.Filename(backend.Handle{Type: backend.ConfigFile}))
if err == nil { if err == nil {
return nil, errors.New("config file already exists") return nil, errors.New("config file already exists")
} }
@ -112,7 +111,7 @@ func (b *Local) IsNotExist(err error) bool {
} }
// Save stores data in the backend at the handle. // Save stores data in the backend at the handle.
func (b *Local) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) (err error) { func (b *Local) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) (err error) {
finalname := b.Filename(h) finalname := b.Filename(h)
dir := filepath.Dir(finalname) dir := filepath.Dir(finalname)
@ -210,11 +209,11 @@ var tempFile = os.CreateTemp // Overridden by test.
// Load runs fn with a reader that yields the contents of the file at h at the // Load runs fn with a reader that yields the contents of the file at h at the
// given offset. // given offset.
func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { func (b *Local) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, b.openReader, fn) return util.DefaultLoad(ctx, h, length, offset, b.openReader, fn)
} }
func (b *Local) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { func (b *Local) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
f, err := fs.Open(b.Filename(h)) f, err := fs.Open(b.Filename(h))
if err != nil { if err != nil {
return nil, err return nil, err
@ -236,17 +235,17 @@ func (b *Local) openReader(_ context.Context, h restic.Handle, length int, offse
} }
// Stat returns information about a blob. // Stat returns information about a blob.
func (b *Local) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) { func (b *Local) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) {
fi, err := fs.Stat(b.Filename(h)) fi, err := fs.Stat(b.Filename(h))
if err != nil { if err != nil {
return restic.FileInfo{}, errors.WithStack(err) return backend.FileInfo{}, errors.WithStack(err)
} }
return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil
} }
// Remove removes the blob with the given name and type. // Remove removes the blob with the given name and type.
func (b *Local) Remove(_ context.Context, h restic.Handle) error { func (b *Local) Remove(_ context.Context, h backend.Handle) error {
fn := b.Filename(h) fn := b.Filename(h)
// reset read-only flag // reset read-only flag
@ -260,7 +259,7 @@ func (b *Local) Remove(_ context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an // List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it. // error occurs (or fn returns an error), List stops and returns it.
func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) (err error) { func (b *Local) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) (err error) {
basedir, subdirs := b.Basedir(t) basedir, subdirs := b.Basedir(t)
if subdirs { if subdirs {
err = visitDirs(ctx, basedir, fn) err = visitDirs(ctx, basedir, fn)
@ -280,7 +279,7 @@ func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.File
// two levels of directory structure (including dir itself as the first level). // two levels of directory structure (including dir itself as the first level).
// Also, visitDirs assumes it sees a directory full of directories, while // Also, visitDirs assumes it sees a directory full of directories, while
// visitFiles wants a directory full or regular files. // visitFiles wants a directory full or regular files.
func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error) error { func visitDirs(ctx context.Context, dir string, fn func(backend.FileInfo) error) error {
d, err := fs.Open(dir) d, err := fs.Open(dir)
if err != nil { if err != nil {
return err return err
@ -307,7 +306,7 @@ func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error)
return ctx.Err() return ctx.Err()
} }
func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error, ignoreNotADirectory bool) error { func visitFiles(ctx context.Context, dir string, fn func(backend.FileInfo) error, ignoreNotADirectory bool) error {
d, err := fs.Open(dir) d, err := fs.Open(dir)
if err != nil { if err != nil {
return err return err
@ -341,7 +340,7 @@ func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error,
default: default:
} }
err := fn(restic.FileInfo{ err := fn(backend.FileInfo{
Name: fi.Name(), Name: fi.Name(),
Size: fi.Size(), Size: fi.Size(),
}) })

4
internal/backend/local/local_internal_test.go
View File

@ -8,7 +8,7 @@ import (
"syscall" "syscall"
"testing" "testing"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
"github.com/cenkalti/backoff/v4" "github.com/cenkalti/backoff/v4"
@ -32,7 +32,7 @@ func TestNoSpacePermanent(t *testing.T) {
rtest.OK(t, be.Close()) rtest.OK(t, be.Close())
}() }()
h := restic.Handle{Type: restic.ConfigFile} h := backend.Handle{Type: backend.ConfigFile}
err = be.Save(context.Background(), h, nil) err = be.Save(context.Background(), h, nil)
_, ok := err.(*backoff.PermanentError) _, ok := err.(*backoff.PermanentError)
rtest.Assert(t, ok, rtest.Assert(t, ok,

4
internal/backend/location/display_location_test.go
View File

@ -3,15 +3,15 @@ package location_test
import ( import (
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test" "github.com/restic/restic/internal/test"
) )
func TestStripPassword(t *testing.T) { func TestStripPassword(t *testing.T) {
registry := location.NewRegistry() registry := location.NewRegistry()
registry.Register( registry.Register(
location.NewHTTPBackendFactory[any, restic.Backend]("test", nil, location.NewHTTPBackendFactory[any, backend.Backend]("test", nil,
func(s string) string { func(s string) string {
return "cleaned" return "cleaned"
}, nil, nil, }, nil, nil,

4
internal/backend/location/location_test.go
View File

@ -3,8 +3,8 @@ package location_test
import ( import (
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test" "github.com/restic/restic/internal/test"
) )
@ -13,7 +13,7 @@ type testConfig struct {
} }
func testFactory() location.Factory { func testFactory() location.Factory {
return location.NewHTTPBackendFactory[testConfig, restic.Backend]( return location.NewHTTPBackendFactory[testConfig, backend.Backend](
"local", "local",
func(s string) (*testConfig, error) { func(s string) (*testConfig, error) {
return &testConfig{loc: s}, nil return &testConfig{loc: s}, nil

16
internal/backend/location/registry.go
View File

@ -4,8 +4,8 @@ import (
"context" "context"
"net/http" "net/http"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/restic"
) )
type Registry struct { type Registry struct {
@ -33,11 +33,11 @@ type Factory interface {
Scheme() string Scheme() string
ParseConfig(s string) (interface{}, error) ParseConfig(s string) (interface{}, error)
StripPassword(s string) string StripPassword(s string) string
Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error)
Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error)
} }
type genericBackendFactory[C any, T restic.Backend] struct { type genericBackendFactory[C any, T backend.Backend] struct {
scheme string scheme string
parseConfigFn func(s string) (*C, error) parseConfigFn func(s string) (*C, error)
stripPasswordFn func(s string) string stripPasswordFn func(s string) string
@ -58,14 +58,14 @@ func (f *genericBackendFactory[C, T]) StripPassword(s string) string {
} }
return s return s
} }
func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) {
return f.createFn(ctx, *cfg.(*C), rt, lim) return f.createFn(ctx, *cfg.(*C), rt, lim)
} }
func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) {
return f.openFn(ctx, *cfg.(*C), rt, lim) return f.openFn(ctx, *cfg.(*C), rt, lim)
} }
func NewHTTPBackendFactory[C any, T restic.Backend]( func NewHTTPBackendFactory[C any, T backend.Backend](
scheme string, scheme string,
parseConfigFn func(s string) (*C, error), parseConfigFn func(s string) (*C, error),
stripPasswordFn func(s string) string, stripPasswordFn func(s string) string,
@ -85,7 +85,7 @@ func NewHTTPBackendFactory[C any, T restic.Backend](
} }
} }
func NewLimitedBackendFactory[C any, T restic.Backend]( func NewLimitedBackendFactory[C any, T backend.Backend](
scheme string, scheme string,
parseConfigFn func(s string) (*C, error), parseConfigFn func(s string) (*C, error),
stripPasswordFn func(s string) string, stripPasswordFn func(s string) string,

22
internal/backend/logger/log.go
View File

@ -4,18 +4,18 @@ import (
"context" "context"
"io" "io"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
) )
type Backend struct { type Backend struct {
restic.Backend backend.Backend
} }
// statically ensure that Backend implements restic.Backend. // statically ensure that Backend implements backend.Backend.
var _ restic.Backend = &Backend{} var _ backend.Backend = &Backend{}
func New(be restic.Backend) *Backend { func New(be backend.Backend) *Backend {
return &Backend{Backend: be} return &Backend{Backend: be}
} }
@ -26,7 +26,7 @@ func (be *Backend) IsNotExist(err error) bool {
} }
// Save adds new Data to the backend. // Save adds new Data to the backend.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
debug.Log("Save(%v, %v)", h, rd.Length()) debug.Log("Save(%v, %v)", h, rd.Length())
err := be.Backend.Save(ctx, h, rd) err := be.Backend.Save(ctx, h, rd)
debug.Log(" save err %v", err) debug.Log(" save err %v", err)
@ -34,28 +34,28 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
} }
// Remove deletes a file from the backend. // Remove deletes a file from the backend.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
debug.Log("Remove(%v)", h) debug.Log("Remove(%v)", h)
err := be.Backend.Remove(ctx, h) err := be.Backend.Remove(ctx, h)
debug.Log(" remove err %v", err) debug.Log(" remove err %v", err)
return err return err
} }
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error { func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(io.Reader) error) error {
debug.Log("Load(%v, length %v, offset %v)", h, length, offset) debug.Log("Load(%v, length %v, offset %v)", h, length, offset)
err := be.Backend.Load(ctx, h, length, offset, fn) err := be.Backend.Load(ctx, h, length, offset, fn)
debug.Log(" load err %v", err) debug.Log(" load err %v", err)
return err return err
} }
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
debug.Log("Stat(%v)", h) debug.Log("Stat(%v)", h)
fi, err := be.Backend.Stat(ctx, h) fi, err := be.Backend.Stat(ctx, h)
debug.Log(" stat err %v", err) debug.Log(" stat err %v", err)
return fi, err return fi, err
} }
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
debug.Log("List(%v)", t) debug.Log("List(%v)", t)
err := be.Backend.List(ctx, t, fn) err := be.Backend.List(ctx, t, fn)
debug.Log(" list err %v", err) debug.Log(" list err %v", err)
@ -76,4 +76,4 @@ func (be *Backend) Close() error {
return err return err
} }
func (be *Backend) Unwrap() restic.Backend { return be.Backend } func (be *Backend) Unwrap() backend.Backend { return be.Backend }

30
internal/backend/mem/mem_backend.go
View File

@ -10,17 +10,17 @@ import (
"sync" "sync"
"github.com/cespare/xxhash/v2" "github.com/cespare/xxhash/v2"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
) )
type memMap map[restic.Handle][]byte type memMap map[backend.Handle][]byte
// make sure that MemoryBackend implements backend.Backend // make sure that MemoryBackend implements backend.Backend
var _ restic.Backend = &MemoryBackend{} var _ backend.Backend = &MemoryBackend{}
// NewFactory creates a persistent mem backend // NewFactory creates a persistent mem backend
func NewFactory() location.Factory { func NewFactory() location.Factory {
@ -69,12 +69,12 @@ func (be *MemoryBackend) IsNotExist(err error) bool {
} }
// Save adds new Data to the backend. // Save adds new Data to the backend.
func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (be *MemoryBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
be.m.Lock() be.m.Lock()
defer be.m.Unlock() defer be.m.Unlock()
h.IsMetadata = false h.IsMetadata = false
if h.Type == restic.ConfigFile { if h.Type == backend.ConfigFile {
h.Name = "" h.Name = ""
} }
@ -112,16 +112,16 @@ func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re
// Load runs fn with a reader that yields the contents of the file at h at the // Load runs fn with a reader that yields the contents of the file at h at the
// given offset. // given offset.
func (be *MemoryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { func (be *MemoryBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
} }
func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { func (be *MemoryBackend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
be.m.Lock() be.m.Lock()
defer be.m.Unlock() defer be.m.Unlock()
h.IsMetadata = false h.IsMetadata = false
if h.Type == restic.ConfigFile { if h.Type == backend.ConfigFile {
h.Name = "" h.Name = ""
} }
@ -143,25 +143,25 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length
} }
// Stat returns information about a file in the backend. // Stat returns information about a file in the backend.
func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { func (be *MemoryBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
be.m.Lock() be.m.Lock()
defer be.m.Unlock() defer be.m.Unlock()
h.IsMetadata = false h.IsMetadata = false
if h.Type == restic.ConfigFile { if h.Type == backend.ConfigFile {
h.Name = "" h.Name = ""
} }
e, ok := be.data[h] e, ok := be.data[h]
if !ok { if !ok {
return restic.FileInfo{}, errNotFound return backend.FileInfo{}, errNotFound
} }
return restic.FileInfo{Size: int64(len(e)), Name: h.Name}, ctx.Err() return backend.FileInfo{Size: int64(len(e)), Name: h.Name}, ctx.Err()
} }
// Remove deletes a file from the backend. // Remove deletes a file from the backend.
func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error { func (be *MemoryBackend) Remove(ctx context.Context, h backend.Handle) error {
be.m.Lock() be.m.Lock()
defer be.m.Unlock() defer be.m.Unlock()
@ -176,7 +176,7 @@ func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error {
} }
// List returns a channel which yields entries from the backend. // List returns a channel which yields entries from the backend.
func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { func (be *MemoryBackend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
entries := make(map[string]int64) entries := make(map[string]int64)
be.m.Lock() be.m.Lock()
@ -190,7 +190,7 @@ func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(re
be.m.Unlock() be.m.Unlock()
for name, size := range entries { for name, size := range entries {
fi := restic.FileInfo{ fi := backend.FileInfo{
Name: name, Name: name,
Size: size, Size: size,
} }

28
internal/backend/mock/backend.go
View File

@ -5,19 +5,19 @@ import (
"hash" "hash"
"io" "io"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
) )
// Backend implements a mock backend. // Backend implements a mock backend.
type Backend struct { type Backend struct {
CloseFn func() error CloseFn func() error
IsNotExistFn func(err error) bool IsNotExistFn func(err error) bool
SaveFn func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error SaveFn func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error
OpenReaderFn func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) OpenReaderFn func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error)
StatFn func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) StatFn func(ctx context.Context, h backend.Handle) (backend.FileInfo, error)
ListFn func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error ListFn func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error
RemoveFn func(ctx context.Context, h restic.Handle) error RemoveFn func(ctx context.Context, h backend.Handle) error
DeleteFn func(ctx context.Context) error DeleteFn func(ctx context.Context) error
ConnectionsFn func() uint ConnectionsFn func() uint
LocationFn func() string LocationFn func() string
@ -84,7 +84,7 @@ func (m *Backend) IsNotExist(err error) bool {
} }
// Save data in the backend. // Save data in the backend.
func (m *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (m *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if m.SaveFn == nil { if m.SaveFn == nil {
return errors.New("not implemented") return errors.New("not implemented")
} }
@ -94,7 +94,7 @@ func (m *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
// Load runs fn with a reader that yields the contents of the file at h at the // Load runs fn with a reader that yields the contents of the file at h at the
// given offset. // given offset.
func (m *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { func (m *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
rd, err := m.openReader(ctx, h, length, offset) rd, err := m.openReader(ctx, h, length, offset)
if err != nil { if err != nil {
return err return err
@ -107,7 +107,7 @@ func (m *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
return rd.Close() return rd.Close()
} }
func (m *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { func (m *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
if m.OpenReaderFn == nil { if m.OpenReaderFn == nil {
return nil, errors.New("not implemented") return nil, errors.New("not implemented")
} }
@ -116,16 +116,16 @@ func (m *Backend) openReader(ctx context.Context, h restic.Handle, length int, o
} }
// Stat an object in the backend. // Stat an object in the backend.
func (m *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { func (m *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
if m.StatFn == nil { if m.StatFn == nil {
return restic.FileInfo{}, errors.New("not implemented") return backend.FileInfo{}, errors.New("not implemented")
} }
return m.StatFn(ctx, h) return m.StatFn(ctx, h)
} }
// List items of type t. // List items of type t.
func (m *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { func (m *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
if m.ListFn == nil { if m.ListFn == nil {
return nil return nil
} }
@ -134,7 +134,7 @@ func (m *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi
} }
// Remove data from the backend. // Remove data from the backend.
func (m *Backend) Remove(ctx context.Context, h restic.Handle) error { func (m *Backend) Remove(ctx context.Context, h backend.Handle) error {
if m.RemoveFn == nil { if m.RemoveFn == nil {
return errors.New("not implemented") return errors.New("not implemented")
} }
@ -152,4 +152,4 @@ func (m *Backend) Delete(ctx context.Context) error {
} }
// Make sure that Backend implements the backend interface. // Make sure that Backend implements the backend interface.
var _ restic.Backend = &Backend{} var _ backend.Backend = &Backend{}

6
internal/backend/rclone/internal_test.go
View File

@ -5,8 +5,8 @@ import (
"os/exec" "os/exec"
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
@ -32,9 +32,9 @@ func TestRcloneExit(t *testing.T) {
t.Log("killed rclone") t.Log("killed rclone")
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
_, err = be.Stat(context.TODO(), restic.Handle{ _, err = be.Stat(context.TODO(), backend.Handle{
Name: "foo", Name: "foo",
Type: restic.PackFile, Type: backend.PackFile,
}) })
rtest.Assert(t, err != nil, "expected an error") rtest.Assert(t, err != nil, "expected an error")
} }

9
internal/backend/readerat.go
View File

@ -6,13 +6,12 @@ import (
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
) )
type backendReaderAt struct { type backendReaderAt struct {
ctx context.Context ctx context.Context
be restic.Backend be Backend
h restic.Handle h Handle
} }
func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) { func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
@ -22,12 +21,12 @@ func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
// ReaderAt returns an io.ReaderAt for a file in the backend. The returned reader // ReaderAt returns an io.ReaderAt for a file in the backend. The returned reader
// should not escape the caller function to avoid unexpected interactions with the // should not escape the caller function to avoid unexpected interactions with the
// embedded context // embedded context
func ReaderAt(ctx context.Context, be restic.Backend, h restic.Handle) io.ReaderAt { func ReaderAt(ctx context.Context, be Backend, h Handle) io.ReaderAt {
return backendReaderAt{ctx: ctx, be: be, h: h} return backendReaderAt{ctx: ctx, be: be, h: h}
} }
// ReadAt reads from the backend handle h at the given position. // ReadAt reads from the backend handle h at the given position.
func ReadAt(ctx context.Context, be restic.Backend, h restic.Handle, offset int64, p []byte) (n int, err error) { func ReadAt(ctx context.Context, be Backend, h Handle, offset int64, p []byte) (n int, err error) {
debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p)) debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p))
err = be.Load(ctx, h, len(p), offset, func(rd io.Reader) (ierr error) { err = be.Load(ctx, h, len(p), offset, func(rd io.Reader) (ierr error) {

4
internal/backend/rest/config.go
View File

@ -5,9 +5,9 @@ import (
"os" "os"
"strings" "strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options" "github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
) )
// Config contains all configuration necessary to connect to a REST server. // Config contains all configuration necessary to connect to a REST server.
@ -73,7 +73,7 @@ func prepareURL(s string) string {
return s return s
} }
var _ restic.ApplyEnvironmenter = &Config{} var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config. // ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) { func (cfg *Config) ApplyEnvironment(prefix string) {

46
internal/backend/rest/rest.go
View File

@ -11,16 +11,16 @@ import (
"path" "path"
"strings" "strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
) )
// make sure the rest backend implements restic.Backend // make sure the rest backend implements backend.Backend
var _ restic.Backend = &Backend{} var _ backend.Backend = &Backend{}
// Backend uses the REST protocol to access data stored on a server. // Backend uses the REST protocol to access data stored on a server.
type Backend struct { type Backend struct {
@ -65,7 +65,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, er
return nil, err return nil, err
} }
_, err = be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) _, err = be.Stat(ctx, backend.Handle{Type: backend.ConfigFile})
if err == nil { if err == nil {
return nil, errors.New("config file already exists") return nil, errors.New("config file already exists")
} }
@ -118,7 +118,7 @@ func (b *Backend) HasAtomicReplace() bool {
} }
// Save stores data in the backend at the handle. // Save stores data in the backend at the handle.
func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
@ -157,7 +157,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
// notExistError is returned whenever the requested file does not exist on the // notExistError is returned whenever the requested file does not exist on the
// server. // server.
type notExistError struct { type notExistError struct {
restic.Handle backend.Handle
} }
func (e *notExistError) Error() string { func (e *notExistError) Error() string {
@ -172,7 +172,7 @@ func (b *Backend) IsNotExist(err error) bool {
// Load runs fn with a reader that yields the contents of the file at h at the // Load runs fn with a reader that yields the contents of the file at h at the
// given offset. // given offset.
func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
r, err := b.openReader(ctx, h, length, offset) r, err := b.openReader(ctx, h, length, offset)
if err != nil { if err != nil {
return err return err
@ -201,7 +201,7 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
return err return err
} }
func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
req, err := http.NewRequestWithContext(ctx, "GET", b.Filename(h), nil) req, err := http.NewRequestWithContext(ctx, "GET", b.Filename(h), nil)
if err != nil { if err != nil {
return nil, errors.WithStack(err) return nil, errors.WithStack(err)
@ -238,37 +238,37 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o
} }
// Stat returns information about a blob. // Stat returns information about a blob.
func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodHead, b.Filename(h), nil) req, err := http.NewRequestWithContext(ctx, http.MethodHead, b.Filename(h), nil)
if err != nil { if err != nil {
return restic.FileInfo{}, errors.WithStack(err) return backend.FileInfo{}, errors.WithStack(err)
} }
req.Header.Set("Accept", ContentTypeV2) req.Header.Set("Accept", ContentTypeV2)
resp, err := b.client.Do(req) resp, err := b.client.Do(req)
if err != nil { if err != nil {
return restic.FileInfo{}, errors.WithStack(err) return backend.FileInfo{}, errors.WithStack(err)
} }
_, _ = io.Copy(io.Discard, resp.Body) _, _ = io.Copy(io.Discard, resp.Body)
if err = resp.Body.Close(); err != nil { if err = resp.Body.Close(); err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Close") return backend.FileInfo{}, errors.Wrap(err, "Close")
} }
if resp.StatusCode == http.StatusNotFound { if resp.StatusCode == http.StatusNotFound {
_ = resp.Body.Close() _ = resp.Body.Close()
return restic.FileInfo{}, &notExistError{h} return backend.FileInfo{}, &notExistError{h}
} }
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
return restic.FileInfo{}, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status) return backend.FileInfo{}, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status)
} }
if resp.ContentLength < 0 { if resp.ContentLength < 0 {
return restic.FileInfo{}, errors.New("negative content length") return backend.FileInfo{}, errors.New("negative content length")
} }
bi := restic.FileInfo{ bi := backend.FileInfo{
Size: resp.ContentLength, Size: resp.ContentLength,
Name: h.Name, Name: h.Name,
} }
@ -277,7 +277,7 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e
} }
// Remove removes the blob with the given name and type. // Remove removes the blob with the given name and type.
func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { func (b *Backend) Remove(ctx context.Context, h backend.Handle) error {
req, err := http.NewRequestWithContext(ctx, "DELETE", b.Filename(h), nil) req, err := http.NewRequestWithContext(ctx, "DELETE", b.Filename(h), nil)
if err != nil { if err != nil {
return errors.WithStack(err) return errors.WithStack(err)
@ -309,8 +309,8 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an // List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it. // error occurs (or fn returns an error), List stops and returns it.
func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
url := b.Dirname(restic.Handle{Type: t}) url := b.Dirname(backend.Handle{Type: t})
if !strings.HasSuffix(url, "/") { if !strings.HasSuffix(url, "/") {
url += "/" url += "/"
} }
@ -346,7 +346,7 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi
// listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET // listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET
// /data/`) only returns the names of the files, so we need to issue an HTTP // /data/`) only returns the names of the files, so we need to issue an HTTP
// HEAD request for each file. // HEAD request for each file.
func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error { func (b *Backend) listv1(ctx context.Context, t backend.FileType, resp *http.Response, fn func(backend.FileInfo) error) error {
debug.Log("parsing API v1 response") debug.Log("parsing API v1 response")
dec := json.NewDecoder(resp.Body) dec := json.NewDecoder(resp.Body)
var list []string var list []string
@ -355,7 +355,7 @@ func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Resp
} }
for _, m := range list { for _, m := range list {
fi, err := b.Stat(ctx, restic.Handle{Name: m, Type: t}) fi, err := b.Stat(ctx, backend.Handle{Name: m, Type: t})
if err != nil { if err != nil {
return err return err
} }
@ -380,7 +380,7 @@ func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Resp
// listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET // listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET
// /data/`) returns the names and sizes of all files. // /data/`) returns the names and sizes of all files.
func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(restic.FileInfo) error) error { func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(backend.FileInfo) error) error {
debug.Log("parsing API v2 response") debug.Log("parsing API v2 response")
dec := json.NewDecoder(resp.Body) dec := json.NewDecoder(resp.Body)
@ -397,7 +397,7 @@ func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(resti
return ctx.Err() return ctx.Err()
} }
fi := restic.FileInfo{ fi := backend.FileInfo{
Name: item.Name, Name: item.Name,
Size: item.Size, Size: item.Size,
} }

14
internal/backend/rest/rest_int_test.go
View File

@ -10,8 +10,8 @@ import (
"strconv" "strconv"
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/rest" "github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/restic"
) )
func TestListAPI(t *testing.T) { func TestListAPI(t *testing.T) {
@ -22,7 +22,7 @@ func TestListAPI(t *testing.T) {
Data string // response data Data string // response data
Requests int Requests int
Result []restic.FileInfo Result []backend.FileInfo
}{ }{
{ {
Name: "content-type-unknown", Name: "content-type-unknown",
@ -32,7 +32,7 @@ func TestListAPI(t *testing.T) {
"3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352",
"8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b" "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b"
]`, ]`,
Result: []restic.FileInfo{ Result: []backend.FileInfo{
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386}, {Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386},
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214}, {Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214},
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393}, {Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393},
@ -47,7 +47,7 @@ func TestListAPI(t *testing.T) {
"3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352",
"8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b" "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b"
]`, ]`,
Result: []restic.FileInfo{ Result: []backend.FileInfo{
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386}, {Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386},
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214}, {Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214},
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393}, {Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393},
@ -62,7 +62,7 @@ func TestListAPI(t *testing.T) {
{"name": "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "size": 1002}, {"name": "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "size": 1002},
{"name": "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", "size": 1003} {"name": "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", "size": 1003}
]`, ]`,
Result: []restic.FileInfo{ Result: []backend.FileInfo{
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 1001}, {Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 1001},
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 1002}, {Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 1002},
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 1003}, {Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 1003},
@ -122,8 +122,8 @@ func TestListAPI(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
var list []restic.FileInfo var list []backend.FileInfo
err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
list = append(list, fi) list = append(list, fi)
return nil return nil
}) })

24
internal/backend/retry/backend_retry.go
View File

@ -7,27 +7,27 @@ import (
"time" "time"
"github.com/cenkalti/backoff/v4" "github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
) )
// Backend retries operations on the backend in case of an error with a // Backend retries operations on the backend in case of an error with a
// backoff. // backoff.
type Backend struct { type Backend struct {
restic.Backend backend.Backend
MaxTries int MaxTries int
Report func(string, error, time.Duration) Report func(string, error, time.Duration)
Success func(string, int) Success func(string, int)
} }
// statically ensure that RetryBackend implements restic.Backend. // statically ensure that RetryBackend implements backend.Backend.
var _ restic.Backend = &Backend{} var _ backend.Backend = &Backend{}
// New wraps be with a backend that retries operations after a // New wraps be with a backend that retries operations after a
// backoff. report is called with a description and the error, if one occurred. // backoff. report is called with a description and the error, if one occurred.
// success is called with the number of retries before a successful operation // success is called with the number of retries before a successful operation
// (it is not called if it succeeded on the first try) // (it is not called if it succeeded on the first try)
func New(be restic.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend { func New(be backend.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend {
return &Backend{ return &Backend{
Backend: be, Backend: be,
MaxTries: maxTries, MaxTries: maxTries,
@ -92,7 +92,7 @@ func (be *Backend) retry(ctx context.Context, msg string, f func() error) error
} }
// Save stores the data in the backend under the given handle. // Save stores the data in the backend under the given handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
return be.retry(ctx, fmt.Sprintf("Save(%v)", h), func() error { return be.retry(ctx, fmt.Sprintf("Save(%v)", h), func() error {
err := rd.Rewind() err := rd.Rewind()
if err != nil { if err != nil {
@ -125,7 +125,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
// given offset. If length is larger than zero, only a portion of the file // given offset. If length is larger than zero, only a portion of the file
// is returned. rd must be closed after use. If an error is returned, the // is returned. rd must be closed after use. If an error is returned, the
// ReadCloser must be nil. // ReadCloser must be nil.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) { func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) {
return be.retry(ctx, fmt.Sprintf("Load(%v, %v, %v)", h, length, offset), return be.retry(ctx, fmt.Sprintf("Load(%v, %v, %v)", h, length, offset),
func() error { func() error {
err := be.Backend.Load(ctx, h, length, offset, consumer) err := be.Backend.Load(ctx, h, length, offset, consumer)
@ -137,7 +137,7 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
} }
// Stat returns information about the File identified by h. // Stat returns information about the File identified by h.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) { func (be *Backend) Stat(ctx context.Context, h backend.Handle) (fi backend.FileInfo, err error) {
err = be.retry(ctx, fmt.Sprintf("Stat(%v)", h), err = be.retry(ctx, fmt.Sprintf("Stat(%v)", h),
func() error { func() error {
var innerError error var innerError error
@ -153,7 +153,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInf
} }
// Remove removes a File with type t and name. // Remove removes a File with type t and name.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) { func (be *Backend) Remove(ctx context.Context, h backend.Handle) (err error) {
return be.retry(ctx, fmt.Sprintf("Remove(%v)", h), func() error { return be.retry(ctx, fmt.Sprintf("Remove(%v)", h), func() error {
return be.Backend.Remove(ctx, h) return be.Backend.Remove(ctx, h)
}) })
@ -163,7 +163,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) {
// error is returned by the underlying backend, the request is retried. When fn // error is returned by the underlying backend, the request is retried. When fn
// returns an error, the operation is aborted and the error is returned to the // returns an error, the operation is aborted and the error is returned to the
// caller. // caller.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
// create a new context that we can cancel when fn returns an error, so // create a new context that we can cancel when fn returns an error, so
// that listing is aborted // that listing is aborted
listCtx, cancel := context.WithCancel(ctx) listCtx, cancel := context.WithCancel(ctx)
@ -173,7 +173,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
var innerErr error // remember when fn returned an error, so we can return that to the caller var innerErr error // remember when fn returned an error, so we can return that to the caller
err := be.retry(listCtx, fmt.Sprintf("List(%v)", t), func() error { err := be.retry(listCtx, fmt.Sprintf("List(%v)", t), func() error {
return be.Backend.List(ctx, t, func(fi restic.FileInfo) error { return be.Backend.List(ctx, t, func(fi backend.FileInfo) error {
if _, ok := listed[fi.Name]; ok { if _, ok := listed[fi.Name]; ok {
return nil return nil
} }
@ -196,6 +196,6 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
return err return err
} }
func (be *Backend) Unwrap() restic.Backend { func (be *Backend) Unwrap() backend.Backend {
return be.Backend return be.Backend
} }

57
internal/backend/retry/backend_retry_test.go
View File

@ -8,6 +8,7 @@ import (
"time" "time"
"github.com/cenkalti/backoff/v4" "github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -18,7 +19,7 @@ func TestBackendSaveRetry(t *testing.T) {
buf := bytes.NewBuffer(nil) buf := bytes.NewBuffer(nil)
errcount := 0 errcount := 0
be := &mock.Backend{ be := &mock.Backend{
SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if errcount == 0 { if errcount == 0 {
errcount++ errcount++
_, err := io.CopyN(io.Discard, rd, 120) _, err := io.CopyN(io.Discard, rd, 120)
@ -38,7 +39,7 @@ func TestBackendSaveRetry(t *testing.T) {
retryBackend := New(be, 10, nil, nil) retryBackend := New(be, 10, nil, nil)
data := test.Random(23, 5*1024*1024+11241) data := test.Random(23, 5*1024*1024+11241)
err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher())) err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher()))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -56,14 +57,14 @@ func TestBackendSaveRetryAtomic(t *testing.T) {
errcount := 0 errcount := 0
calledRemove := false calledRemove := false
be := &mock.Backend{ be := &mock.Backend{
SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if errcount == 0 { if errcount == 0 {
errcount++ errcount++
return errors.New("injected error") return errors.New("injected error")
} }
return nil return nil
}, },
RemoveFn: func(ctx context.Context, h restic.Handle) error { RemoveFn: func(ctx context.Context, h backend.Handle) error {
calledRemove = true calledRemove = true
return nil return nil
}, },
@ -74,7 +75,7 @@ func TestBackendSaveRetryAtomic(t *testing.T) {
retryBackend := New(be, 10, nil, nil) retryBackend := New(be, 10, nil, nil)
data := test.Random(23, 5*1024*1024+11241) data := test.Random(23, 5*1024*1024+11241)
err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher())) err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher()))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -91,15 +92,15 @@ func TestBackendListRetry(t *testing.T) {
retry := 0 retry := 0
be := &mock.Backend{ be := &mock.Backend{
ListFn: func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { ListFn: func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
// fail during first retry, succeed during second // fail during first retry, succeed during second
retry++ retry++
if retry == 1 { if retry == 1 {
_ = fn(restic.FileInfo{Name: ID1}) _ = fn(backend.FileInfo{Name: ID1})
return errors.New("test list error") return errors.New("test list error")
} }
_ = fn(restic.FileInfo{Name: ID1}) _ = fn(backend.FileInfo{Name: ID1})
_ = fn(restic.FileInfo{Name: ID2}) _ = fn(backend.FileInfo{Name: ID2})
return nil return nil
}, },
} }
@ -108,7 +109,7 @@ func TestBackendListRetry(t *testing.T) {
retryBackend := New(be, 10, nil, nil) retryBackend := New(be, 10, nil, nil)
var listed []string var listed []string
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
listed = append(listed, fi.Name) listed = append(listed, fi.Name)
return nil return nil
}) })
@ -121,10 +122,10 @@ func TestBackendListRetryErrorFn(t *testing.T) {
var names = []string{"id1", "id2", "foo", "bar"} var names = []string{"id1", "id2", "foo", "bar"}
be := &mock.Backend{ be := &mock.Backend{
ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error { ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error {
t.Logf("List called for %v", tpe) t.Logf("List called for %v", tpe)
for _, name := range names { for _, name := range names {
err := fn(restic.FileInfo{Name: name}) err := fn(backend.FileInfo{Name: name})
if err != nil { if err != nil {
return err return err
} }
@ -141,7 +142,7 @@ func TestBackendListRetryErrorFn(t *testing.T) {
var listed []string var listed []string
run := 0 run := 0
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
t.Logf("fn called for %v", fi.Name) t.Logf("fn called for %v", fi.Name)
run++ run++
// return an error for the third item in the list // return an error for the third item in the list
@ -172,7 +173,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
retries := 0 retries := 0
be := &mock.Backend{ be := &mock.Backend{
ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error { ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error {
t.Logf("List called for %v, retries %v", tpe, retries) t.Logf("List called for %v, retries %v", tpe, retries)
retries++ retries++
for i, name := range names { for i, name := range names {
@ -180,7 +181,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
return ErrBackendTest return ErrBackendTest
} }
err := fn(restic.FileInfo{Name: name}) err := fn(backend.FileInfo{Name: name})
if err != nil { if err != nil {
return err return err
} }
@ -195,7 +196,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
retryBackend := New(be, maxRetries, nil, nil) retryBackend := New(be, maxRetries, nil, nil)
var listed []string var listed []string
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
t.Logf("fn called for %v", fi.Name) t.Logf("fn called for %v", fi.Name)
listed = append(listed, fi.Name) listed = append(listed, fi.Name)
return nil return nil
@ -252,7 +253,7 @@ func TestBackendLoadRetry(t *testing.T) {
attempt := 0 attempt := 0
be := mock.NewBackend() be := mock.NewBackend()
be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
// returns failing reader on first invocation, good reader on subsequent invocations // returns failing reader on first invocation, good reader on subsequent invocations
attempt++ attempt++
if attempt > 1 { if attempt > 1 {
@ -265,7 +266,7 @@ func TestBackendLoadRetry(t *testing.T) {
retryBackend := New(be, 10, nil, nil) retryBackend := New(be, 10, nil, nil)
var buf []byte var buf []byte
err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) { err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
buf, err = io.ReadAll(rd) buf, err = io.ReadAll(rd)
return err return err
}) })
@ -280,7 +281,7 @@ func TestBackendLoadNotExists(t *testing.T) {
attempt := 0 attempt := 0
be := mock.NewBackend() be := mock.NewBackend()
be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
attempt++ attempt++
if attempt > 1 { if attempt > 1 {
t.Fail() t.Fail()
@ -295,7 +296,7 @@ func TestBackendLoadNotExists(t *testing.T) {
TestFastRetries(t) TestFastRetries(t)
retryBackend := New(be, 10, nil, nil) retryBackend := New(be, 10, nil, nil)
err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) { err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
return nil return nil
}) })
test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err) test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err)
@ -308,13 +309,13 @@ func TestBackendStatNotExists(t *testing.T) {
attempt := 0 attempt := 0
be := mock.NewBackend() be := mock.NewBackend()
be.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { be.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
attempt++ attempt++
if attempt > 1 { if attempt > 1 {
t.Fail() t.Fail()
return restic.FileInfo{}, errors.New("must not retry") return backend.FileInfo{}, errors.New("must not retry")
} }
return restic.FileInfo{}, notFound return backend.FileInfo{}, notFound
} }
be.IsNotExistFn = func(err error) bool { be.IsNotExistFn = func(err error) bool {
return errors.Is(err, notFound) return errors.Is(err, notFound)
@ -323,7 +324,7 @@ func TestBackendStatNotExists(t *testing.T) {
TestFastRetries(t) TestFastRetries(t)
retryBackend := New(be, 10, nil, nil) retryBackend := New(be, 10, nil, nil)
_, err := retryBackend.Stat(context.TODO(), restic.Handle{}) _, err := retryBackend.Stat(context.TODO(), backend.Handle{})
test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err) test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err)
test.Equals(t, 1, attempt) test.Equals(t, 1, attempt)
} }
@ -337,7 +338,7 @@ func TestBackendCanceledContext(t *testing.T) {
// check that we received the expected context canceled error instead // check that we received the expected context canceled error instead
TestFastRetries(t) TestFastRetries(t)
retryBackend := New(mock.NewBackend(), 2, nil, nil) retryBackend := New(mock.NewBackend(), 2, nil, nil)
h := restic.Handle{Type: restic.PackFile, Name: restic.NewRandomID().String()} h := backend.Handle{Type: backend.PackFile, Name: restic.NewRandomID().String()}
// create an already canceled context // create an already canceled context
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
@ -346,15 +347,15 @@ func TestBackendCanceledContext(t *testing.T) {
_, err := retryBackend.Stat(ctx, h) _, err := retryBackend.Stat(ctx, h)
assertIsCanceled(t, err) assertIsCanceled(t, err)
err = retryBackend.Save(ctx, h, restic.NewByteReader([]byte{}, nil)) err = retryBackend.Save(ctx, h, backend.NewByteReader([]byte{}, nil))
assertIsCanceled(t, err) assertIsCanceled(t, err)
err = retryBackend.Remove(ctx, h) err = retryBackend.Remove(ctx, h)
assertIsCanceled(t, err) assertIsCanceled(t, err)
err = retryBackend.Load(ctx, restic.Handle{}, 0, 0, func(rd io.Reader) (err error) { err = retryBackend.Load(ctx, backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
return nil return nil
}) })
assertIsCanceled(t, err) assertIsCanceled(t, err)
err = retryBackend.List(ctx, restic.PackFile, func(restic.FileInfo) error { err = retryBackend.List(ctx, backend.PackFile, func(backend.FileInfo) error {
return nil return nil
}) })
assertIsCanceled(t, err) assertIsCanceled(t, err)

2
internal/restic/rewind_reader.go → internal/backend/rewind_reader.go
View File

@ -1,4 +1,4 @@
package restic package backend
import ( import (
"bytes" "bytes"

2
internal/restic/rewind_reader_test.go → internal/backend/rewind_reader_test.go
View File

@ -1,4 +1,4 @@
package restic package backend
import ( import (
"bytes" "bytes"

4
internal/backend/s3/config.go
View File

@ -6,9 +6,9 @@ import (
"path" "path"
"strings" "strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options" "github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
) )
// Config contains all configuration necessary to connect to an s3 compatible // Config contains all configuration necessary to connect to an s3 compatible
@ -94,7 +94,7 @@ func createConfig(endpoint, bucket, prefix string, useHTTP bool) (*Config, error
return &cfg, nil return &cfg, nil
} }
var _ restic.ApplyEnvironmenter = &Config{} var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config. // ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) { func (cfg *Config) ApplyEnvironment(prefix string) {

30
internal/backend/s3/s3.go
View File

@ -11,12 +11,12 @@ import (
"strings" "strings"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/credentials"
@ -30,7 +30,7 @@ type Backend struct {
} }
// make sure that *Backend implements backend.Backend // make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{} var _ backend.Backend = &Backend{}
func NewFactory() location.Factory { func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open) return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open)
@ -127,13 +127,13 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro
// Open opens the S3 backend at bucket and region. The bucket is created if it // Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet. // does not exist yet.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
return open(ctx, cfg, rt) return open(ctx, cfg, rt)
} }
// Create opens the S3 backend at bucket and region and creates the bucket if // Create opens the S3 backend at bucket and region and creates the bucket if
// it does not exist yet. // it does not exist yet.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
be, err := open(ctx, cfg, rt) be, err := open(ctx, cfg, rt)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "open") return nil, errors.Wrap(err, "open")
@ -272,7 +272,7 @@ func (be *Backend) Path() string {
} }
// Save stores data in the backend at the handle. // Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h) objName := be.Filename(h)
opts := minio.PutObjectOptions{StorageClass: be.cfg.StorageClass} opts := minio.PutObjectOptions{StorageClass: be.cfg.StorageClass}
@ -294,14 +294,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
// Load runs fn with a reader that yields the contents of the file at h at the // Load runs fn with a reader that yields the contents of the file at h at the
// given offset. // given offset.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
} }
func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
objName := be.Filename(h) objName := be.Filename(h)
opts := minio.GetObjectOptions{} opts := minio.GetObjectOptions{}
@ -326,7 +326,7 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int,
} }
// Stat returns information about a blob. // Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h) objName := be.Filename(h)
var obj *minio.Object var obj *minio.Object
@ -334,7 +334,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
obj, err = be.client.GetObject(ctx, be.cfg.Bucket, objName, opts) obj, err = be.client.GetObject(ctx, be.cfg.Bucket, objName, opts)
if err != nil { if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "client.GetObject") return backend.FileInfo{}, errors.Wrap(err, "client.GetObject")
} }
// make sure that the object is closed properly. // make sure that the object is closed properly.
@ -347,14 +347,14 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
fi, err := obj.Stat() fi, err := obj.Stat()
if err != nil { if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Stat") return backend.FileInfo{}, errors.Wrap(err, "Stat")
} }
return restic.FileInfo{Size: fi.Size, Name: h.Name}, nil return backend.FileInfo{Size: fi.Size, Name: h.Name}, nil
} }
// Remove removes the blob with the given name and type. // Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h) objName := be.Filename(h)
err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{}) err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{})
@ -368,7 +368,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an // List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it. // error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, recursive := be.Basedir(t) prefix, recursive := be.Basedir(t)
// make sure prefix ends with a slash // make sure prefix ends with a slash
@ -400,7 +400,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue continue
} }
fi := restic.FileInfo{ fi := backend.FileInfo{
Name: path.Base(m), Name: path.Base(m),
Size: obj.Size, Size: obj.Size,
} }
@ -431,7 +431,7 @@ func (be *Backend) Delete(ctx context.Context) error {
func (be *Backend) Close() error { return nil } func (be *Backend) Close() error { return nil }
// Rename moves a file based on the new layout l. // Rename moves a file based on the new layout l.
func (be *Backend) Rename(ctx context.Context, h restic.Handle, l layout.Layout) error { func (be *Backend) Rename(ctx context.Context, h backend.Handle, l layout.Layout) error {
debug.Log("Rename %v to %v", h, l) debug.Log("Rename %v to %v", h, l)
oldname := be.Filename(h) oldname := be.Filename(h)
newname := l.Filename(h) newname := l.Filename(h)

4
internal/backend/s3/s3_test.go
View File

@ -14,11 +14,11 @@ import (
"testing" "testing"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/options" "github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
@ -117,7 +117,7 @@ func newMinioTestSuite(t testing.TB) (*test.Suite[s3.Config], func()) {
return &cfg, nil return &cfg, nil
}, },
Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be restic.Backend, err error) { Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be backend.Backend, err error) {
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
be, err = s3.Create(ctx, cfg, rt) be, err = s3.Create(ctx, cfg, rt)
if err != nil { if err != nil {

28
internal/backend/sema/backend.go
View File

@ -6,22 +6,22 @@ import (
"sync" "sync"
"github.com/cenkalti/backoff/v4" "github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
) )
// make sure that connectionLimitedBackend implements restic.Backend // make sure that connectionLimitedBackend implements backend.Backend
var _ restic.Backend = &connectionLimitedBackend{} var _ backend.Backend = &connectionLimitedBackend{}
// connectionLimitedBackend limits the number of concurrent operations. // connectionLimitedBackend limits the number of concurrent operations.
type connectionLimitedBackend struct { type connectionLimitedBackend struct {
restic.Backend backend.Backend
sem semaphore sem semaphore
freezeLock sync.Mutex freezeLock sync.Mutex
} }
// NewBackend creates a backend that limits the concurrent operations on the underlying backend // NewBackend creates a backend that limits the concurrent operations on the underlying backend
func NewBackend(be restic.Backend) restic.Backend { func NewBackend(be backend.Backend) backend.Backend {
sem, err := newSemaphore(be.Connections()) sem, err := newSemaphore(be.Connections())
if err != nil { if err != nil {
panic(err) panic(err)
@ -35,9 +35,9 @@ func NewBackend(be restic.Backend) restic.Backend {
// typeDependentLimit acquire a token unless the FileType is a lock file. The returned function // typeDependentLimit acquire a token unless the FileType is a lock file. The returned function
// must be called to release the token. // must be called to release the token.
func (be *connectionLimitedBackend) typeDependentLimit(t restic.FileType) func() { func (be *connectionLimitedBackend) typeDependentLimit(t backend.FileType) func() {
// allow concurrent lock file operations to ensure that the lock refresh is always possible // allow concurrent lock file operations to ensure that the lock refresh is always possible
if t == restic.LockFile { if t == backend.LockFile {
return func() {} return func() {}
} }
be.sem.GetToken() be.sem.GetToken()
@ -59,7 +59,7 @@ func (be *connectionLimitedBackend) Unfreeze() {
} }
// Save adds new Data to the backend. // Save adds new Data to the backend.
func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (be *connectionLimitedBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if err := h.Valid(); err != nil { if err := h.Valid(); err != nil {
return backoff.Permanent(err) return backoff.Permanent(err)
} }
@ -75,7 +75,7 @@ func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, r
// Load runs fn with a reader that yields the contents of the file at h at the // Load runs fn with a reader that yields the contents of the file at h at the
// given offset. // given offset.
func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { func (be *connectionLimitedBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
if err := h.Valid(); err != nil { if err := h.Valid(); err != nil {
return backoff.Permanent(err) return backoff.Permanent(err)
} }
@ -96,22 +96,22 @@ func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, l
} }
// Stat returns information about a file in the backend. // Stat returns information about a file in the backend.
func (be *connectionLimitedBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { func (be *connectionLimitedBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
if err := h.Valid(); err != nil { if err := h.Valid(); err != nil {
return restic.FileInfo{}, backoff.Permanent(err) return backend.FileInfo{}, backoff.Permanent(err)
} }
defer be.typeDependentLimit(h.Type)() defer be.typeDependentLimit(h.Type)()
if ctx.Err() != nil { if ctx.Err() != nil {
return restic.FileInfo{}, ctx.Err() return backend.FileInfo{}, ctx.Err()
} }
return be.Backend.Stat(ctx, h) return be.Backend.Stat(ctx, h)
} }
// Remove deletes a file from the backend. // Remove deletes a file from the backend.
func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle) error { func (be *connectionLimitedBackend) Remove(ctx context.Context, h backend.Handle) error {
if err := h.Valid(); err != nil { if err := h.Valid(); err != nil {
return backoff.Permanent(err) return backoff.Permanent(err)
} }
@ -125,6 +125,6 @@ func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle)
return be.Backend.Remove(ctx, h) return be.Backend.Remove(ctx, h)
} }
func (be *connectionLimitedBackend) Unwrap() restic.Backend { func (be *connectionLimitedBackend) Unwrap() backend.Backend {
return be.Backend return be.Backend
} }

64
internal/backend/sema/backend_test.go
View File

@ -8,37 +8,37 @@ import (
"testing" "testing"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test" "github.com/restic/restic/internal/test"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
func TestParameterValidationSave(t *testing.T) { func TestParameterValidationSave(t *testing.T) {
m := mock.NewBackend() m := mock.NewBackend()
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
return nil return nil
} }
be := sema.NewBackend(m) be := sema.NewBackend(m)
err := be.Save(context.TODO(), restic.Handle{}, nil) err := be.Save(context.TODO(), backend.Handle{}, nil)
test.Assert(t, err != nil, "Save() with invalid handle did not return an error") test.Assert(t, err != nil, "Save() with invalid handle did not return an error")
} }
func TestParameterValidationLoad(t *testing.T) { func TestParameterValidationLoad(t *testing.T) {
m := mock.NewBackend() m := mock.NewBackend()
m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { m.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return io.NopCloser(nil), nil return io.NopCloser(nil), nil
} }
be := sema.NewBackend(m) be := sema.NewBackend(m)
nilCb := func(rd io.Reader) error { return nil } nilCb := func(rd io.Reader) error { return nil }
err := be.Load(context.TODO(), restic.Handle{}, 10, 0, nilCb) err := be.Load(context.TODO(), backend.Handle{}, 10, 0, nilCb)
test.Assert(t, err != nil, "Load() with invalid handle did not return an error") test.Assert(t, err != nil, "Load() with invalid handle did not return an error")
h := restic.Handle{Type: restic.PackFile, Name: "foobar"} h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
err = be.Load(context.TODO(), h, 10, -1, nilCb) err = be.Load(context.TODO(), h, 10, -1, nilCb)
test.Assert(t, err != nil, "Save() with negative offset did not return an error") test.Assert(t, err != nil, "Save() with negative offset did not return an error")
err = be.Load(context.TODO(), h, -1, 0, nilCb) err = be.Load(context.TODO(), h, -1, 0, nilCb)
@ -47,23 +47,23 @@ func TestParameterValidationLoad(t *testing.T) {
func TestParameterValidationStat(t *testing.T) { func TestParameterValidationStat(t *testing.T) {
m := mock.NewBackend() m := mock.NewBackend()
m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { m.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
return restic.FileInfo{}, nil return backend.FileInfo{}, nil
} }
be := sema.NewBackend(m) be := sema.NewBackend(m)
_, err := be.Stat(context.TODO(), restic.Handle{}) _, err := be.Stat(context.TODO(), backend.Handle{})
test.Assert(t, err != nil, "Stat() with invalid handle did not return an error") test.Assert(t, err != nil, "Stat() with invalid handle did not return an error")
} }
func TestParameterValidationRemove(t *testing.T) { func TestParameterValidationRemove(t *testing.T) {
m := mock.NewBackend() m := mock.NewBackend()
m.RemoveFn = func(ctx context.Context, h restic.Handle) error { m.RemoveFn = func(ctx context.Context, h backend.Handle) error {
return nil return nil
} }
be := sema.NewBackend(m) be := sema.NewBackend(m)
err := be.Remove(context.TODO(), restic.Handle{}) err := be.Remove(context.TODO(), backend.Handle{})
test.Assert(t, err != nil, "Remove() with invalid handle did not return an error") test.Assert(t, err != nil, "Remove() with invalid handle did not return an error")
} }
@ -71,7 +71,7 @@ func TestUnwrap(t *testing.T) {
m := mock.NewBackend() m := mock.NewBackend()
be := sema.NewBackend(m) be := sema.NewBackend(m)
unwrapper := be.(restic.BackendUnwrapper) unwrapper := be.(backend.Unwrapper)
test.Assert(t, unwrapper.Unwrap() == m, "Unwrap() returned wrong backend") test.Assert(t, unwrapper.Unwrap() == m, "Unwrap() returned wrong backend")
} }
@ -100,7 +100,7 @@ func countingBlocker() (func(), func(int) int) {
return wait, unblock return wait, unblock
} }
func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be restic.Backend) func() error, unblock func(int) int, isUnlimited bool) { func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be backend.Backend) func() error, unblock func(int) int, isUnlimited bool) {
expectBlocked := int(2) expectBlocked := int(2)
workerCount := expectBlocked + 1 workerCount := expectBlocked + 1
@ -125,13 +125,13 @@ func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(b
func TestConcurrencyLimitSave(t *testing.T) { func TestConcurrencyLimitSave(t *testing.T) {
wait, unblock := countingBlocker() wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) { concurrencyTester(t, func(m *mock.Backend) {
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
wait() wait()
return nil return nil
} }
}, func(be restic.Backend) func() error { }, func(be backend.Backend) func() error {
return func() error { return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"} h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
return be.Save(context.TODO(), h, nil) return be.Save(context.TODO(), h, nil)
} }
}, unblock, false) }, unblock, false)
@ -140,13 +140,13 @@ func TestConcurrencyLimitSave(t *testing.T) {
func TestConcurrencyLimitLoad(t *testing.T) { func TestConcurrencyLimitLoad(t *testing.T) {
wait, unblock := countingBlocker() wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) { concurrencyTester(t, func(m *mock.Backend) {
m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { m.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
wait() wait()
return io.NopCloser(nil), nil return io.NopCloser(nil), nil
} }
}, func(be restic.Backend) func() error { }, func(be backend.Backend) func() error {
return func() error { return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"} h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
nilCb := func(rd io.Reader) error { return nil } nilCb := func(rd io.Reader) error { return nil }
return be.Load(context.TODO(), h, 10, 0, nilCb) return be.Load(context.TODO(), h, 10, 0, nilCb)
} }
@ -156,13 +156,13 @@ func TestConcurrencyLimitLoad(t *testing.T) {
func TestConcurrencyLimitStat(t *testing.T) { func TestConcurrencyLimitStat(t *testing.T) {
wait, unblock := countingBlocker() wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) { concurrencyTester(t, func(m *mock.Backend) {
m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { m.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
wait() wait()
return restic.FileInfo{}, nil return backend.FileInfo{}, nil
} }
}, func(be restic.Backend) func() error { }, func(be backend.Backend) func() error {
return func() error { return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"} h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
_, err := be.Stat(context.TODO(), h) _, err := be.Stat(context.TODO(), h)
return err return err
} }
@ -172,13 +172,13 @@ func TestConcurrencyLimitStat(t *testing.T) {
func TestConcurrencyLimitDelete(t *testing.T) { func TestConcurrencyLimitDelete(t *testing.T) {
wait, unblock := countingBlocker() wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) { concurrencyTester(t, func(m *mock.Backend) {
m.RemoveFn = func(ctx context.Context, h restic.Handle) error { m.RemoveFn = func(ctx context.Context, h backend.Handle) error {
wait() wait()
return nil return nil
} }
}, func(be restic.Backend) func() error { }, func(be backend.Backend) func() error {
return func() error { return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"} h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
return be.Remove(context.TODO(), h) return be.Remove(context.TODO(), h)
} }
}, unblock, false) }, unblock, false)
@ -187,13 +187,13 @@ func TestConcurrencyLimitDelete(t *testing.T) {
func TestConcurrencyUnlimitedLockSave(t *testing.T) { func TestConcurrencyUnlimitedLockSave(t *testing.T) {
wait, unblock := countingBlocker() wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) { concurrencyTester(t, func(m *mock.Backend) {
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
wait() wait()
return nil return nil
} }
}, func(be restic.Backend) func() error { }, func(be backend.Backend) func() error {
return func() error { return func() error {
h := restic.Handle{Type: restic.LockFile, Name: "foobar"} h := backend.Handle{Type: backend.LockFile, Name: "foobar"}
return be.Save(context.TODO(), h, nil) return be.Save(context.TODO(), h, nil)
} }
}, unblock, true) }, unblock, true)
@ -202,13 +202,13 @@ func TestConcurrencyUnlimitedLockSave(t *testing.T) {
func TestFreeze(t *testing.T) { func TestFreeze(t *testing.T) {
var counter int64 var counter int64
m := mock.NewBackend() m := mock.NewBackend()
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
atomic.AddInt64(&counter, 1) atomic.AddInt64(&counter, 1)
return nil return nil
} }
m.ConnectionsFn = func() uint { return 2 } m.ConnectionsFn = func() uint { return 2 }
be := sema.NewBackend(m) be := sema.NewBackend(m)
fb := be.(restic.FreezeBackend) fb := be.(backend.FreezeBackend)
// Freeze backend // Freeze backend
fb.Freeze() fb.Freeze()
@ -218,7 +218,7 @@ func TestFreeze(t *testing.T) {
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
h := restic.Handle{Type: restic.PackFile, Name: "foobar"} h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
test.OK(t, be.Save(context.TODO(), h, nil)) test.OK(t, be.Save(context.TODO(), h, nil))
}() }()

4
internal/backend/sftp/layout_test.go
View File

@ -6,8 +6,8 @@ import (
"path/filepath" "path/filepath"
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/sftp" "github.com/restic/restic/internal/backend/sftp"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
@ -56,7 +56,7 @@ func TestLayout(t *testing.T) {
} }
packs := make(map[string]bool) packs := make(map[string]bool)
err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
packs[fi.Name] = false packs[fi.Name] = false
return nil return nil
}) })

27
internal/backend/sftp/sftp.go
View File

@ -20,7 +20,6 @@ import (
"github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/cenkalti/backoff/v4" "github.com/cenkalti/backoff/v4"
"github.com/pkg/sftp" "github.com/pkg/sftp"
@ -42,7 +41,7 @@ type SFTP struct {
util.Modes util.Modes
} }
var _ restic.Backend = &SFTP{} var _ backend.Backend = &SFTP{}
func NewFactory() location.Factory { func NewFactory() location.Factory {
return location.NewLimitedBackendFactory("sftp", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) return location.NewLimitedBackendFactory("sftp", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open))
@ -153,7 +152,7 @@ func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) {
debug.Log("layout: %v\n", sftp.Layout) debug.Log("layout: %v\n", sftp.Layout)
fi, err := sftp.c.Stat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile})) fi, err := sftp.c.Stat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile}))
m := util.DeriveModesFromFileInfo(fi, err) m := util.DeriveModesFromFileInfo(fi, err)
debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir)
@ -263,7 +262,7 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) {
sftp.Modes = util.DefaultModes sftp.Modes = util.DefaultModes
// test if config file already exists // test if config file already exists
_, err = sftp.c.Lstat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile})) _, err = sftp.c.Lstat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile}))
if err == nil { if err == nil {
return nil, errors.New("config file already exists") return nil, errors.New("config file already exists")
} }
@ -314,7 +313,7 @@ func tempSuffix() string {
} }
// Save stores data in the backend at the handle. // Save stores data in the backend at the handle.
func (r *SFTP) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) error { func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) error {
if err := r.clientError(); err != nil { if err := r.clientError(); err != nil {
return err return err
} }
@ -414,11 +413,11 @@ func (r *SFTP) checkNoSpace(dir string, size int64, origErr error) error {
// Load runs fn with a reader that yields the contents of the file at h at the // Load runs fn with a reader that yields the contents of the file at h at the
// given offset. // given offset.
func (r *SFTP) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { func (r *SFTP) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, r.openReader, fn) return util.DefaultLoad(ctx, h, length, offset, r.openReader, fn)
} }
func (r *SFTP) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { func (r *SFTP) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
f, err := r.c.Open(r.Filename(h)) f, err := r.c.Open(r.Filename(h))
if err != nil { if err != nil {
return nil, err return nil, err
@ -442,21 +441,21 @@ func (r *SFTP) openReader(_ context.Context, h restic.Handle, length int, offset
} }
// Stat returns information about a blob. // Stat returns information about a blob.
func (r *SFTP) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) { func (r *SFTP) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) {
if err := r.clientError(); err != nil { if err := r.clientError(); err != nil {
return restic.FileInfo{}, err return backend.FileInfo{}, err
} }
fi, err := r.c.Lstat(r.Filename(h)) fi, err := r.c.Lstat(r.Filename(h))
if err != nil { if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Lstat") return backend.FileInfo{}, errors.Wrap(err, "Lstat")
} }
return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil
} }
// Remove removes the content stored at name. // Remove removes the content stored at name.
func (r *SFTP) Remove(_ context.Context, h restic.Handle) error { func (r *SFTP) Remove(_ context.Context, h backend.Handle) error {
if err := r.clientError(); err != nil { if err := r.clientError(); err != nil {
return err return err
} }
@ -466,7 +465,7 @@ func (r *SFTP) Remove(_ context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an // List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it. // error occurs (or fn returns an error), List stops and returns it.
func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { func (r *SFTP) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
basedir, subdirs := r.Basedir(t) basedir, subdirs := r.Basedir(t)
walker := r.c.Walk(basedir) walker := r.c.Walk(basedir)
for { for {
@ -499,7 +498,7 @@ func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileI
debug.Log("send %v\n", path.Base(walker.Path())) debug.Log("send %v\n", path.Base(walker.Path()))
rfi := restic.FileInfo{ rfi := backend.FileInfo{
Name: path.Base(walker.Path()), Name: path.Base(walker.Path()),
Size: fi.Size(), Size: fi.Size(),
} }

4
internal/backend/swift/config.go
View File

@ -4,9 +4,9 @@ import (
"os" "os"
"strings" "strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options" "github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
) )
// Config contains basic configuration needed to specify swift location for a swift server // Config contains basic configuration needed to specify swift location for a swift server
@ -74,7 +74,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil return &cfg, nil
} }
var _ restic.ApplyEnvironmenter = &Config{} var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config. // ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) { func (cfg *Config) ApplyEnvironment(prefix string) {

26
internal/backend/swift/swift.go
View File

@ -13,12 +13,12 @@ import (
"strings" "strings"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/ncw/swift/v2" "github.com/ncw/swift/v2"
) )
@ -32,8 +32,8 @@ type beSwift struct {
layout.Layout layout.Layout
} }
// ensure statically that *beSwift implements restic.Backend. // ensure statically that *beSwift implements backend.Backend.
var _ restic.Backend = &beSwift{} var _ backend.Backend = &beSwift{}
func NewFactory() location.Factory { func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("swift", ParseConfig, location.NoPassword, Open, Open) return location.NewHTTPBackendFactory("swift", ParseConfig, location.NoPassword, Open, Open)
@ -41,7 +41,7 @@ func NewFactory() location.Factory {
// Open opens the swift backend at a container in region. The container is // Open opens the swift backend at a container in region. The container is
// created if it does not exist yet. // created if it does not exist yet.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
debug.Log("config %#v", cfg) debug.Log("config %#v", cfg)
be := &beSwift{ be := &beSwift{
@ -134,11 +134,11 @@ func (be *beSwift) HasAtomicReplace() bool {
// Load runs fn with a reader that yields the contents of the file at h at the // Load runs fn with a reader that yields the contents of the file at h at the
// given offset. // given offset.
func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { func (be *beSwift) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
} }
func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { func (be *beSwift) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
objName := be.Filename(h) objName := be.Filename(h)
@ -160,7 +160,7 @@ func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int,
} }
// Save stores data in the backend at the handle. // Save stores data in the backend at the handle.
func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (be *beSwift) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h) objName := be.Filename(h)
encoding := "binary/octet-stream" encoding := "binary/octet-stream"
@ -174,19 +174,19 @@ func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
} }
// Stat returns information about a blob. // Stat returns information about a blob.
func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { func (be *beSwift) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h) objName := be.Filename(h)
obj, _, err := be.conn.Object(ctx, be.container, objName) obj, _, err := be.conn.Object(ctx, be.container, objName)
if err != nil { if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "conn.Object") return backend.FileInfo{}, errors.Wrap(err, "conn.Object")
} }
return restic.FileInfo{Size: obj.Bytes, Name: h.Name}, nil return backend.FileInfo{Size: obj.Bytes, Name: h.Name}, nil
} }
// Remove removes the blob with the given name and type. // Remove removes the blob with the given name and type.
func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error { func (be *beSwift) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h) objName := be.Filename(h)
err := be.conn.ObjectDelete(ctx, be.container, objName) err := be.conn.ObjectDelete(ctx, be.container, objName)
@ -195,7 +195,7 @@ func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an // List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it. // error occurs (or fn returns an error), List stops and returns it.
func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { func (be *beSwift) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, _ := be.Basedir(t) prefix, _ := be.Basedir(t)
prefix += "/" prefix += "/"
@ -212,7 +212,7 @@ func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue continue
} }
fi := restic.FileInfo{ fi := backend.FileInfo{
Name: m, Name: m,
Size: obj.Bytes, Size: obj.Bytes,
} }

4
internal/backend/swift/swift_test.go
View File

@ -6,9 +6,9 @@ import (
"testing" "testing"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/swift" "github.com/restic/restic/internal/backend/swift"
"github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
@ -20,7 +20,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] {
// wait for removals for at least 5m // wait for removals for at least 5m
WaitForDelayedRemoval: 5 * time.Minute, WaitForDelayedRemoval: 5 * time.Minute,
ErrorHandler: func(t testing.TB, be restic.Backend, err error) error { ErrorHandler: func(t testing.TB, be backend.Backend, err error) error {
if err == nil { if err == nil {
return nil return nil
} }

13
internal/backend/test/benchmarks.go
View File

@ -6,22 +6,23 @@ import (
"io" "io"
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test" "github.com/restic/restic/internal/test"
) )
func saveRandomFile(t testing.TB, be restic.Backend, length int) ([]byte, restic.Handle) { func saveRandomFile(t testing.TB, be backend.Backend, length int) ([]byte, backend.Handle) {
data := test.Random(23, length) data := test.Random(23, length)
id := restic.Hash(data) id := restic.Hash(data)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()} handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := be.Save(context.TODO(), handle, restic.NewByteReader(data, be.Hasher())) err := be.Save(context.TODO(), handle, backend.NewByteReader(data, be.Hasher()))
if err != nil { if err != nil {
t.Fatalf("Save() error: %+v", err) t.Fatalf("Save() error: %+v", err)
} }
return data, handle return data, handle
} }
func remove(t testing.TB, be restic.Backend, h restic.Handle) { func remove(t testing.TB, be backend.Backend, h backend.Handle) {
if err := be.Remove(context.TODO(), h); err != nil { if err := be.Remove(context.TODO(), h); err != nil {
t.Fatalf("Remove() returned error: %v", err) t.Fatalf("Remove() returned error: %v", err)
} }
@ -146,9 +147,9 @@ func (s *Suite[C]) BenchmarkSave(t *testing.B) {
length := 1<<24 + 2123 length := 1<<24 + 2123
data := test.Random(23, length) data := test.Random(23, length)
id := restic.Hash(data) id := restic.Hash(data)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()} handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
rd := restic.NewByteReader(data, be.Hasher()) rd := backend.NewByteReader(data, be.Hasher())
t.SetBytes(int64(length)) t.SetBytes(int64(length))
t.ResetTimer() t.ResetTimer()

2
internal/backend/test/doc.go
View File

@ -17,7 +17,7 @@
// //
// func newTestSuite(t testing.TB) *test.Suite { // func newTestSuite(t testing.TB) *test.Suite {
// return &test.Suite{ // return &test.Suite{
// Create: func(cfg interface{}) (restic.Backend, error) { // Create: func(cfg interface{}) (backend.Backend, error) {
// [...] // [...]
// }, // },
// [...] // [...]

13
internal/backend/test/suite.go
View File

@ -11,7 +11,6 @@ import (
"github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test" "github.com/restic/restic/internal/test"
) )
@ -35,7 +34,7 @@ type Suite[C any] struct {
WaitForDelayedRemoval time.Duration WaitForDelayedRemoval time.Duration
// ErrorHandler allows ignoring certain errors. // ErrorHandler allows ignoring certain errors.
ErrorHandler func(testing.TB, restic.Backend, error) error ErrorHandler func(testing.TB, backend.Backend, error) error
} }
// RunTests executes all defined tests as subtests of t. // RunTests executes all defined tests as subtests of t.
@ -156,7 +155,7 @@ func (s *Suite[C]) RunBenchmarks(b *testing.B) {
s.cleanup(b) s.cleanup(b)
} }
func (s *Suite[C]) createOrError() (restic.Backend, error) { func (s *Suite[C]) createOrError() (backend.Backend, error) {
tr, err := backend.Transport(backend.TransportOptions{}) tr, err := backend.Transport(backend.TransportOptions{})
if err != nil { if err != nil {
return nil, fmt.Errorf("cannot create transport for tests: %v", err) return nil, fmt.Errorf("cannot create transport for tests: %v", err)
@ -167,7 +166,7 @@ func (s *Suite[C]) createOrError() (restic.Backend, error) {
return nil, err return nil, err
} }
_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile}) _, err = be.Stat(context.TODO(), backend.Handle{Type: backend.ConfigFile})
if err != nil && !be.IsNotExist(err) { if err != nil && !be.IsNotExist(err) {
return nil, err return nil, err
} }
@ -179,7 +178,7 @@ func (s *Suite[C]) createOrError() (restic.Backend, error) {
return be, nil return be, nil
} }
func (s *Suite[C]) create(t testing.TB) restic.Backend { func (s *Suite[C]) create(t testing.TB) backend.Backend {
be, err := s.createOrError() be, err := s.createOrError()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -187,7 +186,7 @@ func (s *Suite[C]) create(t testing.TB) restic.Backend {
return be return be
} }
func (s *Suite[C]) open(t testing.TB) restic.Backend { func (s *Suite[C]) open(t testing.TB) backend.Backend {
tr, err := backend.Transport(backend.TransportOptions{}) tr, err := backend.Transport(backend.TransportOptions{})
if err != nil { if err != nil {
t.Fatalf("cannot create transport for tests: %v", err) t.Fatalf("cannot create transport for tests: %v", err)
@ -208,7 +207,7 @@ func (s *Suite[C]) cleanup(t testing.TB) {
s.close(t, be) s.close(t, be)
} }
func (s *Suite[C]) close(t testing.TB, be restic.Backend) { func (s *Suite[C]) close(t testing.TB, be backend.Backend) {
err := be.Close() err := be.Close()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)

100
internal/backend/test/tests.go
View File

@ -27,7 +27,7 @@ func seedRand(t testing.TB) {
t.Logf("rand initialized with seed %d", seed) t.Logf("rand initialized with seed %d", seed)
} }
func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, error) { func beTest(ctx context.Context, be backend.Backend, h backend.Handle) (bool, error) {
_, err := be.Stat(ctx, h) _, err := be.Stat(ctx, h)
if err != nil && be.IsNotExist(err) { if err != nil && be.IsNotExist(err) {
return false, nil return false, nil
@ -49,7 +49,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
defer s.close(t, b) defer s.close(t, b)
// remove a config if present // remove a config if present
cfgHandle := restic.Handle{Type: restic.ConfigFile} cfgHandle := backend.Handle{Type: backend.ConfigFile}
cfgPresent, err := beTest(context.TODO(), b, cfgHandle) cfgPresent, err := beTest(context.TODO(), b, cfgHandle)
if err != nil { if err != nil {
t.Fatalf("unable to test for config: %+v", err) t.Fatalf("unable to test for config: %+v", err)
@ -60,7 +60,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
} }
// save a config // save a config
store(t, b, restic.ConfigFile, []byte("test config")) store(t, b, backend.ConfigFile, []byte("test config"))
// now create the backend again, this must fail // now create the backend again, this must fail
_, err = s.createOrError() _, err = s.createOrError()
@ -69,7 +69,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
} }
// remove config // remove config
err = b.Remove(context.TODO(), restic.Handle{Type: restic.ConfigFile, Name: ""}) err = b.Remove(context.TODO(), backend.Handle{Type: backend.ConfigFile, Name: ""})
if err != nil { if err != nil {
t.Fatalf("unexpected error removing config: %+v", err) t.Fatalf("unexpected error removing config: %+v", err)
} }
@ -94,13 +94,13 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
var testString = "Config" var testString = "Config"
// create config and read it back // create config and read it back
_, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.ConfigFile}) _, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.ConfigFile})
if err == nil { if err == nil {
t.Fatalf("did not get expected error for non-existing config") t.Fatalf("did not get expected error for non-existing config")
} }
test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize error from LoadAll(): %v", err) test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize error from LoadAll(): %v", err)
err = b.Save(context.TODO(), restic.Handle{Type: restic.ConfigFile}, restic.NewByteReader([]byte(testString), b.Hasher())) err = b.Save(context.TODO(), backend.Handle{Type: backend.ConfigFile}, backend.NewByteReader([]byte(testString), b.Hasher()))
if err != nil { if err != nil {
t.Fatalf("Save() error: %+v", err) t.Fatalf("Save() error: %+v", err)
} }
@ -108,7 +108,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
// try accessing the config with different names, should all return the // try accessing the config with different names, should all return the
// same config // same config
for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} { for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
h := restic.Handle{Type: restic.ConfigFile, Name: name} h := backend.Handle{Type: backend.ConfigFile, Name: name}
buf, err := backend.LoadAll(context.TODO(), nil, b, h) buf, err := backend.LoadAll(context.TODO(), nil, b, h)
if err != nil { if err != nil {
t.Fatalf("unable to read config with name %q: %+v", name, err) t.Fatalf("unable to read config with name %q: %+v", name, err)
@ -120,7 +120,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
} }
// remove the config // remove the config
remove(t, b, restic.Handle{Type: restic.ConfigFile}) remove(t, b, backend.Handle{Type: backend.ConfigFile})
} }
// TestLoad tests the backend's Load function. // TestLoad tests the backend's Load function.
@ -130,7 +130,7 @@ func (s *Suite[C]) TestLoad(t *testing.T) {
b := s.open(t) b := s.open(t)
defer s.close(t, b) defer s.close(t, b)
err := testLoad(b, restic.Handle{Type: restic.PackFile, Name: "foobar"}) err := testLoad(b, backend.Handle{Type: backend.PackFile, Name: "foobar"})
if err == nil { if err == nil {
t.Fatalf("Load() did not return an error for non-existing blob") t.Fatalf("Load() did not return an error for non-existing blob")
} }
@ -141,8 +141,8 @@ func (s *Suite[C]) TestLoad(t *testing.T) {
data := test.Random(23, length) data := test.Random(23, length)
id := restic.Hash(data) id := restic.Hash(data)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()} handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
err = b.Save(context.TODO(), handle, restic.NewByteReader(data, b.Hasher())) err = b.Save(context.TODO(), handle, backend.NewByteReader(data, b.Hasher()))
if err != nil { if err != nil {
t.Fatalf("Save() error: %+v", err) t.Fatalf("Save() error: %+v", err)
} }
@ -243,7 +243,7 @@ func (s *Suite[C]) TestList(t *testing.T) {
// Check that the backend is empty to start with // Check that the backend is empty to start with
var found []string var found []string
err := b.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
found = append(found, fi.Name) found = append(found, fi.Name)
return nil return nil
}) })
@ -259,8 +259,8 @@ func (s *Suite[C]) TestList(t *testing.T) {
for i := 0; i < numTestFiles; i++ { for i := 0; i < numTestFiles; i++ {
data := test.Random(rand.Int(), rand.Intn(100)+55) data := test.Random(rand.Int(), rand.Intn(100)+55)
id := restic.Hash(data) id := restic.Hash(data)
h := restic.Handle{Type: restic.PackFile, Name: id.String()} h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -284,7 +284,7 @@ func (s *Suite[C]) TestList(t *testing.T) {
s.SetListMaxItems(test.maxItems) s.SetListMaxItems(test.maxItems)
} }
err := b.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
id, err := restic.ParseID(fi.Name) id, err := restic.ParseID(fi.Name)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -320,9 +320,9 @@ func (s *Suite[C]) TestList(t *testing.T) {
} }
t.Logf("remove %d files", numTestFiles) t.Logf("remove %d files", numTestFiles)
handles := make([]restic.Handle, 0, len(list1)) handles := make([]backend.Handle, 0, len(list1))
for id := range list1 { for id := range list1 {
handles = append(handles, restic.Handle{Type: restic.PackFile, Name: id.String()}) handles = append(handles, backend.Handle{Type: backend.PackFile, Name: id.String()})
} }
err = s.delayedRemove(t, b, handles...) err = s.delayedRemove(t, b, handles...)
@ -340,13 +340,13 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
b := s.open(t) b := s.open(t)
defer s.close(t, b) defer s.close(t, b)
testFiles := make([]restic.Handle, 0, numTestFiles) testFiles := make([]backend.Handle, 0, numTestFiles)
for i := 0; i < numTestFiles; i++ { for i := 0; i < numTestFiles; i++ {
data := []byte(fmt.Sprintf("random test blob %v", i)) data := []byte(fmt.Sprintf("random test blob %v", i))
id := restic.Hash(data) id := restic.Hash(data)
h := restic.Handle{Type: restic.PackFile, Name: id.String()} h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -358,7 +358,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
cancel() cancel()
// pass in a cancelled context // pass in a cancelled context
err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
t.Errorf("got FileInfo %v for cancelled context", fi) t.Errorf("got FileInfo %v for cancelled context", fi)
return nil return nil
}) })
@ -373,7 +373,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
defer cancel() defer cancel()
i := 0 i := 0
err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
i++ i++
// cancel the context on the first file // cancel the context on the first file
if i == 1 { if i == 1 {
@ -396,7 +396,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
defer cancel() defer cancel()
i := 0 i := 0
err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
// cancel the context at the last file // cancel the context at the last file
i++ i++
if i == numTestFiles { if i == numTestFiles {
@ -423,7 +423,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
i := 0 i := 0
// pass in a context with a timeout // pass in a context with a timeout
err := b.List(ctxTimeout, restic.PackFile, func(fi restic.FileInfo) error { err := b.List(ctxTimeout, backend.PackFile, func(fi backend.FileInfo) error {
i++ i++
// wait until the context is cancelled // wait until the context is cancelled
@ -494,11 +494,11 @@ func (s *Suite[C]) TestSave(t *testing.T) {
data := test.Random(23, length) data := test.Random(23, length)
id = sha256.Sum256(data) id = sha256.Sum256(data)
h := restic.Handle{ h := backend.Handle{
Type: restic.PackFile, Type: backend.PackFile,
Name: id.String(), Name: id.String(),
} }
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
test.OK(t, err) test.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), nil, b, h) buf, err := backend.LoadAll(context.TODO(), nil, b, h)
@ -546,7 +546,7 @@ func (s *Suite[C]) TestSave(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
h := restic.Handle{Type: restic.PackFile, Name: id.String()} h := backend.Handle{Type: backend.PackFile, Name: id.String()}
// wrap the tempfile in an errorCloser, so we can detect if the backend // wrap the tempfile in an errorCloser, so we can detect if the backend
// closes the reader // closes the reader
@ -585,7 +585,7 @@ func (s *Suite[C]) TestSave(t *testing.T) {
} }
type incompleteByteReader struct { type incompleteByteReader struct {
restic.ByteReader backend.ByteReader
} }
func (r *incompleteByteReader) Length() int64 { func (r *incompleteByteReader) Length() int64 {
@ -609,8 +609,8 @@ func (s *Suite[C]) TestSaveError(t *testing.T) {
copy(id[:], data) copy(id[:], data)
// test that incomplete uploads fail // test that incomplete uploads fail
h := restic.Handle{Type: restic.PackFile, Name: id.String()} h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &incompleteByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())}) err := b.Save(context.TODO(), h, &incompleteByteReader{ByteReader: *backend.NewByteReader(data, b.Hasher())})
// try to delete possible leftovers // try to delete possible leftovers
_ = s.delayedRemove(t, b, h) _ = s.delayedRemove(t, b, h)
if err == nil { if err == nil {
@ -619,7 +619,7 @@ func (s *Suite[C]) TestSaveError(t *testing.T) {
} }
type wrongByteReader struct { type wrongByteReader struct {
restic.ByteReader backend.ByteReader
} }
func (b *wrongByteReader) Hash() []byte { func (b *wrongByteReader) Hash() []byte {
@ -648,8 +648,8 @@ func (s *Suite[C]) TestSaveWrongHash(t *testing.T) {
copy(id[:], data) copy(id[:], data)
// test that upload with hash mismatch fails // test that upload with hash mismatch fails
h := restic.Handle{Type: restic.PackFile, Name: id.String()} h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())}) err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *backend.NewByteReader(data, b.Hasher())})
exists, err2 := beTest(context.TODO(), b, h) exists, err2 := beTest(context.TODO(), b, h)
if err2 != nil { if err2 != nil {
t.Fatal(err2) t.Fatal(err2)
@ -674,23 +674,23 @@ var testStrings = []struct {
{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"}, {"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
} }
func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) restic.Handle { func store(t testing.TB, b backend.Backend, tpe backend.FileType, data []byte) backend.Handle {
id := restic.Hash(data) id := restic.Hash(data)
h := restic.Handle{Name: id.String(), Type: tpe} h := backend.Handle{Name: id.String(), Type: tpe}
err := b.Save(context.TODO(), h, restic.NewByteReader([]byte(data), b.Hasher())) err := b.Save(context.TODO(), h, backend.NewByteReader([]byte(data), b.Hasher()))
test.OK(t, err) test.OK(t, err)
return h return h
} }
// testLoad loads a blob (but discards its contents). // testLoad loads a blob (but discards its contents).
func testLoad(b restic.Backend, h restic.Handle) error { func testLoad(b backend.Backend, h backend.Handle) error {
return b.Load(context.TODO(), h, 0, 0, func(rd io.Reader) (ierr error) { return b.Load(context.TODO(), h, 0, 0, func(rd io.Reader) (ierr error) {
_, ierr = io.Copy(io.Discard, rd) _, ierr = io.Copy(io.Discard, rd)
return ierr return ierr
}) })
} }
func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error { func (s *Suite[C]) delayedRemove(t testing.TB, be backend.Backend, handles ...backend.Handle) error {
// Some backend (swift, I'm looking at you) may implement delayed // Some backend (swift, I'm looking at you) may implement delayed
// removal of data. Let's wait a bit if this happens. // removal of data. Let's wait a bit if this happens.
@ -734,11 +734,11 @@ func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...res
return nil return nil
} }
func delayedList(t testing.TB, b restic.Backend, tpe restic.FileType, max int, maxwait time.Duration) restic.IDs { func delayedList(t testing.TB, b backend.Backend, tpe backend.FileType, max int, maxwait time.Duration) restic.IDs {
list := restic.NewIDSet() list := restic.NewIDSet()
start := time.Now() start := time.Now()
for i := 0; i < max; i++ { for i := 0; i < max; i++ {
err := b.List(context.TODO(), tpe, func(fi restic.FileInfo) error { err := b.List(context.TODO(), tpe, func(fi backend.FileInfo) error {
id := restic.TestParseID(fi.Name) id := restic.TestParseID(fi.Name)
list.Insert(id) list.Insert(id)
return nil return nil
@ -763,9 +763,9 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
test.Assert(t, !b.IsNotExist(nil), "IsNotExist() recognized nil error") test.Assert(t, !b.IsNotExist(nil), "IsNotExist() recognized nil error")
for _, tpe := range []restic.FileType{ for _, tpe := range []backend.FileType{
restic.PackFile, restic.KeyFile, restic.LockFile, backend.PackFile, backend.KeyFile, backend.LockFile,
restic.SnapshotFile, restic.IndexFile, backend.SnapshotFile, backend.IndexFile,
} { } {
// detect non-existing files // detect non-existing files
for _, ts := range testStrings { for _, ts := range testStrings {
@ -773,7 +773,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
test.OK(t, err) test.OK(t, err)
// test if blob is already in repository // test if blob is already in repository
h := restic.Handle{Type: tpe, Name: id.String()} h := backend.Handle{Type: tpe, Name: id.String()}
ret, err := beTest(context.TODO(), b, h) ret, err := beTest(context.TODO(), b, h)
test.OK(t, err) test.OK(t, err)
test.Assert(t, !ret, "blob was found to exist before creating") test.Assert(t, !ret, "blob was found to exist before creating")
@ -799,7 +799,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
store(t, b, tpe, []byte(ts.data)) store(t, b, tpe, []byte(ts.data))
// test Load() // test Load()
h := restic.Handle{Type: tpe, Name: ts.id} h := backend.Handle{Type: tpe, Name: ts.id}
buf, err := backend.LoadAll(context.TODO(), nil, b, h) buf, err := backend.LoadAll(context.TODO(), nil, b, h)
test.OK(t, err) test.OK(t, err)
test.Equals(t, ts.data, string(buf)) test.Equals(t, ts.data, string(buf))
@ -823,7 +823,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
// test adding the first file again // test adding the first file again
ts := testStrings[0] ts := testStrings[0]
h := restic.Handle{Type: tpe, Name: ts.id} h := backend.Handle{Type: tpe, Name: ts.id}
// remove and recreate // remove and recreate
err := s.delayedRemove(t, b, h) err := s.delayedRemove(t, b, h)
@ -835,7 +835,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
test.Assert(t, !ok, "removed blob still present") test.Assert(t, !ok, "removed blob still present")
// create blob // create blob
err = b.Save(context.TODO(), h, restic.NewByteReader([]byte(ts.data), b.Hasher())) err = b.Save(context.TODO(), h, backend.NewByteReader([]byte(ts.data), b.Hasher()))
test.OK(t, err) test.OK(t, err)
// list items // list items
@ -859,12 +859,12 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list) t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list)
} }
var handles []restic.Handle var handles []backend.Handle
for _, ts := range testStrings { for _, ts := range testStrings {
id, err := restic.ParseID(ts.id) id, err := restic.ParseID(ts.id)
test.OK(t, err) test.OK(t, err)
h := restic.Handle{Type: tpe, Name: id.String()} h := backend.Handle{Type: tpe, Name: id.String()}
found, err := beTest(context.TODO(), b, h) found, err := beTest(context.TODO(), b, h)
test.OK(t, err) test.OK(t, err)

26
internal/backend/util/defaults.go
View File

@ -4,12 +4,12 @@ import (
"context" "context"
"io" "io"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/backend"
) )
// DefaultLoad implements Backend.Load using lower-level openReader func // DefaultLoad implements Backend.Load using lower-level openReader func
func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64, func DefaultLoad(ctx context.Context, h backend.Handle, length int, offset int64,
openReader func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error), openReader func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error),
fn func(rd io.Reader) error) error { fn func(rd io.Reader) error) error {
rd, err := openReader(ctx, h, length, offset) rd, err := openReader(ctx, h, length, offset)
@ -25,23 +25,23 @@ func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64,
} }
// DefaultDelete removes all restic keys in the bucket. It will not remove the bucket itself. // DefaultDelete removes all restic keys in the bucket. It will not remove the bucket itself.
func DefaultDelete(ctx context.Context, be restic.Backend) error { func DefaultDelete(ctx context.Context, be backend.Backend) error {
alltypes := []restic.FileType{ alltypes := []backend.FileType{
restic.PackFile, backend.PackFile,
restic.KeyFile, backend.KeyFile,
restic.LockFile, backend.LockFile,
restic.SnapshotFile, backend.SnapshotFile,
restic.IndexFile} backend.IndexFile}
for _, t := range alltypes { for _, t := range alltypes {
err := be.List(ctx, t, func(fi restic.FileInfo) error { err := be.List(ctx, t, func(fi backend.FileInfo) error {
return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) return be.Remove(ctx, backend.Handle{Type: t, Name: fi.Name})
}) })
if err != nil { if err != nil {
return nil return nil
} }
} }
err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) err := be.Remove(ctx, backend.Handle{Type: backend.ConfigFile})
if err != nil && be.IsNotExist(err) { if err != nil && be.IsNotExist(err) {
err = nil err = nil
} }

10
internal/backend/util/defaults_test.go
View File

@ -5,9 +5,9 @@ import (
"io" "io"
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
@ -26,11 +26,11 @@ func (rd *mockReader) Close() error {
func TestDefaultLoad(t *testing.T) { func TestDefaultLoad(t *testing.T) {
h := restic.Handle{Name: "id", Type: restic.PackFile} h := backend.Handle{Name: "id", Type: backend.PackFile}
rd := &mockReader{} rd := &mockReader{}
// happy case, assert correct parameters are passed around and content stream is closed // happy case, assert correct parameters are passed around and content stream is closed
err := util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) { err := util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) {
rtest.Equals(t, h, ih) rtest.Equals(t, h, ih)
rtest.Equals(t, int(10), length) rtest.Equals(t, int(10), length)
rtest.Equals(t, int64(11), offset) rtest.Equals(t, int64(11), offset)
@ -44,7 +44,7 @@ func TestDefaultLoad(t *testing.T) {
rtest.Equals(t, true, rd.closed) rtest.Equals(t, true, rd.closed)
// unhappy case, assert producer errors are handled correctly // unhappy case, assert producer errors are handled correctly
err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) { err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return nil, errors.Errorf("producer error") return nil, errors.Errorf("producer error")
}, func(ird io.Reader) error { }, func(ird io.Reader) error {
t.Fatalf("unexpected consumer invocation") t.Fatalf("unexpected consumer invocation")
@ -54,7 +54,7 @@ func TestDefaultLoad(t *testing.T) {
// unhappy case, assert consumer errors are handled correctly // unhappy case, assert consumer errors are handled correctly
rd = &mockReader{} rd = &mockReader{}
err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) { err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return rd, nil return rd, nil
}, func(ird io.Reader) error { }, func(ird io.Reader) error {
return errors.Errorf("consumer error") return errors.Errorf("consumer error")

17
internal/backend/utils.go
View File

@ -11,7 +11,6 @@ import (
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
) )
func verifyContentMatchesName(s string, data []byte) (bool, error) { func verifyContentMatchesName(s string, data []byte) (bool, error) {
@ -33,7 +32,7 @@ func verifyContentMatchesName(s string, data []byte) (bool, error) {
// LoadAll reads all data stored in the backend for the handle into the given // LoadAll reads all data stored in the backend for the handle into the given
// buffer, which is truncated. If the buffer is not large enough or nil, a new // buffer, which is truncated. If the buffer is not large enough or nil, a new
// one is allocated. // one is allocated.
func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle) ([]byte, error) { func LoadAll(ctx context.Context, buf []byte, be Backend, h Handle) ([]byte, error) {
retriedInvalidData := false retriedInvalidData := false
err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error { err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
// make sure this is idempotent, in case an error occurs this function may be called multiple times! // make sure this is idempotent, in case an error occurs this function may be called multiple times!
@ -47,7 +46,7 @@ func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle
// retry loading damaged data only once. If a file fails to download correctly // retry loading damaged data only once. If a file fails to download correctly
// the second time, then it is likely corrupted at the backend. Return the data // the second time, then it is likely corrupted at the backend. Return the data
// to the caller in that case to let it decide what to do with the data. // to the caller in that case to let it decide what to do with the data.
if !retriedInvalidData && h.Type != restic.ConfigFile { if !retriedInvalidData && h.Type != ConfigFile {
if matches, err := verifyContentMatchesName(h.Name, buf); err == nil && !matches { if matches, err := verifyContentMatchesName(h.Name, buf); err == nil && !matches {
debug.Log("retry loading broken blob %v", h) debug.Log("retry loading broken blob %v", h)
retriedInvalidData = true retriedInvalidData = true
@ -77,11 +76,11 @@ func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
} }
type memorizedLister struct { type memorizedLister struct {
fileInfos []restic.FileInfo fileInfos []FileInfo
tpe restic.FileType tpe FileType
} }
func (m *memorizedLister) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { func (m *memorizedLister) List(ctx context.Context, t FileType, fn func(FileInfo) error) error {
if t != m.tpe { if t != m.tpe {
return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t) return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t)
} }
@ -97,13 +96,13 @@ func (m *memorizedLister) List(ctx context.Context, t restic.FileType, fn func(r
return ctx.Err() return ctx.Err()
} }
func MemorizeList(ctx context.Context, be restic.Lister, t restic.FileType) (restic.Lister, error) { func MemorizeList(ctx context.Context, be Lister, t FileType) (Lister, error) {
if _, ok := be.(*memorizedLister); ok { if _, ok := be.(*memorizedLister); ok {
return be, nil return be, nil
} }
var fileInfos []restic.FileInfo var fileInfos []FileInfo
err := be.List(ctx, t, func(fi restic.FileInfo) error { err := be.List(ctx, t, func(fi FileInfo) error {
fileInfos = append(fileInfos, fi) fileInfos = append(fileInfos, fi)
return nil return nil
}) })

40
internal/backend/utils_test.go
View File

@ -26,11 +26,11 @@ func TestLoadAll(t *testing.T) {
data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB) data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB)
id := restic.Hash(data) id := restic.Hash(data)
h := restic.Handle{Name: id.String(), Type: restic.PackFile} h := backend.Handle{Name: id.String(), Type: backend.PackFile}
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
rtest.OK(t, err) rtest.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), buf, b, restic.Handle{Type: restic.PackFile, Name: id.String()}) buf, err := backend.LoadAll(context.TODO(), buf, b, backend.Handle{Type: backend.PackFile, Name: id.String()})
rtest.OK(t, err) rtest.OK(t, err)
if len(buf) != len(data) { if len(buf) != len(data) {
@ -45,10 +45,10 @@ func TestLoadAll(t *testing.T) {
} }
} }
func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle { func save(t testing.TB, be backend.Backend, buf []byte) backend.Handle {
id := restic.Hash(buf) id := restic.Hash(buf)
h := restic.Handle{Name: id.String(), Type: restic.PackFile} h := backend.Handle{Name: id.String(), Type: backend.PackFile}
err := be.Save(context.TODO(), h, restic.NewByteReader(buf, be.Hasher())) err := be.Save(context.TODO(), h, backend.NewByteReader(buf, be.Hasher()))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -56,10 +56,10 @@ func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle {
} }
type quickRetryBackend struct { type quickRetryBackend struct {
restic.Backend backend.Backend
} }
func (be *quickRetryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { func (be *quickRetryBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
err := be.Backend.Load(ctx, h, length, offset, fn) err := be.Backend.Load(ctx, h, length, offset, fn)
if err != nil { if err != nil {
// retry // retry
@ -76,19 +76,19 @@ func TestLoadAllBroken(t *testing.T) {
// damage buffer // damage buffer
data[0] ^= 0xff data[0] ^= 0xff
b.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { b.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(data)), nil return io.NopCloser(bytes.NewReader(data)), nil
} }
// must fail on first try // must fail on first try
_, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.PackFile, Name: id.String()}) _, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.PackFile, Name: id.String()})
if err == nil { if err == nil {
t.Fatalf("missing expected error") t.Fatalf("missing expected error")
} }
// must return the broken data after a retry // must return the broken data after a retry
be := &quickRetryBackend{Backend: b} be := &quickRetryBackend{Backend: b}
buf, err := backend.LoadAll(context.TODO(), nil, be, restic.Handle{Type: restic.PackFile, Name: id.String()}) buf, err := backend.LoadAll(context.TODO(), nil, be, backend.Handle{Type: backend.PackFile, Name: id.String()})
rtest.OK(t, err) rtest.OK(t, err)
if !bytes.Equal(buf, data) { if !bytes.Equal(buf, data) {
@ -104,7 +104,7 @@ func TestLoadAllAppend(t *testing.T) {
h2 := save(t, b, randomData) h2 := save(t, b, randomData)
var tests = []struct { var tests = []struct {
handle restic.Handle handle backend.Handle
buf []byte buf []byte
want []byte want []byte
}{ }{
@ -152,11 +152,11 @@ func TestLoadAllAppend(t *testing.T) {
func TestMemoizeList(t *testing.T) { func TestMemoizeList(t *testing.T) {
// setup backend to serve as data source for memoized list // setup backend to serve as data source for memoized list
be := mock.NewBackend() be := mock.NewBackend()
files := []restic.FileInfo{ files := []backend.FileInfo{
{Size: 42, Name: restic.NewRandomID().String()}, {Size: 42, Name: restic.NewRandomID().String()},
{Size: 45, Name: restic.NewRandomID().String()}, {Size: 45, Name: restic.NewRandomID().String()},
} }
be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { be.ListFn = func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
for _, fi := range files { for _, fi := range files {
if err := fn(fi); err != nil { if err := fn(fi); err != nil {
return err return err
@ -165,17 +165,17 @@ func TestMemoizeList(t *testing.T) {
return nil return nil
} }
mem, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile) mem, err := backend.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.OK(t, err) rtest.OK(t, err)
err = mem.List(context.TODO(), restic.IndexFile, func(fi restic.FileInfo) error { err = mem.List(context.TODO(), backend.IndexFile, func(fi backend.FileInfo) error {
t.Fatal("file type mismatch") t.Fatal("file type mismatch")
return nil // the memoized lister must return an error by itself return nil // the memoized lister must return an error by itself
}) })
rtest.Assert(t, err != nil, "missing error on file typ mismatch") rtest.Assert(t, err != nil, "missing error on file typ mismatch")
var memFiles []restic.FileInfo var memFiles []backend.FileInfo
err = mem.List(context.TODO(), restic.SnapshotFile, func(fi restic.FileInfo) error { err = mem.List(context.TODO(), backend.SnapshotFile, func(fi backend.FileInfo) error {
memFiles = append(memFiles, fi) memFiles = append(memFiles, fi)
return nil return nil
}) })
@ -186,9 +186,9 @@ func TestMemoizeList(t *testing.T) {
func TestMemoizeListError(t *testing.T) { func TestMemoizeListError(t *testing.T) {
// setup backend to serve as data source for memoized list // setup backend to serve as data source for memoized list
be := mock.NewBackend() be := mock.NewBackend()
be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { be.ListFn = func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
return fmt.Errorf("list error") return fmt.Errorf("list error")
} }
_, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile) _, err := backend.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.Assert(t, err != nil, "missing error on list error") rtest.Assert(t, err != nil, "missing error on list error")
} }

34
internal/cache/backend.go vendored
View File

@ -5,35 +5,35 @@ import (
"io" "io"
"sync" "sync"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
) )
// Backend wraps a restic.Backend and adds a cache. // Backend wraps a restic.Backend and adds a cache.
type Backend struct { type Backend struct {
restic.Backend backend.Backend
*Cache *Cache
// inProgress contains the handle for all files that are currently // inProgress contains the handle for all files that are currently
// downloaded. The channel in the value is closed as soon as the download // downloaded. The channel in the value is closed as soon as the download
// is finished. // is finished.
inProgressMutex sync.Mutex inProgressMutex sync.Mutex
inProgress map[restic.Handle]chan struct{} inProgress map[backend.Handle]chan struct{}
} }
// ensure Backend implements restic.Backend // ensure Backend implements backend.Backend
var _ restic.Backend = &Backend{} var _ backend.Backend = &Backend{}
func newBackend(be restic.Backend, c *Cache) *Backend { func newBackend(be backend.Backend, c *Cache) *Backend {
return &Backend{ return &Backend{
Backend: be, Backend: be,
Cache: c, Cache: c,
inProgress: make(map[restic.Handle]chan struct{}), inProgress: make(map[backend.Handle]chan struct{}),
} }
} }
// Remove deletes a file from the backend and the cache if it has been cached. // Remove deletes a file from the backend and the cache if it has been cached.
func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { func (b *Backend) Remove(ctx context.Context, h backend.Handle) error {
debug.Log("cache Remove(%v)", h) debug.Log("cache Remove(%v)", h)
err := b.Backend.Remove(ctx, h) err := b.Backend.Remove(ctx, h)
if err != nil { if err != nil {
@ -43,18 +43,18 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
return b.Cache.remove(h) return b.Cache.remove(h)
} }
func autoCacheTypes(h restic.Handle) bool { func autoCacheTypes(h backend.Handle) bool {
switch h.Type { switch h.Type {
case restic.IndexFile, restic.SnapshotFile: case backend.IndexFile, backend.SnapshotFile:
return true return true
case restic.PackFile: case backend.PackFile:
return h.IsMetadata return h.IsMetadata
} }
return false return false
} }
// Save stores a new file in the backend and the cache. // Save stores a new file in the backend and the cache.
func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if !autoCacheTypes(h) { if !autoCacheTypes(h) {
return b.Backend.Save(ctx, h, rd) return b.Backend.Save(ctx, h, rd)
} }
@ -89,7 +89,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
return nil return nil
} }
func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error { func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error {
finish := make(chan struct{}) finish := make(chan struct{})
b.inProgressMutex.Lock() b.inProgressMutex.Lock()
@ -133,7 +133,7 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
} }
// loadFromCache will try to load the file from the cache. // loadFromCache will try to load the file from the cache.
func (b *Backend) loadFromCache(h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) { func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
rd, err := b.Cache.load(h, length, offset) rd, err := b.Cache.load(h, length, offset)
if err != nil { if err != nil {
return false, err return false, err
@ -148,7 +148,7 @@ func (b *Backend) loadFromCache(h restic.Handle, length int, offset int64, consu
} }
// Load loads a file from the cache or the backend. // Load loads a file from the cache or the backend.
func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
b.inProgressMutex.Lock() b.inProgressMutex.Lock()
waitForFinish, inProgress := b.inProgress[h] waitForFinish, inProgress := b.inProgress[h]
b.inProgressMutex.Unlock() b.inProgressMutex.Unlock()
@ -194,7 +194,7 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
// Stat tests whether the backend has a file. If it does not exist but still // Stat tests whether the backend has a file. If it does not exist but still
// exists in the cache, it is removed from the cache. // exists in the cache, it is removed from the cache.
func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
debug.Log("cache Stat(%v)", h) debug.Log("cache Stat(%v)", h)
fi, err := b.Backend.Stat(ctx, h) fi, err := b.Backend.Stat(ctx, h)
@ -215,6 +215,6 @@ func (b *Backend) IsNotExist(err error) bool {
return b.Backend.IsNotExist(err) return b.Backend.IsNotExist(err)
} }
func (b *Backend) Unwrap() restic.Backend { func (b *Backend) Unwrap() backend.Backend {
return b.Backend return b.Backend
} }

20
internal/cache/backend_test.go vendored
View File

@ -16,7 +16,7 @@ import (
"github.com/restic/restic/internal/test" "github.com/restic/restic/internal/test"
) )
func loadAndCompare(t testing.TB, be restic.Backend, h restic.Handle, data []byte) { func loadAndCompare(t testing.TB, be backend.Backend, h backend.Handle, data []byte) {
buf, err := backend.LoadAll(context.TODO(), nil, be, h) buf, err := backend.LoadAll(context.TODO(), nil, be, h)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -31,25 +31,25 @@ func loadAndCompare(t testing.TB, be restic.Backend, h restic.Handle, data []byt
} }
} }
func save(t testing.TB, be restic.Backend, h restic.Handle, data []byte) { func save(t testing.TB, be backend.Backend, h backend.Handle, data []byte) {
err := be.Save(context.TODO(), h, restic.NewByteReader(data, be.Hasher())) err := be.Save(context.TODO(), h, backend.NewByteReader(data, be.Hasher()))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
} }
func remove(t testing.TB, be restic.Backend, h restic.Handle) { func remove(t testing.TB, be backend.Backend, h backend.Handle) {
err := be.Remove(context.TODO(), h) err := be.Remove(context.TODO(), h)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
} }
func randomData(n int) (restic.Handle, []byte) { func randomData(n int) (backend.Handle, []byte) {
data := test.Random(rand.Int(), n) data := test.Random(rand.Int(), n)
id := restic.Hash(data) id := restic.Hash(data)
h := restic.Handle{ h := backend.Handle{
Type: restic.IndexFile, Type: backend.IndexFile,
Name: id.String(), Name: id.String(),
} }
return h, data return h, data
@ -114,11 +114,11 @@ func TestBackend(t *testing.T) {
} }
type loadErrorBackend struct { type loadErrorBackend struct {
restic.Backend backend.Backend
loadError error loadError error
} }
func (be loadErrorBackend) Load(_ context.Context, _ restic.Handle, _ int, _ int64, _ func(rd io.Reader) error) error { func (be loadErrorBackend) Load(_ context.Context, _ backend.Handle, _ int, _ int64, _ func(rd io.Reader) error) error {
time.Sleep(10 * time.Millisecond) time.Sleep(10 * time.Millisecond)
return be.loadError return be.loadError
} }
@ -137,7 +137,7 @@ func TestErrorBackend(t *testing.T) {
loadError: testErr, loadError: testErr,
} }
loadTest := func(wg *sync.WaitGroup, be restic.Backend) { loadTest := func(wg *sync.WaitGroup, be backend.Backend) {
defer wg.Done() defer wg.Done()
buf, err := backend.LoadAll(context.TODO(), nil, be, h) buf, err := backend.LoadAll(context.TODO(), nil, be, h)

3
internal/cache/cache.go vendored
View File

@ -9,6 +9,7 @@ import (
"time" "time"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -234,7 +235,7 @@ func IsOld(t time.Time, maxAge time.Duration) bool {
} }
// Wrap returns a backend with a cache. // Wrap returns a backend with a cache.
func (c *Cache) Wrap(be restic.Backend) restic.Backend { func (c *Cache) Wrap(be backend.Backend) backend.Backend {
return newBackend(be, c) return newBackend(be, c)
} }

14
internal/cache/file.go vendored
View File

@ -14,7 +14,7 @@ import (
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
) )
func (c *Cache) filename(h restic.Handle) string { func (c *Cache) filename(h backend.Handle) string {
if len(h.Name) < 2 { if len(h.Name) < 2 {
panic("Name is empty or too short") panic("Name is empty or too short")
} }
@ -22,7 +22,7 @@ func (c *Cache) filename(h restic.Handle) string {
return filepath.Join(c.path, cacheLayoutPaths[h.Type], subdir, h.Name) return filepath.Join(c.path, cacheLayoutPaths[h.Type], subdir, h.Name)
} }
func (c *Cache) canBeCached(t restic.FileType) bool { func (c *Cache) canBeCached(t backend.FileType) bool {
if c == nil { if c == nil {
return false return false
} }
@ -34,7 +34,7 @@ func (c *Cache) canBeCached(t restic.FileType) bool {
// Load returns a reader that yields the contents of the file with the // Load returns a reader that yields the contents of the file with the
// given handle. rd must be closed after use. If an error is returned, the // given handle. rd must be closed after use. If an error is returned, the
// ReadCloser is nil. // ReadCloser is nil.
func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { func (c *Cache) load(h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
debug.Log("Load(%v, %v, %v) from cache", h, length, offset) debug.Log("Load(%v, %v, %v) from cache", h, length, offset)
if !c.canBeCached(h.Type) { if !c.canBeCached(h.Type) {
return nil, errors.New("cannot be cached") return nil, errors.New("cannot be cached")
@ -78,7 +78,7 @@ func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser,
} }
// Save saves a file in the cache. // Save saves a file in the cache.
func (c *Cache) Save(h restic.Handle, rd io.Reader) error { func (c *Cache) Save(h backend.Handle, rd io.Reader) error {
debug.Log("Save to cache: %v", h) debug.Log("Save to cache: %v", h)
if rd == nil { if rd == nil {
return errors.New("Save() called with nil reader") return errors.New("Save() called with nil reader")
@ -139,7 +139,7 @@ func (c *Cache) Save(h restic.Handle, rd io.Reader) error {
} }
// Remove deletes a file. When the file is not cache, no error is returned. // Remove deletes a file. When the file is not cache, no error is returned.
func (c *Cache) remove(h restic.Handle) error { func (c *Cache) remove(h backend.Handle) error {
if !c.Has(h) { if !c.Has(h) {
return nil return nil
} }
@ -165,7 +165,7 @@ func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
continue continue
} }
if err = fs.Remove(c.filename(restic.Handle{Type: t, Name: id.String()})); err != nil { if err = fs.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil {
return err return err
} }
} }
@ -207,7 +207,7 @@ func (c *Cache) list(t restic.FileType) (restic.IDSet, error) {
} }
// Has returns true if the file is cached. // Has returns true if the file is cached.
func (c *Cache) Has(h restic.Handle) bool { func (c *Cache) Has(h backend.Handle) bool {
if !c.canBeCached(h.Type) { if !c.canBeCached(h.Type) {
return false return false
} }

15
internal/cache/file_test.go vendored
View File

@ -10,6 +10,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -18,12 +19,12 @@ import (
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
func generateRandomFiles(t testing.TB, tpe restic.FileType, c *Cache) restic.IDSet { func generateRandomFiles(t testing.TB, tpe backend.FileType, c *Cache) restic.IDSet {
ids := restic.NewIDSet() ids := restic.NewIDSet()
for i := 0; i < rand.Intn(15)+10; i++ { for i := 0; i < rand.Intn(15)+10; i++ {
buf := test.Random(rand.Int(), 1<<19) buf := test.Random(rand.Int(), 1<<19)
id := restic.Hash(buf) id := restic.Hash(buf)
h := restic.Handle{Type: tpe, Name: id.String()} h := backend.Handle{Type: tpe, Name: id.String()}
if c.Has(h) { if c.Has(h) {
t.Errorf("index %v present before save", id) t.Errorf("index %v present before save", id)
@ -46,7 +47,7 @@ func randomID(s restic.IDSet) restic.ID {
panic("set is empty") panic("set is empty")
} }
func load(t testing.TB, c *Cache, h restic.Handle) []byte { func load(t testing.TB, c *Cache, h backend.Handle) []byte {
rd, err := c.load(h, 0, 0) rd, err := c.load(h, 0, 0)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -101,7 +102,7 @@ func TestFiles(t *testing.T) {
ids := generateRandomFiles(t, tpe, c) ids := generateRandomFiles(t, tpe, c)
id := randomID(ids) id := randomID(ids)
h := restic.Handle{Type: tpe, Name: id.String()} h := backend.Handle{Type: tpe, Name: id.String()}
id2 := restic.Hash(load(t, c, h)) id2 := restic.Hash(load(t, c, h))
if !id.Equal(id2) { if !id.Equal(id2) {
@ -146,7 +147,7 @@ func TestFileLoad(t *testing.T) {
data := test.Random(rand.Int(), 5234142) data := test.Random(rand.Int(), 5234142)
id := restic.ID{} id := restic.ID{}
copy(id[:], data) copy(id[:], data)
h := restic.Handle{ h := backend.Handle{
Type: restic.PackFile, Type: restic.PackFile,
Name: id.String(), Name: id.String(),
} }
@ -230,7 +231,7 @@ func TestFileSaveConcurrent(t *testing.T) {
) )
rand.Read(id[:]) rand.Read(id[:])
h := restic.Handle{ h := backend.Handle{
Type: restic.PackFile, Type: restic.PackFile,
Name: id.String(), Name: id.String(),
} }
@ -275,7 +276,7 @@ func TestFileSaveAfterDamage(t *testing.T) {
// save a few bytes of data in the cache // save a few bytes of data in the cache
data := test.Random(123456789, 42) data := test.Random(123456789, 42)
id := restic.Hash(data) id := restic.Hash(data)
h := restic.Handle{ h := backend.Handle{
Type: restic.PackFile, Type: restic.PackFile,
Name: id.String(), Name: id.String(),
} }

10
internal/checker/checker.go
View File

@ -39,7 +39,7 @@ type Checker struct {
trackUnused bool trackUnused bool
masterIndex *index.MasterIndex masterIndex *index.MasterIndex
snapshots restic.Lister snapshots backend.Lister
repo restic.Repository repo restic.Repository
} }
@ -134,7 +134,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
if p != nil { if p != nil {
var numIndexFiles uint64 var numIndexFiles uint64
err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error { err := indexList.List(ctx, restic.IndexFile, func(fi backend.FileInfo) error {
_, err := restic.ParseID(fi.Name) _, err := restic.ParseID(fi.Name)
if err != nil { if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name) debug.Log("unable to parse %v as an ID", fi.Name)
@ -245,7 +245,7 @@ func IsOrphanedPack(err error) bool {
return errors.As(err, &e) && e.Orphaned return errors.As(err, &e) && e.Orphaned
} }
func isS3Legacy(b restic.Backend) bool { func isS3Legacy(b backend.Backend) bool {
// unwrap cache // unwrap cache
if be, ok := b.(*cache.Backend); ok { if be, ok := b.(*cache.Backend); ok {
b = be.Backend b = be.Backend
@ -367,7 +367,7 @@ func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan restic.TreeI
} }
} }
func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.Repository) (ids restic.IDs, errs []error) { func loadSnapshotTreeIDs(ctx context.Context, lister backend.Lister, repo restic.Repository) (ids restic.IDs, errs []error) {
err := restic.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { err := restic.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil { if err != nil {
errs = append(errs, err) errs = append(errs, err)
@ -563,7 +563,7 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r
// calculate hash on-the-fly while reading the pack and capture pack header // calculate hash on-the-fly while reading the pack and capture pack header
var hash restic.ID var hash restic.ID
var hdrBuf []byte var hdrBuf []byte
hashingLoader := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { hashingLoader := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return r.Backend().Load(ctx, h, int(size), 0, func(rd io.Reader) error { return r.Backend().Load(ctx, h, int(size), 0, func(rd io.Reader) error {
hrd := hashing.NewReader(rd, sha256.New()) hrd := hashing.NewReader(rd, sha256.New())
bufRd.Reset(hrd) bufRd.Reset(hrd)

17
internal/checker/checker_test.go
View File

@ -13,6 +13,7 @@ import (
"time" "time"
"github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/hashing" "github.com/restic/restic/internal/hashing"
@ -96,7 +97,7 @@ func TestMissingPack(t *testing.T) {
repo := repository.TestOpenLocal(t, repodir) repo := repository.TestOpenLocal(t, repodir)
packHandle := restic.Handle{ packHandle := backend.Handle{
Type: restic.PackFile, Type: restic.PackFile,
Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6", Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6",
} }
@ -129,7 +130,7 @@ func TestUnreferencedPack(t *testing.T) {
// index 3f1a only references pack 60e0 // index 3f1a only references pack 60e0
packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e" packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
indexHandle := restic.Handle{ indexHandle := backend.Handle{
Type: restic.IndexFile, Type: restic.IndexFile,
Name: "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44", Name: "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44",
} }
@ -160,7 +161,7 @@ func TestUnreferencedBlobs(t *testing.T) {
repo := repository.TestOpenLocal(t, repodir) repo := repository.TestOpenLocal(t, repodir)
snapshotHandle := restic.Handle{ snapshotHandle := backend.Handle{
Type: restic.SnapshotFile, Type: restic.SnapshotFile,
Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02", Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02",
} }
@ -202,7 +203,7 @@ func TestModifiedIndex(t *testing.T) {
done := make(chan struct{}) done := make(chan struct{})
defer close(done) defer close(done)
h := restic.Handle{ h := backend.Handle{
Type: restic.IndexFile, Type: restic.IndexFile,
Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd", Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
} }
@ -238,7 +239,7 @@ func TestModifiedIndex(t *testing.T) {
// save the index again with a modified name so that the hash doesn't match // save the index again with a modified name so that the hash doesn't match
// the content any more // the content any more
h2 := restic.Handle{ h2 := backend.Handle{
Type: restic.IndexFile, Type: restic.IndexFile,
Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd", Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
} }
@ -247,7 +248,7 @@ func TestModifiedIndex(t *testing.T) {
if hw != nil { if hw != nil {
hash = hw.Sum(nil) hash = hw.Sum(nil)
} }
rd, err := restic.NewFileReader(tmpfile, hash) rd, err := backend.NewFileReader(tmpfile, hash)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -304,11 +305,11 @@ func TestDuplicatePacksInIndex(t *testing.T) {
// errorBackend randomly modifies data after reading. // errorBackend randomly modifies data after reading.
type errorBackend struct { type errorBackend struct {
restic.Backend backend.Backend
ProduceErrors bool ProduceErrors bool
} }
func (b errorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { func (b errorBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
if b.ProduceErrors { if b.ProduceErrors {
return consumer(errorReadCloser{rd}) return consumer(errorReadCloser{rd})

3
internal/index/index_parallel.go
View File

@ -5,13 +5,14 @@ import (
"runtime" "runtime"
"sync" "sync"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
) )
// ForAllIndexes loads all index files in parallel and calls the given callback. // ForAllIndexes loads all index files in parallel and calls the given callback.
// It is guaranteed that the function is not run concurrently. If the callback // It is guaranteed that the function is not run concurrently. If the callback
// returns an error, this function is cancelled and also returns that error. // returns an error, this function is cancelled and also returns that error.
func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Repository, func ForAllIndexes(ctx context.Context, lister backend.Lister, repo restic.Repository,
fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error { fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error {
// decoding an index can take quite some time such that this can be both CPU- or IO-bound // decoding an index can take quite some time such that this can be both CPU- or IO-bound

3
internal/index/master_index_test.go
View File

@ -8,6 +8,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/index" "github.com/restic/restic/internal/index"
@ -369,7 +370,7 @@ func testIndexSave(t *testing.T, version uint) {
for id := range obsoletes { for id := range obsoletes {
t.Logf("remove index %v", id.Str()) t.Logf("remove index %v", id.Str())
h := restic.Handle{Type: restic.IndexFile, Name: id.String()} h := backend.Handle{Type: restic.IndexFile, Name: id.String()}
err = repo.Backend().Remove(context.TODO(), h) err = repo.Backend().Remove(context.TODO(), h)
if err != nil { if err != nil {
t.Errorf("error removing index %v: %v", id, err) t.Errorf("error removing index %v: %v", id, err)

9
internal/migrations/s3_layout.go
View File

@ -6,6 +6,7 @@ import (
"os" "os"
"path" "path"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
@ -23,7 +24,7 @@ type S3Layout struct{}
// Check tests whether the migration can be applied. // Check tests whether the migration can be applied.
func (m *S3Layout) Check(_ context.Context, repo restic.Repository) (bool, string, error) { func (m *S3Layout) Check(_ context.Context, repo restic.Repository) (bool, string, error) {
be := restic.AsBackend[*s3.Backend](repo.Backend()) be := backend.AsBackend[*s3.Backend](repo.Backend())
if be == nil { if be == nil {
debug.Log("backend is not s3") debug.Log("backend is not s3")
return false, "backend is not s3", nil return false, "backend is not s3", nil
@ -63,8 +64,8 @@ func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layou
fmt.Fprintf(os.Stderr, "renaming file returned error: %v\n", err) fmt.Fprintf(os.Stderr, "renaming file returned error: %v\n", err)
} }
return be.List(ctx, t, func(fi restic.FileInfo) error { return be.List(ctx, t, func(fi backend.FileInfo) error {
h := restic.Handle{Type: t, Name: fi.Name} h := backend.Handle{Type: t, Name: fi.Name}
debug.Log("move %v", h) debug.Log("move %v", h)
return retry(maxErrors, printErr, func() error { return retry(maxErrors, printErr, func() error {
@ -75,7 +76,7 @@ func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layou
// Apply runs the migration. // Apply runs the migration.
func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error { func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error {
be := restic.AsBackend[*s3.Backend](repo.Backend()) be := backend.AsBackend[*s3.Backend](repo.Backend())
if be == nil { if be == nil {
debug.Log("backend is not s3") debug.Log("backend is not s3")
return errors.New("backend is not s3") return errors.New("backend is not s3")

7
internal/migrations/upgrade_repo_v2.go
View File

@ -7,6 +7,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
) )
@ -57,7 +58,7 @@ func (*UpgradeRepoV2) RepoCheck() bool {
return true return true
} }
func (*UpgradeRepoV2) upgrade(ctx context.Context, repo restic.Repository) error { func (*UpgradeRepoV2) upgrade(ctx context.Context, repo restic.Repository) error {
h := restic.Handle{Type: restic.ConfigFile} h := backend.Handle{Type: backend.ConfigFile}
if !repo.Backend().HasAtomicReplace() { if !repo.Backend().HasAtomicReplace() {
// remove the original file for backends which do not support atomic overwriting // remove the original file for backends which do not support atomic overwriting
@ -85,7 +86,7 @@ func (m *UpgradeRepoV2) Apply(ctx context.Context, repo restic.Repository) error
return fmt.Errorf("create temp dir failed: %w", err) return fmt.Errorf("create temp dir failed: %w", err)
} }
h := restic.Handle{Type: restic.ConfigFile} h := backend.Handle{Type: restic.ConfigFile}
// read raw config file and save it to a temp dir, just in case // read raw config file and save it to a temp dir, just in case
var rawConfigFile []byte var rawConfigFile []byte
@ -115,7 +116,7 @@ func (m *UpgradeRepoV2) Apply(ctx context.Context, repo restic.Repository) error
// try contingency methods, reupload the original file // try contingency methods, reupload the original file
_ = repo.Backend().Remove(ctx, h) _ = repo.Backend().Remove(ctx, h)
err = repo.Backend().Save(ctx, h, restic.NewByteReader(rawConfigFile, nil)) err = repo.Backend().Save(ctx, h, backend.NewByteReader(rawConfigFile, nil))
if err != nil { if err != nil {
repoError.ReuploadOldConfigError = err repoError.ReuploadOldConfigError = err
} }

8
internal/migrations/upgrade_repo_v2_test.go
View File

@ -7,9 +7,9 @@ import (
"sync" "sync"
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test" "github.com/restic/restic/internal/test"
) )
@ -37,14 +37,14 @@ func TestUpgradeRepoV2(t *testing.T) {
} }
type failBackend struct { type failBackend struct {
restic.Backend backend.Backend
mu sync.Mutex mu sync.Mutex
ConfigFileSavesUntilError uint ConfigFileSavesUntilError uint
} }
func (be *failBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { func (be *failBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if h.Type != restic.ConfigFile { if h.Type != backend.ConfigFile {
return be.Backend.Save(ctx, h, rd) return be.Backend.Save(ctx, h, rd)
} }

8
internal/pack/pack_test.go
View File

@ -127,8 +127,8 @@ func TestUnpackReadSeeker(t *testing.T) {
b := mem.New() b := mem.New()
id := restic.Hash(packData) id := restic.Hash(packData)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()} handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher()))) rtest.OK(t, b.Save(context.TODO(), handle, backend.NewByteReader(packData, b.Hasher())))
verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize) verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize)
} }
@ -140,7 +140,7 @@ func TestShortPack(t *testing.T) {
b := mem.New() b := mem.New()
id := restic.Hash(packData) id := restic.Hash(packData)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()} handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher()))) rtest.OK(t, b.Save(context.TODO(), handle, backend.NewByteReader(packData, b.Hasher())))
verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize) verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize)
} }

6
internal/repository/key.go
View File

@ -178,7 +178,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int,
// LoadKey loads a key from the backend. // LoadKey loads a key from the backend.
func LoadKey(ctx context.Context, s *Repository, id restic.ID) (k *Key, err error) { func LoadKey(ctx context.Context, s *Repository, id restic.ID) (k *Key, err error) {
h := restic.Handle{Type: restic.KeyFile, Name: id.String()} h := backend.Handle{Type: restic.KeyFile, Name: id.String()}
data, err := backend.LoadAll(ctx, nil, s.be, h) data, err := backend.LoadAll(ctx, nil, s.be, h)
if err != nil { if err != nil {
return nil, err return nil, err
@ -270,12 +270,12 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str
id := restic.Hash(buf) id := restic.Hash(buf)
// store in repository and return // store in repository and return
h := restic.Handle{ h := backend.Handle{
Type: restic.KeyFile, Type: restic.KeyFile,
Name: id.String(), Name: id.String(),
} }
err = s.be.Save(ctx, h, restic.NewByteReader(buf, s.be.Hasher())) err = s.be.Save(ctx, h, backend.NewByteReader(buf, s.be.Hasher()))
if err != nil { if err != nil {
return nil, err return nil, err
} }

7
internal/repository/packer_manager.go
View File

@ -8,6 +8,7 @@ import (
"runtime" "runtime"
"sync" "sync"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/hashing" "github.com/restic/restic/internal/hashing"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -145,7 +146,7 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe
// calculate sha256 hash in a second pass // calculate sha256 hash in a second pass
var rd io.Reader var rd io.Reader
rd, err = restic.NewFileReader(p.tmpfile, nil) rd, err = backend.NewFileReader(p.tmpfile, nil)
if err != nil { if err != nil {
return err return err
} }
@ -163,12 +164,12 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe
} }
id := restic.IDFromHash(hr.Sum(nil)) id := restic.IDFromHash(hr.Sum(nil))
h := restic.Handle{Type: restic.PackFile, Name: id.String(), IsMetadata: t.IsMetadata()} h := backend.Handle{Type: backend.PackFile, Name: id.String(), IsMetadata: t.IsMetadata()}
var beHash []byte var beHash []byte
if beHr != nil { if beHr != nil {
beHash = beHr.Sum(nil) beHash = beHr.Sum(nil)
} }
rrd, err := restic.NewFileReader(p.tmpfile, beHash) rrd, err := backend.NewFileReader(p.tmpfile, beHash)
if err != nil { if err != nil {
return err return err
} }

5
internal/repository/repack_test.go
View File

@ -6,6 +6,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/index" "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -157,7 +158,7 @@ func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs rest
} }
for id := range repackedBlobs { for id := range repackedBlobs {
err = repo.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}) err = repo.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -191,7 +192,7 @@ func rebuildIndex(t *testing.T, repo restic.Repository) {
} }
err = repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error { err = repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error {
h := restic.Handle{ h := backend.Handle{
Type: restic.IndexFile, Type: restic.IndexFile,
Name: id.String(), Name: id.String(),
} }

30
internal/repository/repository.go
View File

@ -36,7 +36,7 @@ const MaxPackSize = 128 * 1024 * 1024
// Repository is used to access a repository in a backend. // Repository is used to access a repository in a backend.
type Repository struct { type Repository struct {
be restic.Backend be backend.Backend
cfg restic.Config cfg restic.Config
key *crypto.Key key *crypto.Key
keyID restic.ID keyID restic.ID
@ -109,7 +109,7 @@ func (c *CompressionMode) Type() string {
} }
// New returns a new repository with backend be. // New returns a new repository with backend be.
func New(be restic.Backend, opts Options) (*Repository, error) { func New(be backend.Backend, opts Options) (*Repository, error) {
if opts.Compression == CompressionInvalid { if opts.Compression == CompressionInvalid {
return nil, errors.New("invalid compression mode") return nil, errors.New("invalid compression mode")
} }
@ -181,7 +181,7 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
h := restic.Handle{Type: t, Name: id.String()} h := backend.Handle{Type: t, Name: id.String()}
retriedInvalidData := false retriedInvalidData := false
var dataErr error var dataErr error
wr := new(bytes.Buffer) wr := new(bytes.Buffer)
@ -232,7 +232,7 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res
} }
type haver interface { type haver interface {
Has(restic.Handle) bool Has(backend.Handle) bool
} }
// sortCachedPacksFirst moves all cached pack files to the front of blobs. // sortCachedPacksFirst moves all cached pack files to the front of blobs.
@ -250,7 +250,7 @@ func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) {
noncached := make([]restic.PackedBlob, 0, len(blobs)/2) noncached := make([]restic.PackedBlob, 0, len(blobs)/2)
for _, blob := range blobs { for _, blob := range blobs {
if cache.Has(restic.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) { if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) {
cached = append(cached, blob) cached = append(cached, blob)
continue continue
} }
@ -284,7 +284,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.
} }
// load blob from pack // load blob from pack
h := restic.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: t.IsMetadata()} h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: t.IsMetadata()}
switch { switch {
case cap(buf) < int(blob.Length): case cap(buf) < int(blob.Length):
@ -494,9 +494,9 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by
} else { } else {
id = restic.Hash(ciphertext) id = restic.Hash(ciphertext)
} }
h := restic.Handle{Type: t, Name: id.String()} h := backend.Handle{Type: t, Name: id.String()}
err = r.be.Save(ctx, h, restic.NewByteReader(ciphertext, r.be.Hasher())) err = r.be.Save(ctx, h, backend.NewByteReader(ciphertext, r.be.Hasher()))
if err != nil { if err != nil {
debug.Log("error saving blob %v: %v", h, err) debug.Log("error saving blob %v: %v", h, err)
return restic.ID{}, err return restic.ID{}, err
@ -561,7 +561,7 @@ func (r *Repository) flushPacks(ctx context.Context) error {
} }
// Backend returns the backend for the repository. // Backend returns the backend for the repository.
func (r *Repository) Backend() restic.Backend { func (r *Repository) Backend() backend.Backend {
return r.be return r.be
} }
@ -591,7 +591,7 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
if p != nil { if p != nil {
var numIndexFiles uint64 var numIndexFiles uint64
err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error { err := indexList.List(ctx, restic.IndexFile, func(fi backend.FileInfo) error {
_, err := restic.ParseID(fi.Name) _, err := restic.ParseID(fi.Name)
if err != nil { if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name) debug.Log("unable to parse %v as an ID", fi.Name)
@ -773,7 +773,7 @@ func (r *Repository) Init(ctx context.Context, version uint, password string, ch
return fmt.Errorf("repository version %v too low", version) return fmt.Errorf("repository version %v too low", version)
} }
_, err := r.be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) _, err := r.be.Stat(ctx, backend.Handle{Type: restic.ConfigFile})
if err != nil && !r.be.IsNotExist(err) { if err != nil && !r.be.IsNotExist(err) {
return err return err
} }
@ -818,7 +818,7 @@ func (r *Repository) KeyID() restic.ID {
// List runs fn for all files of type t in the repo. // List runs fn for all files of type t in the repo.
func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error { func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error {
return r.be.List(ctx, t, func(fi restic.FileInfo) error { return r.be.List(ctx, t, func(fi backend.FileInfo) error {
id, err := restic.ParseID(fi.Name) id, err := restic.ParseID(fi.Name)
if err != nil { if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name) debug.Log("unable to parse %v as an ID", fi.Name)
@ -831,7 +831,7 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic
// ListPack returns the list of blobs saved in the pack id and the length of // ListPack returns the list of blobs saved in the pack id and the length of
// the the pack header. // the the pack header.
func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, uint32, error) { func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, uint32, error) {
h := restic.Handle{Type: restic.PackFile, Name: id.String()} h := backend.Handle{Type: restic.PackFile, Name: id.String()}
return pack.List(r.Key(), backend.ReaderAt(ctx, r.Backend(), h), size) return pack.List(r.Key(), backend.ReaderAt(ctx, r.Backend(), h), size)
} }
@ -881,7 +881,7 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte
return newID, known, size, err return newID, known, size, err
} }
type BackendLoadFn func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error type BackendLoadFn func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error
// Skip sections with more than 4MB unused blobs // Skip sections with more than 4MB unused blobs
const maxUnusedRange = 4 * 1024 * 1024 const maxUnusedRange = 4 * 1024 * 1024
@ -922,7 +922,7 @@ func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, pack
} }
func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
h := restic.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: false} h := backend.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: false}
dataStart := blobs[0].Offset dataStart := blobs[0].Offset
dataEnd := blobs[len(blobs)-1].Offset + blobs[len(blobs)-1].Length dataEnd := blobs[len(blobs)-1].Offset + blobs[len(blobs)-1].Length

13
internal/repository/repository_internal_test.go
View File

@ -5,13 +5,14 @@ import (
"sort" "sort"
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
type mapcache map[restic.Handle]bool type mapcache map[backend.Handle]bool
func (c mapcache) Has(h restic.Handle) bool { return c[h] } func (c mapcache) Has(h backend.Handle) bool { return c[h] }
func TestSortCachedPacksFirst(t *testing.T) { func TestSortCachedPacksFirst(t *testing.T) {
var ( var (
@ -27,15 +28,15 @@ func TestSortCachedPacksFirst(t *testing.T) {
blobs[i] = restic.PackedBlob{PackID: id} blobs[i] = restic.PackedBlob{PackID: id}
if i%3 == 0 { if i%3 == 0 {
h := restic.Handle{Name: id.String(), Type: restic.PackFile} h := backend.Handle{Name: id.String(), Type: backend.PackFile}
cache[h] = true cache[h] = true
} }
} }
copy(sorted[:], blobs[:]) copy(sorted[:], blobs[:])
sort.SliceStable(sorted[:], func(i, j int) bool { sort.SliceStable(sorted[:], func(i, j int) bool {
hi := restic.Handle{Type: restic.PackFile, Name: sorted[i].PackID.String()} hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID.String()}
hj := restic.Handle{Type: restic.PackFile, Name: sorted[j].PackID.String()} hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID.String()}
return cache.Has(hi) && !cache.Has(hj) return cache.Has(hi) && !cache.Has(hj)
}) })
@ -58,7 +59,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) {
blobs[i] = restic.PackedBlob{PackID: id} blobs[i] = restic.PackedBlob{PackID: id}
if i%3 == 0 { if i%3 == 0 {
h := restic.Handle{Name: id.String(), Type: restic.PackFile} h := backend.Handle{Name: id.String(), Type: backend.PackFile}
cache[h] = true cache[h] = true
} }
} }

13
internal/repository/repository_test.go
View File

@ -16,6 +16,7 @@ import (
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/klauspost/compress/zstd" "github.com/klauspost/compress/zstd"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/index" "github.com/restic/restic/internal/index"
@ -278,13 +279,13 @@ func TestRepositoryLoadUnpackedBroken(t *testing.T) {
data := rtest.Random(23, 12345) data := rtest.Random(23, 12345)
id := restic.Hash(data) id := restic.Hash(data)
h := restic.Handle{Type: restic.IndexFile, Name: id.String()} h := backend.Handle{Type: restic.IndexFile, Name: id.String()}
// damage buffer // damage buffer
data[0] ^= 0xff data[0] ^= 0xff
repo := repository.TestOpenLocal(t, repodir) repo := repository.TestOpenLocal(t, repodir)
// store broken file // store broken file
err := repo.Backend().Save(context.TODO(), h, restic.NewByteReader(data, nil)) err := repo.Backend().Save(context.TODO(), h, backend.NewByteReader(data, nil))
rtest.OK(t, err) rtest.OK(t, err)
// without a retry backend this will just return an error that the file is broken // without a retry backend this will just return an error that the file is broken
@ -296,10 +297,10 @@ func TestRepositoryLoadUnpackedBroken(t *testing.T) {
} }
type damageOnceBackend struct { type damageOnceBackend struct {
restic.Backend backend.Backend
} }
func (be *damageOnceBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { func (be *damageOnceBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
// don't break the config file as we can't retry it // don't break the config file as we can't retry it
if h.Type == restic.ConfigFile { if h.Type == restic.ConfigFile {
return be.Backend.Load(ctx, h, length, offset, fn) return be.Backend.Load(ctx, h, length, offset, fn)
@ -352,7 +353,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) {
rtest.OK(b, err) rtest.OK(b, err)
b.Logf("index saved as %v", id.Str()) b.Logf("index saved as %v", id.Str())
fi, err := repo.Backend().Stat(context.TODO(), restic.Handle{Type: restic.IndexFile, Name: id.String()}) fi, err := repo.Backend().Stat(context.TODO(), backend.Handle{Type: restic.IndexFile, Name: id.String()})
rtest.OK(b, err) rtest.OK(b, err)
b.Logf("filesize is %v", fi.Size) b.Logf("filesize is %v", fi.Size)
@ -528,7 +529,7 @@ func testStreamPack(t *testing.T, version uint) {
packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress) packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress)
loadCalls := 0 loadCalls := 0
load := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { load := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
data := packfile data := packfile
if offset > int64(len(data)) { if offset > int64(len(data)) {

7
internal/repository/testing.go
View File

@ -6,6 +6,7 @@ import (
"os" "os"
"testing" "testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/backend/retry" "github.com/restic/restic/internal/backend/retry"
@ -34,7 +35,7 @@ func TestUseLowSecurityKDFParameters(t logger) {
} }
// TestBackend returns a fully configured in-memory backend. // TestBackend returns a fully configured in-memory backend.
func TestBackend(_ testing.TB) restic.Backend { func TestBackend(_ testing.TB) backend.Backend {
return mem.New() return mem.New()
} }
@ -43,7 +44,7 @@ const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173)
// TestRepositoryWithBackend returns a repository initialized with a test // TestRepositoryWithBackend returns a repository initialized with a test
// password. If be is nil, an in-memory backend is used. A constant polynomial // password. If be is nil, an in-memory backend is used. A constant polynomial
// is used for the chunker and low-security test parameters. // is used for the chunker and low-security test parameters.
func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) restic.Repository { func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint) restic.Repository {
t.Helper() t.Helper()
TestUseLowSecurityKDFParameters(t) TestUseLowSecurityKDFParameters(t)
restic.TestDisableCheckPolynomial(t) restic.TestDisableCheckPolynomial(t)
@ -98,7 +99,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository {
// TestOpenLocal opens a local repository. // TestOpenLocal opens a local repository.
func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) { func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) {
var be restic.Backend var be backend.Backend
be, err := local.Open(context.TODO(), local.Config{Path: dir, Connections: 2}) be, err := local.Open(context.TODO(), local.Config{Path: dir, Connections: 2})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)

5
internal/restic/backend_find.go
View File

@ -4,6 +4,7 @@ import (
"context" "context"
"fmt" "fmt"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
) )
@ -26,13 +27,13 @@ func (e *NoIDByPrefixError) Error() string {
// Find loads the list of all files of type t and searches for names which // Find loads the list of all files of type t and searches for names which
// start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned. // start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned.
// If more than one is found, nil and ErrMultipleIDMatches is returned. // If more than one is found, nil and ErrMultipleIDMatches is returned.
func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error) { func Find(ctx context.Context, be backend.Lister, t FileType, prefix string) (ID, error) {
match := ID{} match := ID{}
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
err := be.List(ctx, t, func(fi FileInfo) error { err := be.List(ctx, t, func(fi backend.FileInfo) error {
// ignore filename which are not an id // ignore filename which are not an id
id, err := ParseID(fi.Name) id, err := ParseID(fi.Name)
if err != nil { if err != nil {

10
internal/restic/backend_find_test.go
View File

@ -4,13 +4,15 @@ import (
"context" "context"
"strings" "strings"
"testing" "testing"
"github.com/restic/restic/internal/backend"
) )
type mockBackend struct { type mockBackend struct {
list func(context.Context, FileType, func(FileInfo) error) error list func(context.Context, FileType, func(backend.FileInfo) error) error
} }
func (m mockBackend) List(ctx context.Context, t FileType, fn func(FileInfo) error) error { func (m mockBackend) List(ctx context.Context, t FileType, fn func(backend.FileInfo) error) error {
return m.list(ctx, t, fn) return m.list(ctx, t, fn)
} }
@ -29,9 +31,9 @@ func TestFind(t *testing.T) {
list := samples list := samples
m := mockBackend{} m := mockBackend{}
m.list = func(ctx context.Context, t FileType, fn func(FileInfo) error) error { m.list = func(ctx context.Context, t FileType, fn func(backend.FileInfo) error) error {
for _, id := range list { for _, id := range list {
err := fn(FileInfo{Name: id.String()}) err := fn(backend.FileInfo{Name: id.String()})
if err != nil { if err != nil {
return err return err
} }

38
internal/restic/backend_test.go
View File

@ -1,38 +0,0 @@
package restic_test
import (
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
type testBackend struct {
restic.Backend
}
func (t *testBackend) Unwrap() restic.Backend {
return nil
}
type otherTestBackend struct {
restic.Backend
}
func (t *otherTestBackend) Unwrap() restic.Backend {
return t.Backend
}
func TestAsBackend(t *testing.T) {
other := otherTestBackend{}
test.Assert(t, restic.AsBackend[*testBackend](other) == nil, "otherTestBackend is not a testBackend backend")
testBe := &testBackend{}
test.Assert(t, restic.AsBackend[*testBackend](testBe) == testBe, "testBackend was not returned")
wrapper := &otherTestBackend{Backend: testBe}
test.Assert(t, restic.AsBackend[*testBackend](wrapper) == testBe, "failed to unwrap testBackend backend")
wrapper.Backend = other
test.Assert(t, restic.AsBackend[*testBackend](wrapper) == nil, "a wrapped otherTestBackend is not a testBackend")
}

17
internal/restic/lock.go
View File

@ -12,6 +12,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
@ -213,7 +214,7 @@ func (l *Lock) Unlock() error {
return nil return nil
} }
return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: l.lockID.String()}) return l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: l.lockID.String()})
} }
var StaleLockTimeout = 30 * time.Minute var StaleLockTimeout = 30 * time.Minute
@ -273,7 +274,7 @@ func (l *Lock) Refresh(ctx context.Context) error {
oldLockID := l.lockID oldLockID := l.lockID
l.lockID = &id l.lockID = &id
return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: oldLockID.String()}) return l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: oldLockID.String()})
} }
// RefreshStaleLock is an extended variant of Refresh that can also refresh stale lock files. // RefreshStaleLock is an extended variant of Refresh that can also refresh stale lock files.
@ -302,13 +303,13 @@ func (l *Lock) RefreshStaleLock(ctx context.Context) error {
exists, err = l.checkExistence(ctx) exists, err = l.checkExistence(ctx)
if err != nil { if err != nil {
// cleanup replacement lock // cleanup replacement lock
_ = l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: id.String()}) _ = l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: id.String()})
return err return err
} }
if !exists { if !exists {
// cleanup replacement lock // cleanup replacement lock
_ = l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: id.String()}) _ = l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: id.String()})
return ErrRemovedLock return ErrRemovedLock
} }
@ -319,7 +320,7 @@ func (l *Lock) RefreshStaleLock(ctx context.Context) error {
oldLockID := l.lockID oldLockID := l.lockID
l.lockID = &id l.lockID = &id
return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: oldLockID.String()}) return l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: oldLockID.String()})
} }
func (l *Lock) checkExistence(ctx context.Context) (bool, error) { func (l *Lock) checkExistence(ctx context.Context) (bool, error) {
@ -328,7 +329,7 @@ func (l *Lock) checkExistence(ctx context.Context) (bool, error) {
exists := false exists := false
err := l.repo.Backend().List(ctx, LockFile, func(fi FileInfo) error { err := l.repo.Backend().List(ctx, LockFile, func(fi backend.FileInfo) error {
if fi.Name == l.lockID.String() { if fi.Name == l.lockID.String() {
exists = true exists = true
} }
@ -387,7 +388,7 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) {
} }
if lock.Stale() { if lock.Stale() {
err = repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()}) err = repo.Backend().Remove(ctx, backend.Handle{Type: LockFile, Name: id.String()})
if err == nil { if err == nil {
processed++ processed++
} }
@ -403,7 +404,7 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) {
func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) { func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) {
var processed uint32 var processed uint32
err := ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error { err := ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error {
err := repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()}) err := repo.Backend().Remove(ctx, backend.Handle{Type: LockFile, Name: id.String()})
if err == nil { if err == nil {
atomic.AddUint32(&processed, 1) atomic.AddUint32(&processed, 1)
} }

11
internal/restic/lock_test.go
View File

@ -8,6 +8,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -53,10 +54,10 @@ func TestMultipleLock(t *testing.T) {
} }
type failLockLoadingBackend struct { type failLockLoadingBackend struct {
restic.Backend backend.Backend
} }
func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { func (be *failLockLoadingBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
if h.Type == restic.LockFile { if h.Type == restic.LockFile {
return fmt.Errorf("error loading lock") return fmt.Errorf("error loading lock")
} }
@ -130,7 +131,7 @@ func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, er
} }
func removeLock(repo restic.Repository, id restic.ID) error { func removeLock(repo restic.Repository, id restic.ID) error {
h := restic.Handle{Type: restic.LockFile, Name: id.String()} h := backend.Handle{Type: restic.LockFile, Name: id.String()}
return repo.Backend().Remove(context.TODO(), h) return repo.Backend().Remove(context.TODO(), h)
} }
@ -191,7 +192,7 @@ func TestLockStale(t *testing.T) {
} }
func lockExists(repo restic.Repository, t testing.TB, id restic.ID) bool { func lockExists(repo restic.Repository, t testing.TB, id restic.ID) bool {
h := restic.Handle{Type: restic.LockFile, Name: id.String()} h := backend.Handle{Type: restic.LockFile, Name: id.String()}
_, err := repo.Backend().Stat(context.TODO(), h) _, err := repo.Backend().Stat(context.TODO(), h)
if err != nil && !repo.Backend().IsNotExist(err) { if err != nil && !repo.Backend().IsNotExist(err) {
t.Fatal(err) t.Fatal(err)
@ -317,7 +318,7 @@ func TestLockRefreshStaleMissing(t *testing.T) {
lockID := checkSingleLock(t, repo) lockID := checkSingleLock(t, repo)
// refresh must fail if lock was removed // refresh must fail if lock was removed
rtest.OK(t, repo.Backend().Remove(context.TODO(), restic.Handle{Type: restic.LockFile, Name: lockID.String()})) rtest.OK(t, repo.Backend().Remove(context.TODO(), backend.Handle{Type: restic.LockFile, Name: lockID.String()}))
time.Sleep(time.Millisecond) time.Sleep(time.Millisecond)
err = lock.RefreshStaleLock(context.TODO()) err = lock.RefreshStaleLock(context.TODO())
rtest.Assert(t, err == restic.ErrRemovedLock, "unexpected error, expected %v, got %v", restic.ErrRemovedLock, err) rtest.Assert(t, err == restic.ErrRemovedLock, "unexpected error, expected %v, got %v", restic.ErrRemovedLock, err)

Some files were not shown because too many files have changed in this diff Show More