From 1b8a67fe76c8e12d55e271fa6b470269a4f8dc43 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 1 Oct 2023 11:40:12 +0200 Subject: [PATCH] move Backend interface to backend package --- cmd/restic/cmd_backup_integration_test.go | 3 +- cmd/restic/cmd_cat.go | 2 +- cmd/restic/cmd_debug.go | 4 +- cmd/restic/cmd_diff.go | 2 +- cmd/restic/cmd_key.go | 7 +- cmd/restic/cmd_key_integration_test.go | 10 +- cmd/restic/cmd_prune_integration_test.go | 6 +- .../cmd_repair_index_integration_test.go | 13 +-- cmd/restic/cmd_repair_packs.go | 3 +- cmd/restic/cmd_rewrite.go | 4 +- cmd/restic/cmd_tag.go | 3 +- cmd/restic/delete.go | 3 +- cmd/restic/find.go | 2 +- cmd/restic/global.go | 12 +-- cmd/restic/integration_helpers_test.go | 7 +- cmd/restic/integration_test.go | 19 ++-- cmd/restic/lock.go | 7 +- cmd/restic/lock_test.go | 19 ++-- internal/archiver/archiver_test.go | 13 +-- internal/backend/azure/azure.go | 26 ++--- internal/backend/azure/azure_test.go | 4 +- internal/backend/azure/config.go | 4 +- internal/backend/b2/b2.go | 28 ++--- internal/backend/b2/config.go | 4 +- internal/{restic => backend}/backend.go | 11 +- internal/backend/backend_test.go | 38 +++++++ internal/backend/dryrun/dry_backend.go | 20 ++-- internal/backend/dryrun/dry_backend_test.go | 16 +-- internal/{restic => backend}/file.go | 2 +- internal/{restic => backend}/file_test.go | 2 +- internal/backend/gs/config.go | 4 +- internal/backend/gs/gs.go | 28 ++--- internal/backend/layout/layout.go | 11 +- internal/backend/layout/layout_default.go | 28 ++--- internal/backend/layout/layout_rest.go | 14 +-- internal/backend/layout/layout_s3legacy.go | 26 ++--- internal/backend/layout/layout_test.go | 80 +++++++------- internal/backend/limiter/limiter_backend.go | 22 ++-- .../backend/limiter/limiter_backend_test.go | 12 +-- internal/backend/local/layout_test.go | 4 +- internal/backend/local/local.go | 31 +++--- internal/backend/local/local_internal_test.go | 4 +- .../backend/location/display_location_test.go | 4 +- internal/backend/location/location_test.go | 4 +- internal/backend/location/registry.go | 16 +-- internal/backend/logger/log.go | 22 ++-- internal/backend/mem/mem_backend.go | 30 +++--- internal/backend/mock/backend.go | 28 ++--- internal/backend/rclone/internal_test.go | 6 +- internal/backend/readerat.go | 9 +- internal/backend/rest/config.go | 4 +- internal/backend/rest/rest.go | 46 ++++---- internal/backend/rest/rest_int_test.go | 14 +-- internal/backend/retry/backend_retry.go | 24 ++--- internal/backend/retry/backend_retry_test.go | 57 +++++----- internal/{restic => backend}/rewind_reader.go | 2 +- .../{restic => backend}/rewind_reader_test.go | 2 +- internal/backend/s3/config.go | 4 +- internal/backend/s3/s3.go | 30 +++--- internal/backend/s3/s3_test.go | 4 +- internal/backend/sema/backend.go | 28 ++--- internal/backend/sema/backend_test.go | 64 +++++------ internal/backend/sftp/layout_test.go | 4 +- internal/backend/sftp/sftp.go | 27 +++-- internal/backend/swift/config.go | 4 +- internal/backend/swift/swift.go | 26 ++--- internal/backend/swift/swift_test.go | 4 +- internal/backend/test/benchmarks.go | 13 +-- internal/backend/test/doc.go | 2 +- internal/backend/test/suite.go | 13 ++- internal/backend/test/tests.go | 100 +++++++++--------- internal/backend/util/defaults.go | 26 ++--- internal/backend/util/defaults_test.go | 10 +- internal/backend/utils.go | 17 ++- internal/backend/utils_test.go | 40 +++---- internal/cache/backend.go | 34 +++--- internal/cache/backend_test.go | 20 ++-- internal/cache/cache.go | 3 +- internal/cache/file.go | 14 +-- internal/cache/file_test.go | 15 +-- internal/checker/checker.go | 10 +- internal/checker/checker_test.go | 17 +-- internal/index/index_parallel.go | 3 +- internal/index/master_index_test.go | 3 +- internal/migrations/s3_layout.go | 9 +- internal/migrations/upgrade_repo_v2.go | 7 +- internal/migrations/upgrade_repo_v2_test.go | 8 +- internal/pack/pack_test.go | 8 +- internal/repository/key.go | 6 +- internal/repository/packer_manager.go | 7 +- internal/repository/repack_test.go | 5 +- internal/repository/repository.go | 30 +++--- .../repository/repository_internal_test.go | 13 +-- internal/repository/repository_test.go | 13 +-- internal/repository/testing.go | 7 +- internal/restic/backend_find.go | 5 +- internal/restic/backend_find_test.go | 10 +- internal/restic/backend_test.go | 38 ------- internal/restic/lock.go | 17 +-- internal/restic/lock_test.go | 11 +- internal/restic/parallel.go | 5 +- internal/restic/repository.go | 18 +++- internal/restic/snapshot.go | 3 +- internal/restic/snapshot_find.go | 9 +- internal/restorer/filerestorer_test.go | 7 +- 105 files changed, 822 insertions(+), 775 deletions(-) rename internal/{restic => backend}/backend.go (93%) create mode 100644 internal/backend/backend_test.go rename internal/{restic => backend}/file.go (98%) rename internal/{restic => backend}/file_test.go (98%) rename internal/{restic => backend}/rewind_reader.go (99%) rename internal/{restic => backend}/rewind_reader_test.go (99%) delete mode 100644 internal/restic/backend_test.go diff --git a/cmd/restic/cmd_backup_integration_test.go b/cmd/restic/cmd_backup_integration_test.go index 76c227e3d..742b6ff6c 100644 --- a/cmd/restic/cmd_backup_integration_test.go +++ b/cmd/restic/cmd_backup_integration_test.go @@ -9,6 +9,7 @@ import ( "runtime" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -265,7 +266,7 @@ func TestBackupTreeLoadError(t *testing.T) { // delete the subdirectory pack first for id := range treePacks { - rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})) + rtest.OK(t, r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()})) } testRunRebuildIndex(t, env.gopts) // now the repo is missing the tree blob in the index; check should report this diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 238614cd0..2bccd649e 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -154,7 +154,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { return nil case "pack": - h := restic.Handle{Type: restic.PackFile, Name: id.String()} + h := backend.Handle{Type: restic.PackFile, Name: id.String()} buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h) if err != nil { return err diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index bf05c448d..443748f7b 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -321,7 +321,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li panic(err) } be := repo.Backend() - h := restic.Handle{ + h := backend.Handle{ Name: packID.String(), Type: restic.PackFile, } @@ -490,7 +490,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error { Printf("examine %v\n", id) - h := restic.Handle{ + h := backend.Handle{ Type: restic.PackFile, Name: id.String(), } diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 125904068..2bbeed15b 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -58,7 +58,7 @@ func init() { f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata") } -func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) { +func loadSnapshot(ctx context.Context, be backend.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) { sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc) if err != nil { return nil, "", errors.Fatal(err.Error()) diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index ab41b4be3..e968a4f7d 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -7,6 +7,7 @@ import ( "strings" "sync" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -149,7 +150,7 @@ func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) e return errors.Fatal("refusing to remove key currently used to access repository") } - h := restic.Handle{Type: restic.KeyFile, Name: id.String()} + h := backend.Handle{Type: restic.KeyFile, Name: id.String()} err := repo.Backend().Remove(ctx, h) if err != nil { return err @@ -176,7 +177,7 @@ func changePassword(ctx context.Context, repo *repository.Repository, gopts Glob return err } - h := restic.Handle{Type: restic.KeyFile, Name: oldID.String()} + h := backend.Handle{Type: restic.KeyFile, Name: oldID.String()} err = repo.Backend().Remove(ctx, h) if err != nil { return err @@ -193,7 +194,7 @@ func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repos err := repo.SearchKey(ctx, pw, 0, key.ID().String()) if err != nil { // the key is invalid, try to remove it - h := restic.Handle{Type: restic.KeyFile, Name: key.ID().String()} + h := backend.Handle{Type: restic.KeyFile, Name: key.ID().String()} _ = repo.Backend().Remove(ctx, h) return errors.Fatalf("failed to access repository with new key: %v", err) } diff --git a/cmd/restic/cmd_key_integration_test.go b/cmd/restic/cmd_key_integration_test.go index 9ea5795ba..f68799dde 100644 --- a/cmd/restic/cmd_key_integration_test.go +++ b/cmd/restic/cmd_key_integration_test.go @@ -6,8 +6,8 @@ import ( "regexp" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -110,11 +110,11 @@ func TestKeyAddRemove(t *testing.T) { } type emptySaveBackend struct { - restic.Backend + backend.Backend } -func (b *emptySaveBackend) Save(ctx context.Context, h restic.Handle, _ restic.RewindReader) error { - return b.Backend.Save(ctx, h, restic.NewByteReader([]byte{}, nil)) +func (b *emptySaveBackend) Save(ctx context.Context, h backend.Handle, _ backend.RewindReader) error { + return b.Backend.Save(ctx, h, backend.NewByteReader([]byte{}, nil)) } func TestKeyProblems(t *testing.T) { @@ -122,7 +122,7 @@ func TestKeyProblems(t *testing.T) { defer cleanup() testRunInit(t, env.gopts) - env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return &emptySaveBackend{r}, nil } diff --git a/cmd/restic/cmd_prune_integration_test.go b/cmd/restic/cmd_prune_integration_test.go index 2cd86d895..53e27ee10 100644 --- a/cmd/restic/cmd_prune_integration_test.go +++ b/cmd/restic/cmd_prune_integration_test.go @@ -6,13 +6,13 @@ import ( "path/filepath" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" rtest "github.com/restic/restic/internal/test" ) func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) { oldHook := gopts.backendTestHook - gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil } + gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil } defer func() { gopts.backendTestHook = oldHook }() @@ -130,7 +130,7 @@ func TestPruneWithDamagedRepository(t *testing.T) { removePacksExcept(env.gopts, t, oldPacks, false) oldHook := env.gopts.backendTestHook - env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil } + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil } defer func() { env.gopts.backendTestHook = oldHook }() diff --git a/cmd/restic/cmd_repair_index_integration_test.go b/cmd/restic/cmd_repair_index_integration_test.go index f451173a3..e3271361a 100644 --- a/cmd/restic/cmd_repair_index_integration_test.go +++ b/cmd/restic/cmd_repair_index_integration_test.go @@ -8,6 +8,7 @@ import ( "sync" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/restic" @@ -70,12 +71,12 @@ func TestRebuildIndexAlwaysFull(t *testing.T) { // indexErrorBackend modifies the first index after reading. type indexErrorBackend struct { - restic.Backend + backend.Backend lock sync.Mutex hasErred bool } -func (b *indexErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { +func (b *indexErrorBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { // protect hasErred b.lock.Lock() @@ -101,7 +102,7 @@ func (erd errorReadCloser) Read(p []byte) (int, error) { } func TestRebuildIndexDamage(t *testing.T) { - testRebuildIndex(t, func(r restic.Backend) (restic.Backend, error) { + testRebuildIndex(t, func(r backend.Backend) (backend.Backend, error) { return &indexErrorBackend{ Backend: r, }, nil @@ -109,11 +110,11 @@ func TestRebuildIndexDamage(t *testing.T) { } type appendOnlyBackend struct { - restic.Backend + backend.Backend } // called via repo.Backend().Remove() -func (b *appendOnlyBackend) Remove(_ context.Context, h restic.Handle) error { +func (b *appendOnlyBackend) Remove(_ context.Context, h backend.Handle) error { return errors.Errorf("Failed to remove %v", h) } @@ -127,7 +128,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) { err := withRestoreGlobalOptions(func() error { globalOptions.stdout = io.Discard - env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return &appendOnlyBackend{r}, nil } return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts) diff --git a/cmd/restic/cmd_repair_packs.go b/cmd/restic/cmd_repair_packs.go index aadfe73be..7d1a3a392 100644 --- a/cmd/restic/cmd_repair_packs.go +++ b/cmd/restic/cmd_repair_packs.go @@ -5,6 +5,7 @@ import ( "io" "os" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -84,7 +85,7 @@ func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repo return errors.Fatalf("%s", err) } - err = repo.Backend().Load(ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error { + err = repo.Backend().Load(ctx, backend.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error { _, err := f.Seek(0, 0) if err != nil { return err diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 9d9ab5d5d..e522d5c3f 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -128,7 +128,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r if dryRun { Verbosef("would delete empty snapshot\n") } else { - h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} + h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} if err = repo.Backend().Remove(ctx, h); err != nil { return false, err } @@ -170,7 +170,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r Verbosef("saved new snapshot %v\n", id.Str()) if forget { - h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} + h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} if err = repo.Backend().Remove(ctx, h); err != nil { return false, err } diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go index 1ad465faa..29c0ec3b2 100644 --- a/cmd/restic/cmd_tag.go +++ b/cmd/restic/cmd_tag.go @@ -5,6 +5,7 @@ import ( "github.com/spf13/cobra" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" @@ -85,7 +86,7 @@ func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Sna debug.Log("new snapshot saved as %v", id) // Remove the old snapshot. - h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} + h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} if err = repo.Backend().Remove(ctx, h); err != nil { return false, err } diff --git a/cmd/restic/delete.go b/cmd/restic/delete.go index 2046ccfde..1b7937bd3 100644 --- a/cmd/restic/delete.go +++ b/cmd/restic/delete.go @@ -5,6 +5,7 @@ import ( "golang.org/x/sync/errgroup" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" ) @@ -45,7 +46,7 @@ func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, rep for i := 0; i < int(workerCount); i++ { wg.Go(func() error { for id := range fileChan { - h := restic.Handle{Type: fileType, Name: id.String()} + h := backend.Handle{Type: fileType, Name: id.String()} err := repo.Backend().Remove(ctx, h) if err != nil { if !gopts.JSON { diff --git a/cmd/restic/find.go b/cmd/restic/find.go index 54d3563b1..6d5d7bbb0 100644 --- a/cmd/restic/find.go +++ b/cmd/restic/find.go @@ -29,7 +29,7 @@ func initSingleSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter) } // FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. -func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, f *restic.SnapshotFilter, snapshotIDs []string) <-chan *restic.Snapshot { +func FindFilteredSnapshots(ctx context.Context, be backend.Lister, loader restic.LoaderUnpacked, f *restic.SnapshotFilter, snapshotIDs []string) <-chan *restic.Snapshot { out := make(chan *restic.Snapshot) go func() { defer close(out) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index fd735edf7..65b4396a2 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -48,7 +48,7 @@ var version = "0.16.1-dev (compiled manually)" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" -type backendWrapper func(r restic.Backend) (restic.Backend, error) +type backendWrapper func(r backend.Backend) (backend.Backend, error) // GlobalOptions hold all global options for restic. type GlobalOptions struct { @@ -553,7 +553,7 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi func parseConfig(loc location.Location, opts options.Options) (interface{}, error) { cfg := loc.Config - if cfg, ok := cfg.(restic.ApplyEnvironmenter); ok { + if cfg, ok := cfg.(backend.ApplyEnvironmenter); ok { cfg.ApplyEnvironment("") } @@ -568,14 +568,14 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro } // Open the backend specified by a location config. -func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { +func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) { debug.Log("parsing location %v", location.StripPassword(gopts.backends, s)) loc, err := location.Parse(gopts.backends, s) if err != nil { return nil, errors.Fatalf("parsing repository location failed: %v", err) } - var be restic.Backend + var be backend.Backend cfg, err := parseConfig(loc, opts) if err != nil { @@ -613,7 +613,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio } // check if config is there - fi, err := be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) + fi, err := be.Stat(ctx, backend.Handle{Type: restic.ConfigFile}) if err != nil { return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(gopts.backends, s)) } @@ -626,7 +626,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio } // Create the backend specified by URI. -func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { +func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) { debug.Log("parsing location %v", location.StripPassword(gopts.backends, s)) loc, err := location.Parse(gopts.backends, s) if err != nil { diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index 2afdbb938..010734a17 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -12,6 +12,7 @@ import ( "sync" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/retry" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" @@ -205,7 +206,7 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) { extended: make(options.Options), // replace this hook with "nil" if listing a filetype more than once is necessary - backendTestHook: func(r restic.Backend) (restic.Backend, error) { return newOrderedListOnceBackend(r), nil }, + backendTestHook: func(r backend.Backend) (backend.Backend, error) { return newOrderedListOnceBackend(r), nil }, // start with default set of backends backends: globalOptions.backends, } @@ -249,7 +250,7 @@ func removePacks(gopts GlobalOptions, t testing.TB, remove restic.IDSet) { rtest.OK(t, err) for id := range remove { - rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})) + rtest.OK(t, r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()})) } } @@ -272,7 +273,7 @@ func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, rem if treePacks.Has(id) != removeTreePacks || keep.Has(id) { return nil } - return r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}) + return r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()}) })) } diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 8ea4d17d9..1304dc7c8 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -41,12 +42,12 @@ func TestCheckRestoreNoLock(t *testing.T) { // backends (like e.g. Amazon S3) as the second listing may be inconsistent to what // is expected by the first listing + some operations. type listOnceBackend struct { - restic.Backend + backend.Backend listedFileType map[restic.FileType]bool strictOrder bool } -func newListOnceBackend(be restic.Backend) *listOnceBackend { +func newListOnceBackend(be backend.Backend) *listOnceBackend { return &listOnceBackend{ Backend: be, listedFileType: make(map[restic.FileType]bool), @@ -54,7 +55,7 @@ func newListOnceBackend(be restic.Backend) *listOnceBackend { } } -func newOrderedListOnceBackend(be restic.Backend) *listOnceBackend { +func newOrderedListOnceBackend(be backend.Backend) *listOnceBackend { return &listOnceBackend{ Backend: be, listedFileType: make(map[restic.FileType]bool), @@ -62,7 +63,7 @@ func newOrderedListOnceBackend(be restic.Backend) *listOnceBackend { } } -func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func(backend.FileInfo) error) error { if t != restic.LockFile && be.listedFileType[t] { return errors.Errorf("tried listing type %v the second time", t) } @@ -77,7 +78,7 @@ func TestListOnce(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil } pruneOpts := PruneOptions{MaxUnused: "0"} @@ -104,10 +105,10 @@ func (r *writeToOnly) WriteTo(w io.Writer) (int64, error) { } type onlyLoadWithWriteToBackend struct { - restic.Backend + backend.Backend } -func (be *onlyLoadWithWriteToBackend) Load(ctx context.Context, h restic.Handle, +func (be *onlyLoadWithWriteToBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { return be.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { @@ -120,7 +121,7 @@ func TestBackendLoadWriteTo(t *testing.T) { defer cleanup() // setup backend which only works if it's WriteTo method is correctly propagated upwards - env.gopts.backendInnerTestHook = func(r restic.Backend) (restic.Backend, error) { + env.gopts.backendInnerTestHook = func(r backend.Backend) (backend.Backend, error) { return &onlyLoadWithWriteToBackend{Backend: r}, nil } @@ -140,7 +141,7 @@ func TestFindListOnce(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil } diff --git a/cmd/restic/lock.go b/cmd/restic/lock.go index 11c1ed8f5..600b7476f 100644 --- a/cmd/restic/lock.go +++ b/cmd/restic/lock.go @@ -6,6 +6,7 @@ import ( "sync" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -131,7 +132,7 @@ type refreshLockRequest struct { result chan bool } -func refreshLocks(ctx context.Context, backend restic.Backend, lockInfo *lockContext, refreshed chan<- struct{}, forceRefresh <-chan refreshLockRequest) { +func refreshLocks(ctx context.Context, backend backend.Backend, lockInfo *lockContext, refreshed chan<- struct{}, forceRefresh <-chan refreshLockRequest) { debug.Log("start") lock := lockInfo.lock ticker := time.NewTicker(refreshInterval) @@ -257,8 +258,8 @@ func monitorLockRefresh(ctx context.Context, lockInfo *lockContext, refreshed <- } } -func tryRefreshStaleLock(ctx context.Context, backend restic.Backend, lock *restic.Lock, cancel context.CancelFunc) bool { - freeze := restic.AsBackend[restic.FreezeBackend](backend) +func tryRefreshStaleLock(ctx context.Context, be backend.Backend, lock *restic.Lock, cancel context.CancelFunc) bool { + freeze := backend.AsBackend[backend.FreezeBackend](be) if freeze != nil { debug.Log("freezing backend") freeze.Freeze() diff --git a/cmd/restic/lock_test.go b/cmd/restic/lock_test.go index 2f8420853..bf22db699 100644 --- a/cmd/restic/lock_test.go +++ b/cmd/restic/lock_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/debug" @@ -104,11 +105,11 @@ func TestLockConflict(t *testing.T) { } type writeOnceBackend struct { - restic.Backend + backend.Backend written bool } -func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (b *writeOnceBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if b.written { return fmt.Errorf("fail after first write") } @@ -117,7 +118,7 @@ func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic. } func TestLockFailedRefresh(t *testing.T) { - repo, cleanup, env := openLockTestRepo(t, func(r restic.Backend) (restic.Backend, error) { + repo, cleanup, env := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) { return &writeOnceBackend{Backend: r}, nil }) defer cleanup() @@ -143,11 +144,11 @@ func TestLockFailedRefresh(t *testing.T) { } type loggingBackend struct { - restic.Backend + backend.Backend t *testing.T } -func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (b *loggingBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { b.t.Logf("save %v @ %v", h, time.Now()) err := b.Backend.Save(ctx, h, rd) b.t.Logf("save finished %v @ %v", h, time.Now()) @@ -155,7 +156,7 @@ func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re } func TestLockSuccessfulRefresh(t *testing.T) { - repo, cleanup, env := openLockTestRepo(t, func(r restic.Backend) (restic.Backend, error) { + repo, cleanup, env := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) { return &loggingBackend{ Backend: r, t: t, @@ -193,12 +194,12 @@ func TestLockSuccessfulRefresh(t *testing.T) { } type slowBackend struct { - restic.Backend + backend.Backend m sync.Mutex sleep time.Duration } -func (b *slowBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (b *slowBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { b.m.Lock() sleep := b.sleep b.m.Unlock() @@ -208,7 +209,7 @@ func (b *slowBackend) Save(ctx context.Context, h restic.Handle, rd restic.Rewin func TestLockSuccessfulStaleRefresh(t *testing.T) { var sb *slowBackend - repo, cleanup, env := openLockTestRepo(t, func(r restic.Backend) (restic.Backend, error) { + repo, cleanup, env := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) { sb = &slowBackend{Backend: r} return sb, nil }) diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index 3c87055d8..c6daed5bb 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/errors" @@ -1842,26 +1843,26 @@ func TestArchiverErrorReporting(t *testing.T) { } type noCancelBackend struct { - restic.Backend + backend.Backend } -func (c *noCancelBackend) Remove(_ context.Context, h restic.Handle) error { +func (c *noCancelBackend) Remove(_ context.Context, h backend.Handle) error { return c.Backend.Remove(context.Background(), h) } -func (c *noCancelBackend) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) error { +func (c *noCancelBackend) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) error { return c.Backend.Save(context.Background(), h, rd) } -func (c *noCancelBackend) Load(_ context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (c *noCancelBackend) Load(_ context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { return c.Backend.Load(context.Background(), h, length, offset, fn) } -func (c *noCancelBackend) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) { +func (c *noCancelBackend) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) { return c.Backend.Stat(context.Background(), h) } -func (c *noCancelBackend) List(_ context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (c *noCancelBackend) List(_ context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { return c.Backend.List(context.Background(), t, fn) } diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index 8fdf3f380..adaa37d97 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -12,12 +12,12 @@ import ( "path" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" @@ -43,7 +43,7 @@ const saveLargeSize = 256 * 1024 * 1024 const defaultListMaxItems = 5000 // make sure that *Backend implements backend.Backend -var _ restic.Backend = &Backend{} +var _ backend.Backend = &Backend{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("azure", ParseConfig, location.NoPassword, Create, Open) @@ -197,7 +197,7 @@ func (be *Backend) Path() string { } // Save stores data in the backend at the handle. -func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) debug.Log("InsertObject(%v, %v)", be.cfg.AccountName, objName) @@ -214,7 +214,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe return err } -func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.RewindReader) error { +func (be *Backend) saveSmall(ctx context.Context, objName string, rd backend.RewindReader) error { blockBlobClient := be.container.NewBlockBlobClient(objName) // upload it as a new "block", use the base64 hash for the ID @@ -239,7 +239,7 @@ func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.Rewi return errors.Wrap(err, "CommitBlockList") } -func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.RewindReader) error { +func (be *Backend) saveLarge(ctx context.Context, objName string, rd backend.RewindReader) error { blockBlobClient := be.container.NewBlockBlobClient(objName) buf := make([]byte, 100*1024*1024) @@ -294,11 +294,11 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { objName := be.Filename(h) blockBlobClient := be.container.NewBlobClient(objName) @@ -317,17 +317,17 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, } // Stat returns information about a blob. -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { objName := be.Filename(h) blobClient := be.container.NewBlobClient(objName) props, err := blobClient.GetProperties(ctx, nil) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "blob.GetProperties") + return backend.FileInfo{}, errors.Wrap(err, "blob.GetProperties") } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Size: *props.ContentLength, Name: h.Name, } @@ -335,7 +335,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, } // Remove removes the blob with the given name and type. -func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { objName := be.Filename(h) blob := be.container.NewBlobClient(objName) @@ -350,7 +350,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { prefix, _ := be.Basedir(t) // make sure prefix ends with a slash @@ -381,7 +381,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F continue } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: path.Base(m), Size: *item.Properties.ContentLength, } diff --git a/internal/backend/azure/azure_test.go b/internal/backend/azure/azure_test.go index 33f65bd52..7df27d325 100644 --- a/internal/backend/azure/azure_test.go +++ b/internal/backend/azure/azure_test.go @@ -122,11 +122,11 @@ func TestUploadLargeFile(t *testing.T) { data := rtest.Random(23, 300*1024*1024) id := restic.Hash(data) - h := restic.Handle{Name: id.String(), Type: restic.PackFile} + h := backend.Handle{Name: id.String(), Type: backend.PackFile} t.Logf("hash of %d bytes: %v", len(data), id) - err = be.Save(ctx, h, restic.NewByteReader(data, be.Hasher())) + err = be.Save(ctx, h, backend.NewByteReader(data, be.Hasher())) if err != nil { t.Fatal(err) } diff --git a/internal/backend/azure/config.go b/internal/backend/azure/config.go index d819b35aa..6ae431f65 100644 --- a/internal/backend/azure/config.go +++ b/internal/backend/azure/config.go @@ -5,9 +5,9 @@ import ( "path" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to an azure compatible @@ -57,7 +57,7 @@ func ParseConfig(s string) (*Config, error) { return &cfg, nil } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index cf80b34b6..451e18810 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -9,12 +9,12 @@ import ( "sync" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/Backblaze/blazer/b2" "github.com/Backblaze/blazer/base" @@ -34,8 +34,8 @@ type b2Backend struct { // Billing happens in 1000 item granlarity, but we are more interested in reducing the number of network round trips const defaultListMaxItems = 10 * 1000 -// ensure statically that *b2Backend implements restic.Backend. -var _ restic.Backend = &b2Backend{} +// ensure statically that *b2Backend implements backend.Backend. +var _ backend.Backend = &b2Backend{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("b2", ParseConfig, location.NoPassword, Create, Open) @@ -85,7 +85,7 @@ func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Clien } // Open opens a connection to the B2 service. -func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { debug.Log("cfg %#v", cfg) ctx, cancel := context.WithCancel(ctx) @@ -118,7 +118,7 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend // Create opens a connection to the B2 service. If the bucket does not exist yet, // it is created. -func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { debug.Log("cfg %#v", cfg) ctx, cancel := context.WithCancel(ctx) @@ -188,14 +188,14 @@ func (be *b2Backend) IsNotExist(err error) bool { // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *b2Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { ctx, cancel := context.WithCancel(ctx) defer cancel() return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *b2Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { name := be.Layout.Filename(h) obj := be.bucket.Object(name) @@ -213,7 +213,7 @@ func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int } // Save stores data in the backend at the handle. -func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *b2Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -237,18 +237,18 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind } // Stat returns information about a blob. -func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { +func (be *b2Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { name := be.Filename(h) obj := be.bucket.Object(name) info, err := obj.Attrs(ctx) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "Stat") + return backend.FileInfo{}, errors.Wrap(err, "Stat") } - return restic.FileInfo{Size: info.Size, Name: h.Name}, nil + return backend.FileInfo{Size: info.Size, Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error { +func (be *b2Backend) Remove(ctx context.Context, h backend.Handle) error { // the retry backend will also repeat the remove method up to 10 times for i := 0; i < 3; i++ { obj := be.bucket.Object(be.Filename(h)) @@ -284,7 +284,7 @@ func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error { } // List returns a channel that yields all names of blobs of type t. -func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *b2Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -299,7 +299,7 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic return err } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: path.Base(obj.Name()), Size: attrs.Size, } diff --git a/internal/backend/b2/config.go b/internal/backend/b2/config.go index 94614e44f..8d947fc1b 100644 --- a/internal/backend/b2/config.go +++ b/internal/backend/b2/config.go @@ -6,9 +6,9 @@ import ( "regexp" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to an b2 compatible @@ -82,7 +82,7 @@ func ParseConfig(s string) (*Config, error) { return &cfg, nil } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/internal/restic/backend.go b/internal/backend/backend.go similarity index 93% rename from internal/restic/backend.go rename to internal/backend/backend.go index ce36706f3..e4a32d93f 100644 --- a/internal/restic/backend.go +++ b/internal/backend/backend.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "context" @@ -70,7 +70,7 @@ type Backend interface { Delete(ctx context.Context) error } -type BackendUnwrapper interface { +type Unwrapper interface { // Unwrap returns the underlying backend or nil if there is none. Unwrap() Backend } @@ -81,7 +81,7 @@ func AsBackend[B Backend](b Backend) B { return be } - if be, ok := b.(BackendUnwrapper); ok { + if be, ok := b.(Unwrapper); ok { b = be.Unwrap() } else { // not the backend we're looking for @@ -110,3 +110,8 @@ type FileInfo struct { type ApplyEnvironmenter interface { ApplyEnvironment(prefix string) } + +// Lister allows listing files in a backend. +type Lister interface { + List(context.Context, FileType, func(FileInfo) error) error +} diff --git a/internal/backend/backend_test.go b/internal/backend/backend_test.go new file mode 100644 index 000000000..28ece55df --- /dev/null +++ b/internal/backend/backend_test.go @@ -0,0 +1,38 @@ +package backend_test + +import ( + "testing" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/test" +) + +type testBackend struct { + backend.Backend +} + +func (t *testBackend) Unwrap() backend.Backend { + return nil +} + +type otherTestBackend struct { + backend.Backend +} + +func (t *otherTestBackend) Unwrap() backend.Backend { + return t.Backend +} + +func TestAsBackend(t *testing.T) { + other := otherTestBackend{} + test.Assert(t, backend.AsBackend[*testBackend](other) == nil, "otherTestBackend is not a testBackend backend") + + testBe := &testBackend{} + test.Assert(t, backend.AsBackend[*testBackend](testBe) == testBe, "testBackend was not returned") + + wrapper := &otherTestBackend{Backend: testBe} + test.Assert(t, backend.AsBackend[*testBackend](wrapper) == testBe, "failed to unwrap testBackend backend") + + wrapper.Backend = other + test.Assert(t, backend.AsBackend[*testBackend](wrapper) == nil, "a wrapped otherTestBackend is not a testBackend") +} diff --git a/internal/backend/dryrun/dry_backend.go b/internal/backend/dryrun/dry_backend.go index f7acb10dd..b3db0210f 100644 --- a/internal/backend/dryrun/dry_backend.go +++ b/internal/backend/dryrun/dry_backend.go @@ -5,8 +5,8 @@ import ( "hash" "io" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" ) // Backend passes reads through to an underlying layer and accepts writes, but @@ -15,20 +15,20 @@ import ( // the repo and does normal operations else. // This is used for `backup --dry-run`. type Backend struct { - b restic.Backend + b backend.Backend } -// statically ensure that Backend implements restic.Backend. -var _ restic.Backend = &Backend{} +// statically ensure that Backend implements backend.Backend. +var _ backend.Backend = &Backend{} -func New(be restic.Backend) *Backend { +func New(be backend.Backend) *Backend { b := &Backend{b: be} debug.Log("created new dry backend") return b } // Save adds new Data to the backend. -func (be *Backend) Save(_ context.Context, h restic.Handle, _ restic.RewindReader) error { +func (be *Backend) Save(_ context.Context, h backend.Handle, _ backend.RewindReader) error { if err := h.Valid(); err != nil { return err } @@ -38,7 +38,7 @@ func (be *Backend) Save(_ context.Context, h restic.Handle, _ restic.RewindReade } // Remove deletes a file from the backend. -func (be *Backend) Remove(_ context.Context, _ restic.Handle) error { +func (be *Backend) Remove(_ context.Context, _ backend.Handle) error { return nil } @@ -72,14 +72,14 @@ func (be *Backend) IsNotExist(err error) bool { return be.b.IsNotExist(err) } -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { return be.b.List(ctx, t, fn) } -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error { +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(io.Reader) error) error { return be.b.Load(ctx, h, length, offset, fn) } -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { return be.b.Stat(ctx, h) } diff --git a/internal/backend/dryrun/dry_backend_test.go b/internal/backend/dryrun/dry_backend_test.go index 69716c340..56962107d 100644 --- a/internal/backend/dryrun/dry_backend_test.go +++ b/internal/backend/dryrun/dry_backend_test.go @@ -8,16 +8,16 @@ import ( "strings" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/dryrun" "github.com/restic/restic/internal/backend/mem" ) // make sure that Backend implements backend.Backend -var _ restic.Backend = &dryrun.Backend{} +var _ backend.Backend = &dryrun.Backend{} -func newBackends() (*dryrun.Backend, restic.Backend) { +func newBackends() (*dryrun.Backend, backend.Backend) { m := mem.New() return dryrun.New(m), m } @@ -30,7 +30,7 @@ func TestDry(t *testing.T) { // won't pass. Instead, perform a series of operations over the backend, testing the state // at each step. steps := []struct { - be restic.Backend + be backend.Backend op string fname string content string @@ -61,13 +61,13 @@ func TestDry(t *testing.T) { for i, step := range steps { var err error - handle := restic.Handle{Type: restic.PackFile, Name: step.fname} + handle := backend.Handle{Type: backend.PackFile, Name: step.fname} switch step.op { case "save": - err = step.be.Save(ctx, handle, restic.NewByteReader([]byte(step.content), step.be.Hasher())) + err = step.be.Save(ctx, handle, backend.NewByteReader([]byte(step.content), step.be.Hasher())) case "list": fileList := []string{} - err = step.be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { + err = step.be.List(ctx, backend.PackFile, func(fi backend.FileInfo) error { fileList = append(fileList, fi.Name) return nil }) @@ -86,7 +86,7 @@ func TestDry(t *testing.T) { case "remove": err = step.be.Remove(ctx, handle) case "stat": - var fi restic.FileInfo + var fi backend.FileInfo fi, err = step.be.Stat(ctx, handle) if err == nil { fis := fmt.Sprintf("%s %d", fi.Name, fi.Size) diff --git a/internal/restic/file.go b/internal/backend/file.go similarity index 98% rename from internal/restic/file.go rename to internal/backend/file.go index 15ee22ccd..990175f9c 100644 --- a/internal/restic/file.go +++ b/internal/backend/file.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "fmt" diff --git a/internal/restic/file_test.go b/internal/backend/file_test.go similarity index 98% rename from internal/restic/file_test.go rename to internal/backend/file_test.go index cc54c2924..45f1c2ee7 100644 --- a/internal/restic/file_test.go +++ b/internal/backend/file_test.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "testing" diff --git a/internal/backend/gs/config.go b/internal/backend/gs/config.go index 61a31113f..7dc181ce9 100644 --- a/internal/backend/gs/config.go +++ b/internal/backend/gs/config.go @@ -5,9 +5,9 @@ import ( "path" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to a Google Cloud Storage @@ -59,7 +59,7 @@ func ParseConfig(s string) (*Config, error) { return &cfg, nil } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index 0865423e5..77d20e056 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -13,11 +13,11 @@ import ( "cloud.google.com/go/storage" "github.com/pkg/errors" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" "golang.org/x/oauth2" "golang.org/x/oauth2/google" @@ -45,8 +45,8 @@ type Backend struct { layout.Layout } -// Ensure that *Backend implements restic.Backend. -var _ restic.Backend = &Backend{} +// Ensure that *Backend implements backend.Backend. +var _ backend.Backend = &Backend{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open) @@ -122,7 +122,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { } // Open opens the gs backend at the specified bucket. -func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Open(_ context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { return open(cfg, rt) } @@ -131,7 +131,7 @@ func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, // // The service account must have the "storage.buckets.create" permission to // create a bucket the does not yet exist. -func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { be, err := open(cfg, rt) if err != nil { return nil, errors.Wrap(err, "open") @@ -203,7 +203,7 @@ func (be *Backend) Path() string { } // Save stores data in the backend at the handle. -func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) // Set chunk size to zero to disable resumable uploads. @@ -253,14 +253,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { ctx, cancel := context.WithCancel(ctx) defer cancel() return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { if length == 0 { // negative length indicates read till end to GCS lib length = -1 @@ -277,20 +277,20 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, } // Stat returns information about a blob. -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { objName := be.Filename(h) attr, err := be.bucket.Object(objName).Attrs(ctx) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "service.Objects.Get") + return backend.FileInfo{}, errors.Wrap(err, "service.Objects.Get") } - return restic.FileInfo{Size: attr.Size, Name: h.Name}, nil + return backend.FileInfo{Size: attr.Size, Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { objName := be.Filename(h) err := be.bucket.Object(objName).Delete(ctx) @@ -304,7 +304,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { prefix, _ := be.Basedir(t) // make sure prefix ends with a slash @@ -330,7 +330,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F continue } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: path.Base(m), Size: int64(attrs.Size), } diff --git a/internal/backend/layout/layout.go b/internal/backend/layout/layout.go index b83f4c05b..b600566a4 100644 --- a/internal/backend/layout/layout.go +++ b/internal/backend/layout/layout.go @@ -7,6 +7,7 @@ import ( "path/filepath" "regexp" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" @@ -15,9 +16,9 @@ import ( // Layout computes paths for file name storage. type Layout interface { - Filename(restic.Handle) string - Dirname(restic.Handle) string - Basedir(restic.FileType) (dir string, subdirs bool) + Filename(backend.Handle) string + Dirname(backend.Handle) string + Basedir(backend.FileType) (dir string, subdirs bool) Paths() []string Name() string } @@ -102,13 +103,13 @@ func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, err } // key file in the "keys" dir (DefaultLayout) - foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile])) + foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[backend.KeyFile])) if err != nil { return nil, err } // key file in the "key" dir (S3LegacyLayout) - foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile])) + foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[backend.KeyFile])) if err != nil { return nil, err } diff --git a/internal/backend/layout/layout_default.go b/internal/backend/layout/layout_default.go index 17c250e8f..9a8419f10 100644 --- a/internal/backend/layout/layout_default.go +++ b/internal/backend/layout/layout_default.go @@ -3,7 +3,7 @@ package layout import ( "encoding/hex" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" ) // DefaultLayout implements the default layout for local and sftp backends, as @@ -15,12 +15,12 @@ type DefaultLayout struct { Join func(...string) string } -var defaultLayoutPaths = map[restic.FileType]string{ - restic.PackFile: "data", - restic.SnapshotFile: "snapshots", - restic.IndexFile: "index", - restic.LockFile: "locks", - restic.KeyFile: "keys", +var defaultLayoutPaths = map[backend.FileType]string{ + backend.PackFile: "data", + backend.SnapshotFile: "snapshots", + backend.IndexFile: "index", + backend.LockFile: "locks", + backend.KeyFile: "keys", } func (l *DefaultLayout) String() string { @@ -33,10 +33,10 @@ func (l *DefaultLayout) Name() string { } // Dirname returns the directory path for a given file type and name. -func (l *DefaultLayout) Dirname(h restic.Handle) string { +func (l *DefaultLayout) Dirname(h backend.Handle) string { p := defaultLayoutPaths[h.Type] - if h.Type == restic.PackFile && len(h.Name) > 2 { + if h.Type == backend.PackFile && len(h.Name) > 2 { p = l.Join(p, h.Name[:2]) + "/" } @@ -44,9 +44,9 @@ func (l *DefaultLayout) Dirname(h restic.Handle) string { } // Filename returns a path to a file, including its name. -func (l *DefaultLayout) Filename(h restic.Handle) string { +func (l *DefaultLayout) Filename(h backend.Handle) string { name := h.Name - if h.Type == restic.ConfigFile { + if h.Type == backend.ConfigFile { return l.Join(l.Path, "config") } @@ -62,15 +62,15 @@ func (l *DefaultLayout) Paths() (dirs []string) { // also add subdirs for i := 0; i < 256; i++ { subdir := hex.EncodeToString([]byte{byte(i)}) - dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[restic.PackFile], subdir)) + dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[backend.PackFile], subdir)) } return dirs } // Basedir returns the base dir name for type t. -func (l *DefaultLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) { - if t == restic.PackFile { +func (l *DefaultLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) { + if t == backend.PackFile { subdirs = true } diff --git a/internal/backend/layout/layout_rest.go b/internal/backend/layout/layout_rest.go index 2aa869995..822dd4a7e 100644 --- a/internal/backend/layout/layout_rest.go +++ b/internal/backend/layout/layout_rest.go @@ -1,6 +1,8 @@ package layout -import "github.com/restic/restic/internal/restic" +import ( + "github.com/restic/restic/internal/backend" +) // RESTLayout implements the default layout for the REST protocol. type RESTLayout struct { @@ -21,8 +23,8 @@ func (l *RESTLayout) Name() string { } // Dirname returns the directory path for a given file type and name. -func (l *RESTLayout) Dirname(h restic.Handle) string { - if h.Type == restic.ConfigFile { +func (l *RESTLayout) Dirname(h backend.Handle) string { + if h.Type == backend.ConfigFile { return l.URL + l.Join(l.Path, "/") } @@ -30,10 +32,10 @@ func (l *RESTLayout) Dirname(h restic.Handle) string { } // Filename returns a path to a file, including its name. -func (l *RESTLayout) Filename(h restic.Handle) string { +func (l *RESTLayout) Filename(h backend.Handle) string { name := h.Name - if h.Type == restic.ConfigFile { + if h.Type == backend.ConfigFile { name = "config" } @@ -49,6 +51,6 @@ func (l *RESTLayout) Paths() (dirs []string) { } // Basedir returns the base dir name for files of type t. -func (l *RESTLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) { +func (l *RESTLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) { return l.URL + l.Join(l.Path, restLayoutPaths[t]), false } diff --git a/internal/backend/layout/layout_s3legacy.go b/internal/backend/layout/layout_s3legacy.go index ac88e77ad..8b90789d8 100644 --- a/internal/backend/layout/layout_s3legacy.go +++ b/internal/backend/layout/layout_s3legacy.go @@ -1,6 +1,8 @@ package layout -import "github.com/restic/restic/internal/restic" +import ( + "github.com/restic/restic/internal/backend" +) // S3LegacyLayout implements the old layout used for s3 cloud storage backends, as // described in the Design document. @@ -10,12 +12,12 @@ type S3LegacyLayout struct { Join func(...string) string } -var s3LayoutPaths = map[restic.FileType]string{ - restic.PackFile: "data", - restic.SnapshotFile: "snapshot", - restic.IndexFile: "index", - restic.LockFile: "lock", - restic.KeyFile: "key", +var s3LayoutPaths = map[backend.FileType]string{ + backend.PackFile: "data", + backend.SnapshotFile: "snapshot", + backend.IndexFile: "index", + backend.LockFile: "lock", + backend.KeyFile: "key", } func (l *S3LegacyLayout) String() string { @@ -44,8 +46,8 @@ func (l *S3LegacyLayout) join(url string, items ...string) string { } // Dirname returns the directory path for a given file type and name. -func (l *S3LegacyLayout) Dirname(h restic.Handle) string { - if h.Type == restic.ConfigFile { +func (l *S3LegacyLayout) Dirname(h backend.Handle) string { + if h.Type == backend.ConfigFile { return l.URL + l.Join(l.Path, "/") } @@ -53,10 +55,10 @@ func (l *S3LegacyLayout) Dirname(h restic.Handle) string { } // Filename returns a path to a file, including its name. -func (l *S3LegacyLayout) Filename(h restic.Handle) string { +func (l *S3LegacyLayout) Filename(h backend.Handle) string { name := h.Name - if h.Type == restic.ConfigFile { + if h.Type == backend.ConfigFile { name = "config" } @@ -72,6 +74,6 @@ func (l *S3LegacyLayout) Paths() (dirs []string) { } // Basedir returns the base dir name for type t. -func (l *S3LegacyLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) { +func (l *S3LegacyLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) { return l.Join(l.Path, s3LayoutPaths[t]), false } diff --git a/internal/backend/layout/layout_test.go b/internal/backend/layout/layout_test.go index fc9c6e214..998f5aeb6 100644 --- a/internal/backend/layout/layout_test.go +++ b/internal/backend/layout/layout_test.go @@ -9,7 +9,7 @@ import ( "sort" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" rtest "github.com/restic/restic/internal/test" ) @@ -19,79 +19,79 @@ func TestDefaultLayout(t *testing.T) { var tests = []struct { path string join func(...string) string - restic.Handle + backend.Handle filename string }{ { tempdir, filepath.Join, - restic.Handle{Type: restic.PackFile, Name: "0123456"}, + backend.Handle{Type: backend.PackFile, Name: "0123456"}, filepath.Join(tempdir, "data", "01", "0123456"), }, { tempdir, filepath.Join, - restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, + backend.Handle{Type: backend.ConfigFile, Name: "CFG"}, filepath.Join(tempdir, "config"), }, { tempdir, filepath.Join, - restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, + backend.Handle{Type: backend.SnapshotFile, Name: "123456"}, filepath.Join(tempdir, "snapshots", "123456"), }, { tempdir, filepath.Join, - restic.Handle{Type: restic.IndexFile, Name: "123456"}, + backend.Handle{Type: backend.IndexFile, Name: "123456"}, filepath.Join(tempdir, "index", "123456"), }, { tempdir, filepath.Join, - restic.Handle{Type: restic.LockFile, Name: "123456"}, + backend.Handle{Type: backend.LockFile, Name: "123456"}, filepath.Join(tempdir, "locks", "123456"), }, { tempdir, filepath.Join, - restic.Handle{Type: restic.KeyFile, Name: "123456"}, + backend.Handle{Type: backend.KeyFile, Name: "123456"}, filepath.Join(tempdir, "keys", "123456"), }, { "", path.Join, - restic.Handle{Type: restic.PackFile, Name: "0123456"}, + backend.Handle{Type: backend.PackFile, Name: "0123456"}, "data/01/0123456", }, { "", path.Join, - restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, + backend.Handle{Type: backend.ConfigFile, Name: "CFG"}, "config", }, { "", path.Join, - restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, + backend.Handle{Type: backend.SnapshotFile, Name: "123456"}, "snapshots/123456", }, { "", path.Join, - restic.Handle{Type: restic.IndexFile, Name: "123456"}, + backend.Handle{Type: backend.IndexFile, Name: "123456"}, "index/123456", }, { "", path.Join, - restic.Handle{Type: restic.LockFile, Name: "123456"}, + backend.Handle{Type: backend.LockFile, Name: "123456"}, "locks/123456", }, { "", path.Join, - restic.Handle{Type: restic.KeyFile, Name: "123456"}, + backend.Handle{Type: backend.KeyFile, Name: "123456"}, "keys/123456", }, } @@ -143,31 +143,31 @@ func TestRESTLayout(t *testing.T) { path := rtest.TempDir(t) var tests = []struct { - restic.Handle + backend.Handle filename string }{ { - restic.Handle{Type: restic.PackFile, Name: "0123456"}, + backend.Handle{Type: backend.PackFile, Name: "0123456"}, filepath.Join(path, "data", "0123456"), }, { - restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, + backend.Handle{Type: backend.ConfigFile, Name: "CFG"}, filepath.Join(path, "config"), }, { - restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, + backend.Handle{Type: backend.SnapshotFile, Name: "123456"}, filepath.Join(path, "snapshots", "123456"), }, { - restic.Handle{Type: restic.IndexFile, Name: "123456"}, + backend.Handle{Type: backend.IndexFile, Name: "123456"}, filepath.Join(path, "index", "123456"), }, { - restic.Handle{Type: restic.LockFile, Name: "123456"}, + backend.Handle{Type: backend.LockFile, Name: "123456"}, filepath.Join(path, "locks", "123456"), }, { - restic.Handle{Type: restic.KeyFile, Name: "123456"}, + backend.Handle{Type: backend.KeyFile, Name: "123456"}, filepath.Join(path, "keys", "123456"), }, } @@ -209,61 +209,61 @@ func TestRESTLayout(t *testing.T) { func TestRESTLayoutURLs(t *testing.T) { var tests = []struct { l Layout - h restic.Handle + h backend.Handle fn string dir string }{ { &RESTLayout{URL: "https://hostname.foo", Path: "", Join: path.Join}, - restic.Handle{Type: restic.PackFile, Name: "foobar"}, + backend.Handle{Type: backend.PackFile, Name: "foobar"}, "https://hostname.foo/data/foobar", "https://hostname.foo/data/", }, { &RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, - restic.Handle{Type: restic.LockFile, Name: "foobar"}, + backend.Handle{Type: backend.LockFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/locks/foobar", "https://hostname.foo:1234/prefix/repo/locks/", }, { &RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, - restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, + backend.Handle{Type: backend.ConfigFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/config", "https://hostname.foo:1234/prefix/repo/", }, { &S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join}, - restic.Handle{Type: restic.PackFile, Name: "foobar"}, + backend.Handle{Type: backend.PackFile, Name: "foobar"}, "https://hostname.foo/data/foobar", "https://hostname.foo/data/", }, { &S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join}, - restic.Handle{Type: restic.LockFile, Name: "foobar"}, + backend.Handle{Type: backend.LockFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/lock/foobar", "https://hostname.foo:1234/prefix/repo/lock/", }, { &S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, - restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, + backend.Handle{Type: backend.ConfigFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/config", "https://hostname.foo:1234/prefix/repo/", }, { &S3LegacyLayout{URL: "", Path: "", Join: path.Join}, - restic.Handle{Type: restic.PackFile, Name: "foobar"}, + backend.Handle{Type: backend.PackFile, Name: "foobar"}, "data/foobar", "data/", }, { &S3LegacyLayout{URL: "", Path: "", Join: path.Join}, - restic.Handle{Type: restic.LockFile, Name: "foobar"}, + backend.Handle{Type: backend.LockFile, Name: "foobar"}, "lock/foobar", "lock/", }, { &S3LegacyLayout{URL: "", Path: "/", Join: path.Join}, - restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, + backend.Handle{Type: backend.ConfigFile, Name: "foobar"}, "/config", "/", }, @@ -288,31 +288,31 @@ func TestS3LegacyLayout(t *testing.T) { path := rtest.TempDir(t) var tests = []struct { - restic.Handle + backend.Handle filename string }{ { - restic.Handle{Type: restic.PackFile, Name: "0123456"}, + backend.Handle{Type: backend.PackFile, Name: "0123456"}, filepath.Join(path, "data", "0123456"), }, { - restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, + backend.Handle{Type: backend.ConfigFile, Name: "CFG"}, filepath.Join(path, "config"), }, { - restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, + backend.Handle{Type: backend.SnapshotFile, Name: "123456"}, filepath.Join(path, "snapshot", "123456"), }, { - restic.Handle{Type: restic.IndexFile, Name: "123456"}, + backend.Handle{Type: backend.IndexFile, Name: "123456"}, filepath.Join(path, "index", "123456"), }, { - restic.Handle{Type: restic.LockFile, Name: "123456"}, + backend.Handle{Type: backend.LockFile, Name: "123456"}, filepath.Join(path, "lock", "123456"), }, { - restic.Handle{Type: restic.KeyFile, Name: "123456"}, + backend.Handle{Type: backend.KeyFile, Name: "123456"}, filepath.Join(path, "key", "123456"), }, } @@ -415,8 +415,8 @@ func TestParseLayout(t *testing.T) { } // test that the functions work (and don't panic) - _ = layout.Dirname(restic.Handle{Type: restic.PackFile}) - _ = layout.Filename(restic.Handle{Type: restic.PackFile, Name: "1234"}) + _ = layout.Dirname(backend.Handle{Type: backend.PackFile}) + _ = layout.Filename(backend.Handle{Type: backend.PackFile, Name: "1234"}) _ = layout.Paths() layoutName := fmt.Sprintf("%T", layout) diff --git a/internal/backend/limiter/limiter_backend.go b/internal/backend/limiter/limiter_backend.go index a91794037..ac1a4188a 100644 --- a/internal/backend/limiter/limiter_backend.go +++ b/internal/backend/limiter/limiter_backend.go @@ -4,12 +4,12 @@ import ( "context" "io" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" ) -func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) { - return func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) { - var be restic.Backend +func WrapBackendConstructor[B backend.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (backend.Backend, error) { + return func(ctx context.Context, cfg C, lim Limiter) (backend.Backend, error) { + var be backend.Backend be, err := constructor(ctx, cfg) if err != nil { return nil, err @@ -24,7 +24,7 @@ func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx contex // LimitBackend wraps a Backend and applies rate limiting to Load() and Save() // calls on the backend. -func LimitBackend(be restic.Backend, l Limiter) restic.Backend { +func LimitBackend(be backend.Backend, l Limiter) backend.Backend { return rateLimitedBackend{ Backend: be, limiter: l, @@ -32,11 +32,11 @@ func LimitBackend(be restic.Backend, l Limiter) restic.Backend { } type rateLimitedBackend struct { - restic.Backend + backend.Backend limiter Limiter } -func (r rateLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (r rateLimitedBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { limited := limitedRewindReader{ RewindReader: rd, limited: r.limiter.Upstream(rd), @@ -46,7 +46,7 @@ func (r rateLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic } type limitedRewindReader struct { - restic.RewindReader + backend.RewindReader limited io.Reader } @@ -55,13 +55,13 @@ func (l limitedRewindReader) Read(b []byte) (int, error) { return l.limited.Read(b) } -func (r rateLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { +func (r rateLimitedBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { return r.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { return consumer(newDownstreamLimitedReader(rd, r.limiter)) }) } -func (r rateLimitedBackend) Unwrap() restic.Backend { return r.Backend } +func (r rateLimitedBackend) Unwrap() backend.Backend { return r.Backend } type limitedReader struct { io.Reader @@ -85,4 +85,4 @@ func (l *limitedReader) WriteTo(w io.Writer) (int64, error) { return l.writerTo.WriteTo(l.limiter.DownstreamWriter(w)) } -var _ restic.Backend = (*rateLimitedBackend)(nil) +var _ backend.Backend = (*rateLimitedBackend)(nil) diff --git a/internal/backend/limiter/limiter_backend_test.go b/internal/backend/limiter/limiter_backend_test.go index 1014dbed1..491d2ef69 100644 --- a/internal/backend/limiter/limiter_backend_test.go +++ b/internal/backend/limiter/limiter_backend_test.go @@ -8,8 +8,8 @@ import ( "io" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mock" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -21,11 +21,11 @@ func randomBytes(t *testing.T, size int) []byte { } func TestLimitBackendSave(t *testing.T) { - testHandle := restic.Handle{Type: restic.PackFile, Name: "test"} + testHandle := backend.Handle{Type: backend.PackFile, Name: "test"} data := randomBytes(t, 1234) be := mock.NewBackend() - be.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + be.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { buf := new(bytes.Buffer) _, err := io.Copy(buf, rd) if err != nil { @@ -39,7 +39,7 @@ func TestLimitBackendSave(t *testing.T) { limiter := NewStaticLimiter(Limits{42 * 1024, 42 * 1024}) limbe := LimitBackend(be, limiter) - rd := restic.NewByteReader(data, nil) + rd := backend.NewByteReader(data, nil) err := limbe.Save(context.TODO(), testHandle, rd) rtest.OK(t, err) } @@ -64,7 +64,7 @@ func (r *tracedReadWriteToCloser) Close() error { } func TestLimitBackendLoad(t *testing.T) { - testHandle := restic.Handle{Type: restic.PackFile, Name: "test"} + testHandle := backend.Handle{Type: backend.PackFile, Name: "test"} data := randomBytes(t, 1234) for _, test := range []struct { @@ -72,7 +72,7 @@ func TestLimitBackendLoad(t *testing.T) { }{{false, false}, {false, true}, {true, false}, {true, true}} { be := mock.NewBackend() src := newTracedReadWriteToCloser(bytes.NewReader(data)) - be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { if length != 0 || offset != 0 { return nil, fmt.Errorf("Not supported") } diff --git a/internal/backend/local/layout_test.go b/internal/backend/local/layout_test.go index a4fccd2cb..46f3996bb 100644 --- a/internal/backend/local/layout_test.go +++ b/internal/backend/local/layout_test.go @@ -5,7 +5,7 @@ import ( "path/filepath" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" rtest "github.com/restic/restic/internal/test" ) @@ -49,7 +49,7 @@ func TestLayout(t *testing.T) { } packs := make(map[string]bool) - err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { packs[fi.Name] = false return nil }) diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index 7a7a819ac..4898bfc7a 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -16,7 +16,6 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" - "github.com/restic/restic/internal/restic" "github.com/cenkalti/backoff/v4" ) @@ -28,8 +27,8 @@ type Local struct { util.Modes } -// ensure statically that *Local implements restic.Backend. -var _ restic.Backend = &Local{} +// ensure statically that *Local implements backend.Backend. +var _ backend.Backend = &Local{} func NewFactory() location.Factory { return location.NewLimitedBackendFactory("local", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) @@ -43,7 +42,7 @@ func open(ctx context.Context, cfg Config) (*Local, error) { return nil, err } - fi, err := fs.Stat(l.Filename(restic.Handle{Type: restic.ConfigFile})) + fi, err := fs.Stat(l.Filename(backend.Handle{Type: backend.ConfigFile})) m := util.DeriveModesFromFileInfo(fi, err) debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) @@ -71,7 +70,7 @@ func Create(ctx context.Context, cfg Config) (*Local, error) { } // test if config file already exists - _, err = fs.Lstat(be.Filename(restic.Handle{Type: restic.ConfigFile})) + _, err = fs.Lstat(be.Filename(backend.Handle{Type: backend.ConfigFile})) if err == nil { return nil, errors.New("config file already exists") } @@ -112,7 +111,7 @@ func (b *Local) IsNotExist(err error) bool { } // Save stores data in the backend at the handle. -func (b *Local) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) (err error) { +func (b *Local) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) (err error) { finalname := b.Filename(h) dir := filepath.Dir(finalname) @@ -210,11 +209,11 @@ var tempFile = os.CreateTemp // Overridden by test. // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (b *Local) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { return util.DefaultLoad(ctx, h, length, offset, b.openReader, fn) } -func (b *Local) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (b *Local) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { f, err := fs.Open(b.Filename(h)) if err != nil { return nil, err @@ -236,17 +235,17 @@ func (b *Local) openReader(_ context.Context, h restic.Handle, length int, offse } // Stat returns information about a blob. -func (b *Local) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) { +func (b *Local) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) { fi, err := fs.Stat(b.Filename(h)) if err != nil { - return restic.FileInfo{}, errors.WithStack(err) + return backend.FileInfo{}, errors.WithStack(err) } - return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil + return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (b *Local) Remove(_ context.Context, h restic.Handle) error { +func (b *Local) Remove(_ context.Context, h backend.Handle) error { fn := b.Filename(h) // reset read-only flag @@ -260,7 +259,7 @@ func (b *Local) Remove(_ context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) (err error) { +func (b *Local) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) (err error) { basedir, subdirs := b.Basedir(t) if subdirs { err = visitDirs(ctx, basedir, fn) @@ -280,7 +279,7 @@ func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.File // two levels of directory structure (including dir itself as the first level). // Also, visitDirs assumes it sees a directory full of directories, while // visitFiles wants a directory full or regular files. -func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error) error { +func visitDirs(ctx context.Context, dir string, fn func(backend.FileInfo) error) error { d, err := fs.Open(dir) if err != nil { return err @@ -307,7 +306,7 @@ func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error) return ctx.Err() } -func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error, ignoreNotADirectory bool) error { +func visitFiles(ctx context.Context, dir string, fn func(backend.FileInfo) error, ignoreNotADirectory bool) error { d, err := fs.Open(dir) if err != nil { return err @@ -341,7 +340,7 @@ func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error, default: } - err := fn(restic.FileInfo{ + err := fn(backend.FileInfo{ Name: fi.Name(), Size: fi.Size(), }) diff --git a/internal/backend/local/local_internal_test.go b/internal/backend/local/local_internal_test.go index 1e80e72ed..6cad26d0a 100644 --- a/internal/backend/local/local_internal_test.go +++ b/internal/backend/local/local_internal_test.go @@ -8,7 +8,7 @@ import ( "syscall" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" rtest "github.com/restic/restic/internal/test" "github.com/cenkalti/backoff/v4" @@ -32,7 +32,7 @@ func TestNoSpacePermanent(t *testing.T) { rtest.OK(t, be.Close()) }() - h := restic.Handle{Type: restic.ConfigFile} + h := backend.Handle{Type: backend.ConfigFile} err = be.Save(context.Background(), h, nil) _, ok := err.(*backoff.PermanentError) rtest.Assert(t, ok, diff --git a/internal/backend/location/display_location_test.go b/internal/backend/location/display_location_test.go index 19502d85b..4011abbf0 100644 --- a/internal/backend/location/display_location_test.go +++ b/internal/backend/location/display_location_test.go @@ -3,15 +3,15 @@ package location_test import ( "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) func TestStripPassword(t *testing.T) { registry := location.NewRegistry() registry.Register( - location.NewHTTPBackendFactory[any, restic.Backend]("test", nil, + location.NewHTTPBackendFactory[any, backend.Backend]("test", nil, func(s string) string { return "cleaned" }, nil, nil, diff --git a/internal/backend/location/location_test.go b/internal/backend/location/location_test.go index b2623032e..fe550a586 100644 --- a/internal/backend/location/location_test.go +++ b/internal/backend/location/location_test.go @@ -3,8 +3,8 @@ package location_test import ( "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) @@ -13,7 +13,7 @@ type testConfig struct { } func testFactory() location.Factory { - return location.NewHTTPBackendFactory[testConfig, restic.Backend]( + return location.NewHTTPBackendFactory[testConfig, backend.Backend]( "local", func(s string) (*testConfig, error) { return &testConfig{loc: s}, nil diff --git a/internal/backend/location/registry.go b/internal/backend/location/registry.go index a8818bd73..b50371add 100644 --- a/internal/backend/location/registry.go +++ b/internal/backend/location/registry.go @@ -4,8 +4,8 @@ import ( "context" "net/http" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/limiter" - "github.com/restic/restic/internal/restic" ) type Registry struct { @@ -33,11 +33,11 @@ type Factory interface { Scheme() string ParseConfig(s string) (interface{}, error) StripPassword(s string) string - Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) - Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) + Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) + Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) } -type genericBackendFactory[C any, T restic.Backend] struct { +type genericBackendFactory[C any, T backend.Backend] struct { scheme string parseConfigFn func(s string) (*C, error) stripPasswordFn func(s string) string @@ -58,14 +58,14 @@ func (f *genericBackendFactory[C, T]) StripPassword(s string) string { } return s } -func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { +func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) { return f.createFn(ctx, *cfg.(*C), rt, lim) } -func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { +func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) { return f.openFn(ctx, *cfg.(*C), rt, lim) } -func NewHTTPBackendFactory[C any, T restic.Backend]( +func NewHTTPBackendFactory[C any, T backend.Backend]( scheme string, parseConfigFn func(s string) (*C, error), stripPasswordFn func(s string) string, @@ -85,7 +85,7 @@ func NewHTTPBackendFactory[C any, T restic.Backend]( } } -func NewLimitedBackendFactory[C any, T restic.Backend]( +func NewLimitedBackendFactory[C any, T backend.Backend]( scheme string, parseConfigFn func(s string) (*C, error), stripPasswordFn func(s string) string, diff --git a/internal/backend/logger/log.go b/internal/backend/logger/log.go index 6c860cfae..6fdf92295 100644 --- a/internal/backend/logger/log.go +++ b/internal/backend/logger/log.go @@ -4,18 +4,18 @@ import ( "context" "io" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" ) type Backend struct { - restic.Backend + backend.Backend } -// statically ensure that Backend implements restic.Backend. -var _ restic.Backend = &Backend{} +// statically ensure that Backend implements backend.Backend. +var _ backend.Backend = &Backend{} -func New(be restic.Backend) *Backend { +func New(be backend.Backend) *Backend { return &Backend{Backend: be} } @@ -26,7 +26,7 @@ func (be *Backend) IsNotExist(err error) bool { } // Save adds new Data to the backend. -func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { debug.Log("Save(%v, %v)", h, rd.Length()) err := be.Backend.Save(ctx, h, rd) debug.Log(" save err %v", err) @@ -34,28 +34,28 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe } // Remove deletes a file from the backend. -func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { debug.Log("Remove(%v)", h) err := be.Backend.Remove(ctx, h) debug.Log(" remove err %v", err) return err } -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error { +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(io.Reader) error) error { debug.Log("Load(%v, length %v, offset %v)", h, length, offset) err := be.Backend.Load(ctx, h, length, offset, fn) debug.Log(" load err %v", err) return err } -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { debug.Log("Stat(%v)", h) fi, err := be.Backend.Stat(ctx, h) debug.Log(" stat err %v", err) return fi, err } -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { debug.Log("List(%v)", t) err := be.Backend.List(ctx, t, fn) debug.Log(" list err %v", err) @@ -76,4 +76,4 @@ func (be *Backend) Close() error { return err } -func (be *Backend) Unwrap() restic.Backend { return be.Backend } +func (be *Backend) Unwrap() backend.Backend { return be.Backend } diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index 0bb49e0b4..2698a8275 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -10,17 +10,17 @@ import ( "sync" "github.com/cespare/xxhash/v2" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) -type memMap map[restic.Handle][]byte +type memMap map[backend.Handle][]byte // make sure that MemoryBackend implements backend.Backend -var _ restic.Backend = &MemoryBackend{} +var _ backend.Backend = &MemoryBackend{} // NewFactory creates a persistent mem backend func NewFactory() location.Factory { @@ -69,12 +69,12 @@ func (be *MemoryBackend) IsNotExist(err error) bool { } // Save adds new Data to the backend. -func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *MemoryBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { be.m.Lock() defer be.m.Unlock() h.IsMetadata = false - if h.Type == restic.ConfigFile { + if h.Type == backend.ConfigFile { h.Name = "" } @@ -112,16 +112,16 @@ func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *MemoryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *MemoryBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *MemoryBackend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { be.m.Lock() defer be.m.Unlock() h.IsMetadata = false - if h.Type == restic.ConfigFile { + if h.Type == backend.ConfigFile { h.Name = "" } @@ -143,25 +143,25 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length } // Stat returns information about a file in the backend. -func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *MemoryBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { be.m.Lock() defer be.m.Unlock() h.IsMetadata = false - if h.Type == restic.ConfigFile { + if h.Type == backend.ConfigFile { h.Name = "" } e, ok := be.data[h] if !ok { - return restic.FileInfo{}, errNotFound + return backend.FileInfo{}, errNotFound } - return restic.FileInfo{Size: int64(len(e)), Name: h.Name}, ctx.Err() + return backend.FileInfo{Size: int64(len(e)), Name: h.Name}, ctx.Err() } // Remove deletes a file from the backend. -func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error { +func (be *MemoryBackend) Remove(ctx context.Context, h backend.Handle) error { be.m.Lock() defer be.m.Unlock() @@ -176,7 +176,7 @@ func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error { } // List returns a channel which yields entries from the backend. -func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *MemoryBackend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { entries := make(map[string]int64) be.m.Lock() @@ -190,7 +190,7 @@ func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(re be.m.Unlock() for name, size := range entries { - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: name, Size: size, } diff --git a/internal/backend/mock/backend.go b/internal/backend/mock/backend.go index 875e55e71..57b1ede19 100644 --- a/internal/backend/mock/backend.go +++ b/internal/backend/mock/backend.go @@ -5,19 +5,19 @@ import ( "hash" "io" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) // Backend implements a mock backend. type Backend struct { CloseFn func() error IsNotExistFn func(err error) bool - SaveFn func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error - OpenReaderFn func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) - StatFn func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) - ListFn func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error - RemoveFn func(ctx context.Context, h restic.Handle) error + SaveFn func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error + OpenReaderFn func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) + StatFn func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) + ListFn func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error + RemoveFn func(ctx context.Context, h backend.Handle) error DeleteFn func(ctx context.Context) error ConnectionsFn func() uint LocationFn func() string @@ -84,7 +84,7 @@ func (m *Backend) IsNotExist(err error) bool { } // Save data in the backend. -func (m *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (m *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if m.SaveFn == nil { return errors.New("not implemented") } @@ -94,7 +94,7 @@ func (m *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (m *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (m *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { rd, err := m.openReader(ctx, h, length, offset) if err != nil { return err @@ -107,7 +107,7 @@ func (m *Backend) Load(ctx context.Context, h restic.Handle, length int, offset return rd.Close() } -func (m *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (m *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { if m.OpenReaderFn == nil { return nil, errors.New("not implemented") } @@ -116,16 +116,16 @@ func (m *Backend) openReader(ctx context.Context, h restic.Handle, length int, o } // Stat an object in the backend. -func (m *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (m *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { if m.StatFn == nil { - return restic.FileInfo{}, errors.New("not implemented") + return backend.FileInfo{}, errors.New("not implemented") } return m.StatFn(ctx, h) } // List items of type t. -func (m *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (m *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { if m.ListFn == nil { return nil } @@ -134,7 +134,7 @@ func (m *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi } // Remove data from the backend. -func (m *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (m *Backend) Remove(ctx context.Context, h backend.Handle) error { if m.RemoveFn == nil { return errors.New("not implemented") } @@ -152,4 +152,4 @@ func (m *Backend) Delete(ctx context.Context) error { } // Make sure that Backend implements the backend interface. -var _ restic.Backend = &Backend{} +var _ backend.Backend = &Backend{} diff --git a/internal/backend/rclone/internal_test.go b/internal/backend/rclone/internal_test.go index 32fe850a0..34d52885e 100644 --- a/internal/backend/rclone/internal_test.go +++ b/internal/backend/rclone/internal_test.go @@ -5,8 +5,8 @@ import ( "os/exec" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -32,9 +32,9 @@ func TestRcloneExit(t *testing.T) { t.Log("killed rclone") for i := 0; i < 10; i++ { - _, err = be.Stat(context.TODO(), restic.Handle{ + _, err = be.Stat(context.TODO(), backend.Handle{ Name: "foo", - Type: restic.PackFile, + Type: backend.PackFile, }) rtest.Assert(t, err != nil, "expected an error") } diff --git a/internal/backend/readerat.go b/internal/backend/readerat.go index ff2e40393..f4164cc6e 100644 --- a/internal/backend/readerat.go +++ b/internal/backend/readerat.go @@ -6,13 +6,12 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) type backendReaderAt struct { ctx context.Context - be restic.Backend - h restic.Handle + be Backend + h Handle } func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) { @@ -22,12 +21,12 @@ func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) { // ReaderAt returns an io.ReaderAt for a file in the backend. The returned reader // should not escape the caller function to avoid unexpected interactions with the // embedded context -func ReaderAt(ctx context.Context, be restic.Backend, h restic.Handle) io.ReaderAt { +func ReaderAt(ctx context.Context, be Backend, h Handle) io.ReaderAt { return backendReaderAt{ctx: ctx, be: be, h: h} } // ReadAt reads from the backend handle h at the given position. -func ReadAt(ctx context.Context, be restic.Backend, h restic.Handle, offset int64, p []byte) (n int, err error) { +func ReadAt(ctx context.Context, be Backend, h Handle, offset int64, p []byte) (n int, err error) { debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p)) err = be.Load(ctx, h, len(p), offset, func(rd io.Reader) (ierr error) { diff --git a/internal/backend/rest/config.go b/internal/backend/rest/config.go index 8458b0df2..8f17d444a 100644 --- a/internal/backend/rest/config.go +++ b/internal/backend/rest/config.go @@ -5,9 +5,9 @@ import ( "os" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to a REST server. @@ -73,7 +73,7 @@ func prepareURL(s string) string { return s } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index 4522539a6..1d1769b56 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -11,16 +11,16 @@ import ( "path" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) -// make sure the rest backend implements restic.Backend -var _ restic.Backend = &Backend{} +// make sure the rest backend implements backend.Backend +var _ backend.Backend = &Backend{} // Backend uses the REST protocol to access data stored on a server. type Backend struct { @@ -65,7 +65,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, er return nil, err } - _, err = be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) + _, err = be.Stat(ctx, backend.Handle{Type: backend.ConfigFile}) if err == nil { return nil, errors.New("config file already exists") } @@ -118,7 +118,7 @@ func (b *Backend) HasAtomicReplace() bool { } // Save stores data in the backend at the handle. -func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -157,7 +157,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea // notExistError is returned whenever the requested file does not exist on the // server. type notExistError struct { - restic.Handle + backend.Handle } func (e *notExistError) Error() string { @@ -172,7 +172,7 @@ func (b *Backend) IsNotExist(err error) bool { // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { r, err := b.openReader(ctx, h, length, offset) if err != nil { return err @@ -201,7 +201,7 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset return err } -func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { req, err := http.NewRequestWithContext(ctx, "GET", b.Filename(h), nil) if err != nil { return nil, errors.WithStack(err) @@ -238,37 +238,37 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o } // Stat returns information about a blob. -func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { req, err := http.NewRequestWithContext(ctx, http.MethodHead, b.Filename(h), nil) if err != nil { - return restic.FileInfo{}, errors.WithStack(err) + return backend.FileInfo{}, errors.WithStack(err) } req.Header.Set("Accept", ContentTypeV2) resp, err := b.client.Do(req) if err != nil { - return restic.FileInfo{}, errors.WithStack(err) + return backend.FileInfo{}, errors.WithStack(err) } _, _ = io.Copy(io.Discard, resp.Body) if err = resp.Body.Close(); err != nil { - return restic.FileInfo{}, errors.Wrap(err, "Close") + return backend.FileInfo{}, errors.Wrap(err, "Close") } if resp.StatusCode == http.StatusNotFound { _ = resp.Body.Close() - return restic.FileInfo{}, ¬ExistError{h} + return backend.FileInfo{}, ¬ExistError{h} } if resp.StatusCode != http.StatusOK { - return restic.FileInfo{}, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status) + return backend.FileInfo{}, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status) } if resp.ContentLength < 0 { - return restic.FileInfo{}, errors.New("negative content length") + return backend.FileInfo{}, errors.New("negative content length") } - bi := restic.FileInfo{ + bi := backend.FileInfo{ Size: resp.ContentLength, Name: h.Name, } @@ -277,7 +277,7 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e } // Remove removes the blob with the given name and type. -func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (b *Backend) Remove(ctx context.Context, h backend.Handle) error { req, err := http.NewRequestWithContext(ctx, "DELETE", b.Filename(h), nil) if err != nil { return errors.WithStack(err) @@ -309,8 +309,8 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - url := b.Dirname(restic.Handle{Type: t}) +func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { + url := b.Dirname(backend.Handle{Type: t}) if !strings.HasSuffix(url, "/") { url += "/" } @@ -346,7 +346,7 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi // listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET // /data/`) only returns the names of the files, so we need to issue an HTTP // HEAD request for each file. -func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error { +func (b *Backend) listv1(ctx context.Context, t backend.FileType, resp *http.Response, fn func(backend.FileInfo) error) error { debug.Log("parsing API v1 response") dec := json.NewDecoder(resp.Body) var list []string @@ -355,7 +355,7 @@ func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Resp } for _, m := range list { - fi, err := b.Stat(ctx, restic.Handle{Name: m, Type: t}) + fi, err := b.Stat(ctx, backend.Handle{Name: m, Type: t}) if err != nil { return err } @@ -380,7 +380,7 @@ func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Resp // listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET // /data/`) returns the names and sizes of all files. -func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(restic.FileInfo) error) error { +func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(backend.FileInfo) error) error { debug.Log("parsing API v2 response") dec := json.NewDecoder(resp.Body) @@ -397,7 +397,7 @@ func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(resti return ctx.Err() } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: item.Name, Size: item.Size, } diff --git a/internal/backend/rest/rest_int_test.go b/internal/backend/rest/rest_int_test.go index e7810c5e3..853a852c7 100644 --- a/internal/backend/rest/rest_int_test.go +++ b/internal/backend/rest/rest_int_test.go @@ -10,8 +10,8 @@ import ( "strconv" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/rest" - "github.com/restic/restic/internal/restic" ) func TestListAPI(t *testing.T) { @@ -22,7 +22,7 @@ func TestListAPI(t *testing.T) { Data string // response data Requests int - Result []restic.FileInfo + Result []backend.FileInfo }{ { Name: "content-type-unknown", @@ -32,7 +32,7 @@ func TestListAPI(t *testing.T) { "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b" ]`, - Result: []restic.FileInfo{ + Result: []backend.FileInfo{ {Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386}, {Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214}, {Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393}, @@ -47,7 +47,7 @@ func TestListAPI(t *testing.T) { "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b" ]`, - Result: []restic.FileInfo{ + Result: []backend.FileInfo{ {Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386}, {Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214}, {Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393}, @@ -62,7 +62,7 @@ func TestListAPI(t *testing.T) { {"name": "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "size": 1002}, {"name": "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", "size": 1003} ]`, - Result: []restic.FileInfo{ + Result: []backend.FileInfo{ {Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 1001}, {Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 1002}, {Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 1003}, @@ -122,8 +122,8 @@ func TestListAPI(t *testing.T) { t.Fatal(err) } - var list []restic.FileInfo - err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + var list []backend.FileInfo + err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { list = append(list, fi) return nil }) diff --git a/internal/backend/retry/backend_retry.go b/internal/backend/retry/backend_retry.go index b23fb41b8..c63338fb6 100644 --- a/internal/backend/retry/backend_retry.go +++ b/internal/backend/retry/backend_retry.go @@ -7,27 +7,27 @@ import ( "time" "github.com/cenkalti/backoff/v4" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" ) // Backend retries operations on the backend in case of an error with a // backoff. type Backend struct { - restic.Backend + backend.Backend MaxTries int Report func(string, error, time.Duration) Success func(string, int) } -// statically ensure that RetryBackend implements restic.Backend. -var _ restic.Backend = &Backend{} +// statically ensure that RetryBackend implements backend.Backend. +var _ backend.Backend = &Backend{} // New wraps be with a backend that retries operations after a // backoff. report is called with a description and the error, if one occurred. // success is called with the number of retries before a successful operation // (it is not called if it succeeded on the first try) -func New(be restic.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend { +func New(be backend.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend { return &Backend{ Backend: be, MaxTries: maxTries, @@ -92,7 +92,7 @@ func (be *Backend) retry(ctx context.Context, msg string, f func() error) error } // Save stores the data in the backend under the given handle. -func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { return be.retry(ctx, fmt.Sprintf("Save(%v)", h), func() error { err := rd.Rewind() if err != nil { @@ -125,7 +125,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe // given offset. If length is larger than zero, only a portion of the file // is returned. rd must be closed after use. If an error is returned, the // ReadCloser must be nil. -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) { +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) { return be.retry(ctx, fmt.Sprintf("Load(%v, %v, %v)", h, length, offset), func() error { err := be.Backend.Load(ctx, h, length, offset, consumer) @@ -137,7 +137,7 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset } // Stat returns information about the File identified by h. -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (fi backend.FileInfo, err error) { err = be.retry(ctx, fmt.Sprintf("Stat(%v)", h), func() error { var innerError error @@ -153,7 +153,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInf } // Remove removes a File with type t and name. -func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) { +func (be *Backend) Remove(ctx context.Context, h backend.Handle) (err error) { return be.retry(ctx, fmt.Sprintf("Remove(%v)", h), func() error { return be.Backend.Remove(ctx, h) }) @@ -163,7 +163,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) { // error is returned by the underlying backend, the request is retried. When fn // returns an error, the operation is aborted and the error is returned to the // caller. -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { // create a new context that we can cancel when fn returns an error, so // that listing is aborted listCtx, cancel := context.WithCancel(ctx) @@ -173,7 +173,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F var innerErr error // remember when fn returned an error, so we can return that to the caller err := be.retry(listCtx, fmt.Sprintf("List(%v)", t), func() error { - return be.Backend.List(ctx, t, func(fi restic.FileInfo) error { + return be.Backend.List(ctx, t, func(fi backend.FileInfo) error { if _, ok := listed[fi.Name]; ok { return nil } @@ -196,6 +196,6 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F return err } -func (be *Backend) Unwrap() restic.Backend { +func (be *Backend) Unwrap() backend.Backend { return be.Backend } diff --git a/internal/backend/retry/backend_retry_test.go b/internal/backend/retry/backend_retry_test.go index a24f3643a..405cdfa59 100644 --- a/internal/backend/retry/backend_retry_test.go +++ b/internal/backend/retry/backend_retry_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/cenkalti/backoff/v4" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -18,7 +19,7 @@ func TestBackendSaveRetry(t *testing.T) { buf := bytes.NewBuffer(nil) errcount := 0 be := &mock.Backend{ - SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if errcount == 0 { errcount++ _, err := io.CopyN(io.Discard, rd, 120) @@ -38,7 +39,7 @@ func TestBackendSaveRetry(t *testing.T) { retryBackend := New(be, 10, nil, nil) data := test.Random(23, 5*1024*1024+11241) - err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher())) + err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher())) if err != nil { t.Fatal(err) } @@ -56,14 +57,14 @@ func TestBackendSaveRetryAtomic(t *testing.T) { errcount := 0 calledRemove := false be := &mock.Backend{ - SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if errcount == 0 { errcount++ return errors.New("injected error") } return nil }, - RemoveFn: func(ctx context.Context, h restic.Handle) error { + RemoveFn: func(ctx context.Context, h backend.Handle) error { calledRemove = true return nil }, @@ -74,7 +75,7 @@ func TestBackendSaveRetryAtomic(t *testing.T) { retryBackend := New(be, 10, nil, nil) data := test.Random(23, 5*1024*1024+11241) - err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher())) + err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher())) if err != nil { t.Fatal(err) } @@ -91,15 +92,15 @@ func TestBackendListRetry(t *testing.T) { retry := 0 be := &mock.Backend{ - ListFn: func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { + ListFn: func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { // fail during first retry, succeed during second retry++ if retry == 1 { - _ = fn(restic.FileInfo{Name: ID1}) + _ = fn(backend.FileInfo{Name: ID1}) return errors.New("test list error") } - _ = fn(restic.FileInfo{Name: ID1}) - _ = fn(restic.FileInfo{Name: ID2}) + _ = fn(backend.FileInfo{Name: ID1}) + _ = fn(backend.FileInfo{Name: ID2}) return nil }, } @@ -108,7 +109,7 @@ func TestBackendListRetry(t *testing.T) { retryBackend := New(be, 10, nil, nil) var listed []string - err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { listed = append(listed, fi.Name) return nil }) @@ -121,10 +122,10 @@ func TestBackendListRetryErrorFn(t *testing.T) { var names = []string{"id1", "id2", "foo", "bar"} be := &mock.Backend{ - ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error { + ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error { t.Logf("List called for %v", tpe) for _, name := range names { - err := fn(restic.FileInfo{Name: name}) + err := fn(backend.FileInfo{Name: name}) if err != nil { return err } @@ -141,7 +142,7 @@ func TestBackendListRetryErrorFn(t *testing.T) { var listed []string run := 0 - err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { t.Logf("fn called for %v", fi.Name) run++ // return an error for the third item in the list @@ -172,7 +173,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) { retries := 0 be := &mock.Backend{ - ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error { + ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error { t.Logf("List called for %v, retries %v", tpe, retries) retries++ for i, name := range names { @@ -180,7 +181,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) { return ErrBackendTest } - err := fn(restic.FileInfo{Name: name}) + err := fn(backend.FileInfo{Name: name}) if err != nil { return err } @@ -195,7 +196,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) { retryBackend := New(be, maxRetries, nil, nil) var listed []string - err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { t.Logf("fn called for %v", fi.Name) listed = append(listed, fi.Name) return nil @@ -252,7 +253,7 @@ func TestBackendLoadRetry(t *testing.T) { attempt := 0 be := mock.NewBackend() - be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { // returns failing reader on first invocation, good reader on subsequent invocations attempt++ if attempt > 1 { @@ -265,7 +266,7 @@ func TestBackendLoadRetry(t *testing.T) { retryBackend := New(be, 10, nil, nil) var buf []byte - err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) { + err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) { buf, err = io.ReadAll(rd) return err }) @@ -280,7 +281,7 @@ func TestBackendLoadNotExists(t *testing.T) { attempt := 0 be := mock.NewBackend() - be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { attempt++ if attempt > 1 { t.Fail() @@ -295,7 +296,7 @@ func TestBackendLoadNotExists(t *testing.T) { TestFastRetries(t) retryBackend := New(be, 10, nil, nil) - err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) { + err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) { return nil }) test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err) @@ -308,13 +309,13 @@ func TestBackendStatNotExists(t *testing.T) { attempt := 0 be := mock.NewBackend() - be.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + be.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { attempt++ if attempt > 1 { t.Fail() - return restic.FileInfo{}, errors.New("must not retry") + return backend.FileInfo{}, errors.New("must not retry") } - return restic.FileInfo{}, notFound + return backend.FileInfo{}, notFound } be.IsNotExistFn = func(err error) bool { return errors.Is(err, notFound) @@ -323,7 +324,7 @@ func TestBackendStatNotExists(t *testing.T) { TestFastRetries(t) retryBackend := New(be, 10, nil, nil) - _, err := retryBackend.Stat(context.TODO(), restic.Handle{}) + _, err := retryBackend.Stat(context.TODO(), backend.Handle{}) test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err) test.Equals(t, 1, attempt) } @@ -337,7 +338,7 @@ func TestBackendCanceledContext(t *testing.T) { // check that we received the expected context canceled error instead TestFastRetries(t) retryBackend := New(mock.NewBackend(), 2, nil, nil) - h := restic.Handle{Type: restic.PackFile, Name: restic.NewRandomID().String()} + h := backend.Handle{Type: backend.PackFile, Name: restic.NewRandomID().String()} // create an already canceled context ctx, cancel := context.WithCancel(context.Background()) @@ -346,15 +347,15 @@ func TestBackendCanceledContext(t *testing.T) { _, err := retryBackend.Stat(ctx, h) assertIsCanceled(t, err) - err = retryBackend.Save(ctx, h, restic.NewByteReader([]byte{}, nil)) + err = retryBackend.Save(ctx, h, backend.NewByteReader([]byte{}, nil)) assertIsCanceled(t, err) err = retryBackend.Remove(ctx, h) assertIsCanceled(t, err) - err = retryBackend.Load(ctx, restic.Handle{}, 0, 0, func(rd io.Reader) (err error) { + err = retryBackend.Load(ctx, backend.Handle{}, 0, 0, func(rd io.Reader) (err error) { return nil }) assertIsCanceled(t, err) - err = retryBackend.List(ctx, restic.PackFile, func(restic.FileInfo) error { + err = retryBackend.List(ctx, backend.PackFile, func(backend.FileInfo) error { return nil }) assertIsCanceled(t, err) diff --git a/internal/restic/rewind_reader.go b/internal/backend/rewind_reader.go similarity index 99% rename from internal/restic/rewind_reader.go rename to internal/backend/rewind_reader.go index c27724e02..762b530aa 100644 --- a/internal/restic/rewind_reader.go +++ b/internal/backend/rewind_reader.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "bytes" diff --git a/internal/restic/rewind_reader_test.go b/internal/backend/rewind_reader_test.go similarity index 99% rename from internal/restic/rewind_reader_test.go rename to internal/backend/rewind_reader_test.go index 8ec79ddcd..2ee287596 100644 --- a/internal/restic/rewind_reader_test.go +++ b/internal/backend/rewind_reader_test.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "bytes" diff --git a/internal/backend/s3/config.go b/internal/backend/s3/config.go index 8dcad9eee..b4d44399f 100644 --- a/internal/backend/s3/config.go +++ b/internal/backend/s3/config.go @@ -6,9 +6,9 @@ import ( "path" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to an s3 compatible @@ -94,7 +94,7 @@ func createConfig(endpoint, bucket, prefix string, useHTTP bool) (*Config, error return &cfg, nil } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index a3c8b7764..98879d0df 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -11,12 +11,12 @@ import ( "strings" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -30,7 +30,7 @@ type Backend struct { } // make sure that *Backend implements backend.Backend -var _ restic.Backend = &Backend{} +var _ backend.Backend = &Backend{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open) @@ -127,13 +127,13 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro // Open opens the S3 backend at bucket and region. The bucket is created if it // does not exist yet. -func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { return open(ctx, cfg, rt) } // Create opens the S3 backend at bucket and region and creates the bucket if // it does not exist yet. -func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { be, err := open(ctx, cfg, rt) if err != nil { return nil, errors.Wrap(err, "open") @@ -272,7 +272,7 @@ func (be *Backend) Path() string { } // Save stores data in the backend at the handle. -func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) opts := minio.PutObjectOptions{StorageClass: be.cfg.StorageClass} @@ -294,14 +294,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { ctx, cancel := context.WithCancel(ctx) defer cancel() return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { objName := be.Filename(h) opts := minio.GetObjectOptions{} @@ -326,7 +326,7 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, } // Stat returns information about a blob. -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { objName := be.Filename(h) var obj *minio.Object @@ -334,7 +334,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf obj, err = be.client.GetObject(ctx, be.cfg.Bucket, objName, opts) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "client.GetObject") + return backend.FileInfo{}, errors.Wrap(err, "client.GetObject") } // make sure that the object is closed properly. @@ -347,14 +347,14 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf fi, err := obj.Stat() if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "Stat") + return backend.FileInfo{}, errors.Wrap(err, "Stat") } - return restic.FileInfo{Size: fi.Size, Name: h.Name}, nil + return backend.FileInfo{Size: fi.Size, Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { objName := be.Filename(h) err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{}) @@ -368,7 +368,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { prefix, recursive := be.Basedir(t) // make sure prefix ends with a slash @@ -400,7 +400,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F continue } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: path.Base(m), Size: obj.Size, } @@ -431,7 +431,7 @@ func (be *Backend) Delete(ctx context.Context) error { func (be *Backend) Close() error { return nil } // Rename moves a file based on the new layout l. -func (be *Backend) Rename(ctx context.Context, h restic.Handle, l layout.Layout) error { +func (be *Backend) Rename(ctx context.Context, h backend.Handle, l layout.Layout) error { debug.Log("Rename %v to %v", h, l) oldname := be.Filename(h) newname := l.Filename(h) diff --git a/internal/backend/s3/s3_test.go b/internal/backend/s3/s3_test.go index 3051d8ddb..470088e07 100644 --- a/internal/backend/s3/s3_test.go +++ b/internal/backend/s3/s3_test.go @@ -14,11 +14,11 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -117,7 +117,7 @@ func newMinioTestSuite(t testing.TB) (*test.Suite[s3.Config], func()) { return &cfg, nil }, - Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be restic.Backend, err error) { + Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be backend.Backend, err error) { for i := 0; i < 10; i++ { be, err = s3.Create(ctx, cfg, rt) if err != nil { diff --git a/internal/backend/sema/backend.go b/internal/backend/sema/backend.go index d60788f26..1d69c52ac 100644 --- a/internal/backend/sema/backend.go +++ b/internal/backend/sema/backend.go @@ -6,22 +6,22 @@ import ( "sync" "github.com/cenkalti/backoff/v4" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) -// make sure that connectionLimitedBackend implements restic.Backend -var _ restic.Backend = &connectionLimitedBackend{} +// make sure that connectionLimitedBackend implements backend.Backend +var _ backend.Backend = &connectionLimitedBackend{} // connectionLimitedBackend limits the number of concurrent operations. type connectionLimitedBackend struct { - restic.Backend + backend.Backend sem semaphore freezeLock sync.Mutex } // NewBackend creates a backend that limits the concurrent operations on the underlying backend -func NewBackend(be restic.Backend) restic.Backend { +func NewBackend(be backend.Backend) backend.Backend { sem, err := newSemaphore(be.Connections()) if err != nil { panic(err) @@ -35,9 +35,9 @@ func NewBackend(be restic.Backend) restic.Backend { // typeDependentLimit acquire a token unless the FileType is a lock file. The returned function // must be called to release the token. -func (be *connectionLimitedBackend) typeDependentLimit(t restic.FileType) func() { +func (be *connectionLimitedBackend) typeDependentLimit(t backend.FileType) func() { // allow concurrent lock file operations to ensure that the lock refresh is always possible - if t == restic.LockFile { + if t == backend.LockFile { return func() {} } be.sem.GetToken() @@ -59,7 +59,7 @@ func (be *connectionLimitedBackend) Unfreeze() { } // Save adds new Data to the backend. -func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *connectionLimitedBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if err := h.Valid(); err != nil { return backoff.Permanent(err) } @@ -75,7 +75,7 @@ func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, r // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *connectionLimitedBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { if err := h.Valid(); err != nil { return backoff.Permanent(err) } @@ -96,22 +96,22 @@ func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, l } // Stat returns information about a file in the backend. -func (be *connectionLimitedBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *connectionLimitedBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { if err := h.Valid(); err != nil { - return restic.FileInfo{}, backoff.Permanent(err) + return backend.FileInfo{}, backoff.Permanent(err) } defer be.typeDependentLimit(h.Type)() if ctx.Err() != nil { - return restic.FileInfo{}, ctx.Err() + return backend.FileInfo{}, ctx.Err() } return be.Backend.Stat(ctx, h) } // Remove deletes a file from the backend. -func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle) error { +func (be *connectionLimitedBackend) Remove(ctx context.Context, h backend.Handle) error { if err := h.Valid(); err != nil { return backoff.Permanent(err) } @@ -125,6 +125,6 @@ func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle) return be.Backend.Remove(ctx, h) } -func (be *connectionLimitedBackend) Unwrap() restic.Backend { +func (be *connectionLimitedBackend) Unwrap() backend.Backend { return be.Backend } diff --git a/internal/backend/sema/backend_test.go b/internal/backend/sema/backend_test.go index a1dd16187..d220f48a3 100644 --- a/internal/backend/sema/backend_test.go +++ b/internal/backend/sema/backend_test.go @@ -8,37 +8,37 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/backend/sema" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" "golang.org/x/sync/errgroup" ) func TestParameterValidationSave(t *testing.T) { m := mock.NewBackend() - m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { return nil } be := sema.NewBackend(m) - err := be.Save(context.TODO(), restic.Handle{}, nil) + err := be.Save(context.TODO(), backend.Handle{}, nil) test.Assert(t, err != nil, "Save() with invalid handle did not return an error") } func TestParameterValidationLoad(t *testing.T) { m := mock.NewBackend() - m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + m.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { return io.NopCloser(nil), nil } be := sema.NewBackend(m) nilCb := func(rd io.Reader) error { return nil } - err := be.Load(context.TODO(), restic.Handle{}, 10, 0, nilCb) + err := be.Load(context.TODO(), backend.Handle{}, 10, 0, nilCb) test.Assert(t, err != nil, "Load() with invalid handle did not return an error") - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} err = be.Load(context.TODO(), h, 10, -1, nilCb) test.Assert(t, err != nil, "Save() with negative offset did not return an error") err = be.Load(context.TODO(), h, -1, 0, nilCb) @@ -47,23 +47,23 @@ func TestParameterValidationLoad(t *testing.T) { func TestParameterValidationStat(t *testing.T) { m := mock.NewBackend() - m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - return restic.FileInfo{}, nil + m.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { + return backend.FileInfo{}, nil } be := sema.NewBackend(m) - _, err := be.Stat(context.TODO(), restic.Handle{}) + _, err := be.Stat(context.TODO(), backend.Handle{}) test.Assert(t, err != nil, "Stat() with invalid handle did not return an error") } func TestParameterValidationRemove(t *testing.T) { m := mock.NewBackend() - m.RemoveFn = func(ctx context.Context, h restic.Handle) error { + m.RemoveFn = func(ctx context.Context, h backend.Handle) error { return nil } be := sema.NewBackend(m) - err := be.Remove(context.TODO(), restic.Handle{}) + err := be.Remove(context.TODO(), backend.Handle{}) test.Assert(t, err != nil, "Remove() with invalid handle did not return an error") } @@ -71,7 +71,7 @@ func TestUnwrap(t *testing.T) { m := mock.NewBackend() be := sema.NewBackend(m) - unwrapper := be.(restic.BackendUnwrapper) + unwrapper := be.(backend.Unwrapper) test.Assert(t, unwrapper.Unwrap() == m, "Unwrap() returned wrong backend") } @@ -100,7 +100,7 @@ func countingBlocker() (func(), func(int) int) { return wait, unblock } -func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be restic.Backend) func() error, unblock func(int) int, isUnlimited bool) { +func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be backend.Backend) func() error, unblock func(int) int, isUnlimited bool) { expectBlocked := int(2) workerCount := expectBlocked + 1 @@ -125,13 +125,13 @@ func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(b func TestConcurrencyLimitSave(t *testing.T) { wait, unblock := countingBlocker() concurrencyTester(t, func(m *mock.Backend) { - m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { wait() return nil } - }, func(be restic.Backend) func() error { + }, func(be backend.Backend) func() error { return func() error { - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} return be.Save(context.TODO(), h, nil) } }, unblock, false) @@ -140,13 +140,13 @@ func TestConcurrencyLimitSave(t *testing.T) { func TestConcurrencyLimitLoad(t *testing.T) { wait, unblock := countingBlocker() concurrencyTester(t, func(m *mock.Backend) { - m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + m.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { wait() return io.NopCloser(nil), nil } - }, func(be restic.Backend) func() error { + }, func(be backend.Backend) func() error { return func() error { - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} nilCb := func(rd io.Reader) error { return nil } return be.Load(context.TODO(), h, 10, 0, nilCb) } @@ -156,13 +156,13 @@ func TestConcurrencyLimitLoad(t *testing.T) { func TestConcurrencyLimitStat(t *testing.T) { wait, unblock := countingBlocker() concurrencyTester(t, func(m *mock.Backend) { - m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + m.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { wait() - return restic.FileInfo{}, nil + return backend.FileInfo{}, nil } - }, func(be restic.Backend) func() error { + }, func(be backend.Backend) func() error { return func() error { - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} _, err := be.Stat(context.TODO(), h) return err } @@ -172,13 +172,13 @@ func TestConcurrencyLimitStat(t *testing.T) { func TestConcurrencyLimitDelete(t *testing.T) { wait, unblock := countingBlocker() concurrencyTester(t, func(m *mock.Backend) { - m.RemoveFn = func(ctx context.Context, h restic.Handle) error { + m.RemoveFn = func(ctx context.Context, h backend.Handle) error { wait() return nil } - }, func(be restic.Backend) func() error { + }, func(be backend.Backend) func() error { return func() error { - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} return be.Remove(context.TODO(), h) } }, unblock, false) @@ -187,13 +187,13 @@ func TestConcurrencyLimitDelete(t *testing.T) { func TestConcurrencyUnlimitedLockSave(t *testing.T) { wait, unblock := countingBlocker() concurrencyTester(t, func(m *mock.Backend) { - m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { wait() return nil } - }, func(be restic.Backend) func() error { + }, func(be backend.Backend) func() error { return func() error { - h := restic.Handle{Type: restic.LockFile, Name: "foobar"} + h := backend.Handle{Type: backend.LockFile, Name: "foobar"} return be.Save(context.TODO(), h, nil) } }, unblock, true) @@ -202,13 +202,13 @@ func TestConcurrencyUnlimitedLockSave(t *testing.T) { func TestFreeze(t *testing.T) { var counter int64 m := mock.NewBackend() - m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { atomic.AddInt64(&counter, 1) return nil } m.ConnectionsFn = func() uint { return 2 } be := sema.NewBackend(m) - fb := be.(restic.FreezeBackend) + fb := be.(backend.FreezeBackend) // Freeze backend fb.Freeze() @@ -218,7 +218,7 @@ func TestFreeze(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} test.OK(t, be.Save(context.TODO(), h, nil)) }() diff --git a/internal/backend/sftp/layout_test.go b/internal/backend/sftp/layout_test.go index fc8d80928..9cf24a753 100644 --- a/internal/backend/sftp/layout_test.go +++ b/internal/backend/sftp/layout_test.go @@ -6,8 +6,8 @@ import ( "path/filepath" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/sftp" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -56,7 +56,7 @@ func TestLayout(t *testing.T) { } packs := make(map[string]bool) - err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { packs[fi.Name] = false return nil }) diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index 7b46ca414..6f6a34548 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -20,7 +20,6 @@ import ( "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/cenkalti/backoff/v4" "github.com/pkg/sftp" @@ -42,7 +41,7 @@ type SFTP struct { util.Modes } -var _ restic.Backend = &SFTP{} +var _ backend.Backend = &SFTP{} func NewFactory() location.Factory { return location.NewLimitedBackendFactory("sftp", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) @@ -153,7 +152,7 @@ func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) { debug.Log("layout: %v\n", sftp.Layout) - fi, err := sftp.c.Stat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile})) + fi, err := sftp.c.Stat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile})) m := util.DeriveModesFromFileInfo(fi, err) debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) @@ -263,7 +262,7 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) { sftp.Modes = util.DefaultModes // test if config file already exists - _, err = sftp.c.Lstat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile})) + _, err = sftp.c.Lstat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile})) if err == nil { return nil, errors.New("config file already exists") } @@ -314,7 +313,7 @@ func tempSuffix() string { } // Save stores data in the backend at the handle. -func (r *SFTP) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) error { +func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) error { if err := r.clientError(); err != nil { return err } @@ -414,11 +413,11 @@ func (r *SFTP) checkNoSpace(dir string, size int64, origErr error) error { // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (r *SFTP) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (r *SFTP) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { return util.DefaultLoad(ctx, h, length, offset, r.openReader, fn) } -func (r *SFTP) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (r *SFTP) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { f, err := r.c.Open(r.Filename(h)) if err != nil { return nil, err @@ -442,21 +441,21 @@ func (r *SFTP) openReader(_ context.Context, h restic.Handle, length int, offset } // Stat returns information about a blob. -func (r *SFTP) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) { +func (r *SFTP) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) { if err := r.clientError(); err != nil { - return restic.FileInfo{}, err + return backend.FileInfo{}, err } fi, err := r.c.Lstat(r.Filename(h)) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "Lstat") + return backend.FileInfo{}, errors.Wrap(err, "Lstat") } - return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil + return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil } // Remove removes the content stored at name. -func (r *SFTP) Remove(_ context.Context, h restic.Handle) error { +func (r *SFTP) Remove(_ context.Context, h backend.Handle) error { if err := r.clientError(); err != nil { return err } @@ -466,7 +465,7 @@ func (r *SFTP) Remove(_ context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (r *SFTP) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { basedir, subdirs := r.Basedir(t) walker := r.c.Walk(basedir) for { @@ -499,7 +498,7 @@ func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileI debug.Log("send %v\n", path.Base(walker.Path())) - rfi := restic.FileInfo{ + rfi := backend.FileInfo{ Name: path.Base(walker.Path()), Size: fi.Size(), } diff --git a/internal/backend/swift/config.go b/internal/backend/swift/config.go index 5be2d9ce0..9adb80522 100644 --- a/internal/backend/swift/config.go +++ b/internal/backend/swift/config.go @@ -4,9 +4,9 @@ import ( "os" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains basic configuration needed to specify swift location for a swift server @@ -74,7 +74,7 @@ func ParseConfig(s string) (*Config, error) { return &cfg, nil } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index e3bfb7062..6943f0180 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -13,12 +13,12 @@ import ( "strings" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/ncw/swift/v2" ) @@ -32,8 +32,8 @@ type beSwift struct { layout.Layout } -// ensure statically that *beSwift implements restic.Backend. -var _ restic.Backend = &beSwift{} +// ensure statically that *beSwift implements backend.Backend. +var _ backend.Backend = &beSwift{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("swift", ParseConfig, location.NoPassword, Open, Open) @@ -41,7 +41,7 @@ func NewFactory() location.Factory { // Open opens the swift backend at a container in region. The container is // created if it does not exist yet. -func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { debug.Log("config %#v", cfg) be := &beSwift{ @@ -134,11 +134,11 @@ func (be *beSwift) HasAtomicReplace() bool { // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *beSwift) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *beSwift) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { objName := be.Filename(h) @@ -160,7 +160,7 @@ func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, } // Save stores data in the backend at the handle. -func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *beSwift) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) encoding := "binary/octet-stream" @@ -174,19 +174,19 @@ func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe } // Stat returns information about a blob. -func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { +func (be *beSwift) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { objName := be.Filename(h) obj, _, err := be.conn.Object(ctx, be.container, objName) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "conn.Object") + return backend.FileInfo{}, errors.Wrap(err, "conn.Object") } - return restic.FileInfo{Size: obj.Bytes, Name: h.Name}, nil + return backend.FileInfo{Size: obj.Bytes, Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error { +func (be *beSwift) Remove(ctx context.Context, h backend.Handle) error { objName := be.Filename(h) err := be.conn.ObjectDelete(ctx, be.container, objName) @@ -195,7 +195,7 @@ func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *beSwift) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { prefix, _ := be.Basedir(t) prefix += "/" @@ -212,7 +212,7 @@ func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.F continue } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: m, Size: obj.Bytes, } diff --git a/internal/backend/swift/swift_test.go b/internal/backend/swift/swift_test.go index 98ee5b1c1..355947cc7 100644 --- a/internal/backend/swift/swift_test.go +++ b/internal/backend/swift/swift_test.go @@ -6,9 +6,9 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/swift" "github.com/restic/restic/internal/backend/test" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -20,7 +20,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] { // wait for removals for at least 5m WaitForDelayedRemoval: 5 * time.Minute, - ErrorHandler: func(t testing.TB, be restic.Backend, err error) error { + ErrorHandler: func(t testing.TB, be backend.Backend, err error) error { if err == nil { return nil } diff --git a/internal/backend/test/benchmarks.go b/internal/backend/test/benchmarks.go index 150ef3987..e4271a386 100644 --- a/internal/backend/test/benchmarks.go +++ b/internal/backend/test/benchmarks.go @@ -6,22 +6,23 @@ import ( "io" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) -func saveRandomFile(t testing.TB, be restic.Backend, length int) ([]byte, restic.Handle) { +func saveRandomFile(t testing.TB, be backend.Backend, length int) ([]byte, backend.Handle) { data := test.Random(23, length) id := restic.Hash(data) - handle := restic.Handle{Type: restic.PackFile, Name: id.String()} - err := be.Save(context.TODO(), handle, restic.NewByteReader(data, be.Hasher())) + handle := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := be.Save(context.TODO(), handle, backend.NewByteReader(data, be.Hasher())) if err != nil { t.Fatalf("Save() error: %+v", err) } return data, handle } -func remove(t testing.TB, be restic.Backend, h restic.Handle) { +func remove(t testing.TB, be backend.Backend, h backend.Handle) { if err := be.Remove(context.TODO(), h); err != nil { t.Fatalf("Remove() returned error: %v", err) } @@ -146,9 +147,9 @@ func (s *Suite[C]) BenchmarkSave(t *testing.B) { length := 1<<24 + 2123 data := test.Random(23, length) id := restic.Hash(data) - handle := restic.Handle{Type: restic.PackFile, Name: id.String()} + handle := backend.Handle{Type: backend.PackFile, Name: id.String()} - rd := restic.NewByteReader(data, be.Hasher()) + rd := backend.NewByteReader(data, be.Hasher()) t.SetBytes(int64(length)) t.ResetTimer() diff --git a/internal/backend/test/doc.go b/internal/backend/test/doc.go index 25bdf0417..c15ed4d82 100644 --- a/internal/backend/test/doc.go +++ b/internal/backend/test/doc.go @@ -17,7 +17,7 @@ // // func newTestSuite(t testing.TB) *test.Suite { // return &test.Suite{ -// Create: func(cfg interface{}) (restic.Backend, error) { +// Create: func(cfg interface{}) (backend.Backend, error) { // [...] // }, // [...] diff --git a/internal/backend/test/suite.go b/internal/backend/test/suite.go index bb77124d7..ad8eb4c5d 100644 --- a/internal/backend/test/suite.go +++ b/internal/backend/test/suite.go @@ -11,7 +11,6 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) @@ -35,7 +34,7 @@ type Suite[C any] struct { WaitForDelayedRemoval time.Duration // ErrorHandler allows ignoring certain errors. - ErrorHandler func(testing.TB, restic.Backend, error) error + ErrorHandler func(testing.TB, backend.Backend, error) error } // RunTests executes all defined tests as subtests of t. @@ -156,7 +155,7 @@ func (s *Suite[C]) RunBenchmarks(b *testing.B) { s.cleanup(b) } -func (s *Suite[C]) createOrError() (restic.Backend, error) { +func (s *Suite[C]) createOrError() (backend.Backend, error) { tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { return nil, fmt.Errorf("cannot create transport for tests: %v", err) @@ -167,7 +166,7 @@ func (s *Suite[C]) createOrError() (restic.Backend, error) { return nil, err } - _, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile}) + _, err = be.Stat(context.TODO(), backend.Handle{Type: backend.ConfigFile}) if err != nil && !be.IsNotExist(err) { return nil, err } @@ -179,7 +178,7 @@ func (s *Suite[C]) createOrError() (restic.Backend, error) { return be, nil } -func (s *Suite[C]) create(t testing.TB) restic.Backend { +func (s *Suite[C]) create(t testing.TB) backend.Backend { be, err := s.createOrError() if err != nil { t.Fatal(err) @@ -187,7 +186,7 @@ func (s *Suite[C]) create(t testing.TB) restic.Backend { return be } -func (s *Suite[C]) open(t testing.TB) restic.Backend { +func (s *Suite[C]) open(t testing.TB) backend.Backend { tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) @@ -208,7 +207,7 @@ func (s *Suite[C]) cleanup(t testing.TB) { s.close(t, be) } -func (s *Suite[C]) close(t testing.TB, be restic.Backend) { +func (s *Suite[C]) close(t testing.TB, be backend.Backend) { err := be.Close() if err != nil { t.Fatal(err) diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index c03db79e3..414bf1c3b 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -27,7 +27,7 @@ func seedRand(t testing.TB) { t.Logf("rand initialized with seed %d", seed) } -func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, error) { +func beTest(ctx context.Context, be backend.Backend, h backend.Handle) (bool, error) { _, err := be.Stat(ctx, h) if err != nil && be.IsNotExist(err) { return false, nil @@ -49,7 +49,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) { defer s.close(t, b) // remove a config if present - cfgHandle := restic.Handle{Type: restic.ConfigFile} + cfgHandle := backend.Handle{Type: backend.ConfigFile} cfgPresent, err := beTest(context.TODO(), b, cfgHandle) if err != nil { t.Fatalf("unable to test for config: %+v", err) @@ -60,7 +60,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) { } // save a config - store(t, b, restic.ConfigFile, []byte("test config")) + store(t, b, backend.ConfigFile, []byte("test config")) // now create the backend again, this must fail _, err = s.createOrError() @@ -69,7 +69,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) { } // remove config - err = b.Remove(context.TODO(), restic.Handle{Type: restic.ConfigFile, Name: ""}) + err = b.Remove(context.TODO(), backend.Handle{Type: backend.ConfigFile, Name: ""}) if err != nil { t.Fatalf("unexpected error removing config: %+v", err) } @@ -94,13 +94,13 @@ func (s *Suite[C]) TestConfig(t *testing.T) { var testString = "Config" // create config and read it back - _, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.ConfigFile}) + _, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.ConfigFile}) if err == nil { t.Fatalf("did not get expected error for non-existing config") } test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize error from LoadAll(): %v", err) - err = b.Save(context.TODO(), restic.Handle{Type: restic.ConfigFile}, restic.NewByteReader([]byte(testString), b.Hasher())) + err = b.Save(context.TODO(), backend.Handle{Type: backend.ConfigFile}, backend.NewByteReader([]byte(testString), b.Hasher())) if err != nil { t.Fatalf("Save() error: %+v", err) } @@ -108,7 +108,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) { // try accessing the config with different names, should all return the // same config for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} { - h := restic.Handle{Type: restic.ConfigFile, Name: name} + h := backend.Handle{Type: backend.ConfigFile, Name: name} buf, err := backend.LoadAll(context.TODO(), nil, b, h) if err != nil { t.Fatalf("unable to read config with name %q: %+v", name, err) @@ -120,7 +120,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) { } // remove the config - remove(t, b, restic.Handle{Type: restic.ConfigFile}) + remove(t, b, backend.Handle{Type: backend.ConfigFile}) } // TestLoad tests the backend's Load function. @@ -130,7 +130,7 @@ func (s *Suite[C]) TestLoad(t *testing.T) { b := s.open(t) defer s.close(t, b) - err := testLoad(b, restic.Handle{Type: restic.PackFile, Name: "foobar"}) + err := testLoad(b, backend.Handle{Type: backend.PackFile, Name: "foobar"}) if err == nil { t.Fatalf("Load() did not return an error for non-existing blob") } @@ -141,8 +141,8 @@ func (s *Suite[C]) TestLoad(t *testing.T) { data := test.Random(23, length) id := restic.Hash(data) - handle := restic.Handle{Type: restic.PackFile, Name: id.String()} - err = b.Save(context.TODO(), handle, restic.NewByteReader(data, b.Hasher())) + handle := backend.Handle{Type: backend.PackFile, Name: id.String()} + err = b.Save(context.TODO(), handle, backend.NewByteReader(data, b.Hasher())) if err != nil { t.Fatalf("Save() error: %+v", err) } @@ -243,7 +243,7 @@ func (s *Suite[C]) TestList(t *testing.T) { // Check that the backend is empty to start with var found []string - err := b.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { found = append(found, fi.Name) return nil }) @@ -259,8 +259,8 @@ func (s *Suite[C]) TestList(t *testing.T) { for i := 0; i < numTestFiles; i++ { data := test.Random(rand.Int(), rand.Intn(100)+55) id := restic.Hash(data) - h := restic.Handle{Type: restic.PackFile, Name: id.String()} - err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher())) if err != nil { t.Fatal(err) } @@ -284,7 +284,7 @@ func (s *Suite[C]) TestList(t *testing.T) { s.SetListMaxItems(test.maxItems) } - err := b.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { id, err := restic.ParseID(fi.Name) if err != nil { t.Fatal(err) @@ -320,9 +320,9 @@ func (s *Suite[C]) TestList(t *testing.T) { } t.Logf("remove %d files", numTestFiles) - handles := make([]restic.Handle, 0, len(list1)) + handles := make([]backend.Handle, 0, len(list1)) for id := range list1 { - handles = append(handles, restic.Handle{Type: restic.PackFile, Name: id.String()}) + handles = append(handles, backend.Handle{Type: backend.PackFile, Name: id.String()}) } err = s.delayedRemove(t, b, handles...) @@ -340,13 +340,13 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { b := s.open(t) defer s.close(t, b) - testFiles := make([]restic.Handle, 0, numTestFiles) + testFiles := make([]backend.Handle, 0, numTestFiles) for i := 0; i < numTestFiles; i++ { data := []byte(fmt.Sprintf("random test blob %v", i)) id := restic.Hash(data) - h := restic.Handle{Type: restic.PackFile, Name: id.String()} - err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher())) if err != nil { t.Fatal(err) } @@ -358,7 +358,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { cancel() // pass in a cancelled context - err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error { t.Errorf("got FileInfo %v for cancelled context", fi) return nil }) @@ -373,7 +373,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { defer cancel() i := 0 - err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error { i++ // cancel the context on the first file if i == 1 { @@ -396,7 +396,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { defer cancel() i := 0 - err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error { // cancel the context at the last file i++ if i == numTestFiles { @@ -423,7 +423,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { i := 0 // pass in a context with a timeout - err := b.List(ctxTimeout, restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(ctxTimeout, backend.PackFile, func(fi backend.FileInfo) error { i++ // wait until the context is cancelled @@ -494,11 +494,11 @@ func (s *Suite[C]) TestSave(t *testing.T) { data := test.Random(23, length) id = sha256.Sum256(data) - h := restic.Handle{ - Type: restic.PackFile, + h := backend.Handle{ + Type: backend.PackFile, Name: id.String(), } - err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) + err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher())) test.OK(t, err) buf, err := backend.LoadAll(context.TODO(), nil, b, h) @@ -546,7 +546,7 @@ func (s *Suite[C]) TestSave(t *testing.T) { t.Fatal(err) } - h := restic.Handle{Type: restic.PackFile, Name: id.String()} + h := backend.Handle{Type: backend.PackFile, Name: id.String()} // wrap the tempfile in an errorCloser, so we can detect if the backend // closes the reader @@ -585,7 +585,7 @@ func (s *Suite[C]) TestSave(t *testing.T) { } type incompleteByteReader struct { - restic.ByteReader + backend.ByteReader } func (r *incompleteByteReader) Length() int64 { @@ -609,8 +609,8 @@ func (s *Suite[C]) TestSaveError(t *testing.T) { copy(id[:], data) // test that incomplete uploads fail - h := restic.Handle{Type: restic.PackFile, Name: id.String()} - err := b.Save(context.TODO(), h, &incompleteByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())}) + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := b.Save(context.TODO(), h, &incompleteByteReader{ByteReader: *backend.NewByteReader(data, b.Hasher())}) // try to delete possible leftovers _ = s.delayedRemove(t, b, h) if err == nil { @@ -619,7 +619,7 @@ func (s *Suite[C]) TestSaveError(t *testing.T) { } type wrongByteReader struct { - restic.ByteReader + backend.ByteReader } func (b *wrongByteReader) Hash() []byte { @@ -648,8 +648,8 @@ func (s *Suite[C]) TestSaveWrongHash(t *testing.T) { copy(id[:], data) // test that upload with hash mismatch fails - h := restic.Handle{Type: restic.PackFile, Name: id.String()} - err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())}) + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *backend.NewByteReader(data, b.Hasher())}) exists, err2 := beTest(context.TODO(), b, h) if err2 != nil { t.Fatal(err2) @@ -674,23 +674,23 @@ var testStrings = []struct { {"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"}, } -func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) restic.Handle { +func store(t testing.TB, b backend.Backend, tpe backend.FileType, data []byte) backend.Handle { id := restic.Hash(data) - h := restic.Handle{Name: id.String(), Type: tpe} - err := b.Save(context.TODO(), h, restic.NewByteReader([]byte(data), b.Hasher())) + h := backend.Handle{Name: id.String(), Type: tpe} + err := b.Save(context.TODO(), h, backend.NewByteReader([]byte(data), b.Hasher())) test.OK(t, err) return h } // testLoad loads a blob (but discards its contents). -func testLoad(b restic.Backend, h restic.Handle) error { +func testLoad(b backend.Backend, h backend.Handle) error { return b.Load(context.TODO(), h, 0, 0, func(rd io.Reader) (ierr error) { _, ierr = io.Copy(io.Discard, rd) return ierr }) } -func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error { +func (s *Suite[C]) delayedRemove(t testing.TB, be backend.Backend, handles ...backend.Handle) error { // Some backend (swift, I'm looking at you) may implement delayed // removal of data. Let's wait a bit if this happens. @@ -734,11 +734,11 @@ func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...res return nil } -func delayedList(t testing.TB, b restic.Backend, tpe restic.FileType, max int, maxwait time.Duration) restic.IDs { +func delayedList(t testing.TB, b backend.Backend, tpe backend.FileType, max int, maxwait time.Duration) restic.IDs { list := restic.NewIDSet() start := time.Now() for i := 0; i < max; i++ { - err := b.List(context.TODO(), tpe, func(fi restic.FileInfo) error { + err := b.List(context.TODO(), tpe, func(fi backend.FileInfo) error { id := restic.TestParseID(fi.Name) list.Insert(id) return nil @@ -763,9 +763,9 @@ func (s *Suite[C]) TestBackend(t *testing.T) { test.Assert(t, !b.IsNotExist(nil), "IsNotExist() recognized nil error") - for _, tpe := range []restic.FileType{ - restic.PackFile, restic.KeyFile, restic.LockFile, - restic.SnapshotFile, restic.IndexFile, + for _, tpe := range []backend.FileType{ + backend.PackFile, backend.KeyFile, backend.LockFile, + backend.SnapshotFile, backend.IndexFile, } { // detect non-existing files for _, ts := range testStrings { @@ -773,7 +773,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) { test.OK(t, err) // test if blob is already in repository - h := restic.Handle{Type: tpe, Name: id.String()} + h := backend.Handle{Type: tpe, Name: id.String()} ret, err := beTest(context.TODO(), b, h) test.OK(t, err) test.Assert(t, !ret, "blob was found to exist before creating") @@ -799,7 +799,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) { store(t, b, tpe, []byte(ts.data)) // test Load() - h := restic.Handle{Type: tpe, Name: ts.id} + h := backend.Handle{Type: tpe, Name: ts.id} buf, err := backend.LoadAll(context.TODO(), nil, b, h) test.OK(t, err) test.Equals(t, ts.data, string(buf)) @@ -823,7 +823,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) { // test adding the first file again ts := testStrings[0] - h := restic.Handle{Type: tpe, Name: ts.id} + h := backend.Handle{Type: tpe, Name: ts.id} // remove and recreate err := s.delayedRemove(t, b, h) @@ -835,7 +835,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) { test.Assert(t, !ok, "removed blob still present") // create blob - err = b.Save(context.TODO(), h, restic.NewByteReader([]byte(ts.data), b.Hasher())) + err = b.Save(context.TODO(), h, backend.NewByteReader([]byte(ts.data), b.Hasher())) test.OK(t, err) // list items @@ -859,12 +859,12 @@ func (s *Suite[C]) TestBackend(t *testing.T) { t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list) } - var handles []restic.Handle + var handles []backend.Handle for _, ts := range testStrings { id, err := restic.ParseID(ts.id) test.OK(t, err) - h := restic.Handle{Type: tpe, Name: id.String()} + h := backend.Handle{Type: tpe, Name: id.String()} found, err := beTest(context.TODO(), b, h) test.OK(t, err) diff --git a/internal/backend/util/defaults.go b/internal/backend/util/defaults.go index c43ab86b7..e5b6fc456 100644 --- a/internal/backend/util/defaults.go +++ b/internal/backend/util/defaults.go @@ -4,12 +4,12 @@ import ( "context" "io" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" ) // DefaultLoad implements Backend.Load using lower-level openReader func -func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64, - openReader func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error), +func DefaultLoad(ctx context.Context, h backend.Handle, length int, offset int64, + openReader func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error), fn func(rd io.Reader) error) error { rd, err := openReader(ctx, h, length, offset) @@ -25,23 +25,23 @@ func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64, } // DefaultDelete removes all restic keys in the bucket. It will not remove the bucket itself. -func DefaultDelete(ctx context.Context, be restic.Backend) error { - alltypes := []restic.FileType{ - restic.PackFile, - restic.KeyFile, - restic.LockFile, - restic.SnapshotFile, - restic.IndexFile} +func DefaultDelete(ctx context.Context, be backend.Backend) error { + alltypes := []backend.FileType{ + backend.PackFile, + backend.KeyFile, + backend.LockFile, + backend.SnapshotFile, + backend.IndexFile} for _, t := range alltypes { - err := be.List(ctx, t, func(fi restic.FileInfo) error { - return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) + err := be.List(ctx, t, func(fi backend.FileInfo) error { + return be.Remove(ctx, backend.Handle{Type: t, Name: fi.Name}) }) if err != nil { return nil } } - err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) + err := be.Remove(ctx, backend.Handle{Type: backend.ConfigFile}) if err != nil && be.IsNotExist(err) { err = nil } diff --git a/internal/backend/util/defaults_test.go b/internal/backend/util/defaults_test.go index c0390d0e5..1dd79208f 100644 --- a/internal/backend/util/defaults_test.go +++ b/internal/backend/util/defaults_test.go @@ -5,9 +5,9 @@ import ( "io" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -26,11 +26,11 @@ func (rd *mockReader) Close() error { func TestDefaultLoad(t *testing.T) { - h := restic.Handle{Name: "id", Type: restic.PackFile} + h := backend.Handle{Name: "id", Type: backend.PackFile} rd := &mockReader{} // happy case, assert correct parameters are passed around and content stream is closed - err := util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) { + err := util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) { rtest.Equals(t, h, ih) rtest.Equals(t, int(10), length) rtest.Equals(t, int64(11), offset) @@ -44,7 +44,7 @@ func TestDefaultLoad(t *testing.T) { rtest.Equals(t, true, rd.closed) // unhappy case, assert producer errors are handled correctly - err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) { + err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) { return nil, errors.Errorf("producer error") }, func(ird io.Reader) error { t.Fatalf("unexpected consumer invocation") @@ -54,7 +54,7 @@ func TestDefaultLoad(t *testing.T) { // unhappy case, assert consumer errors are handled correctly rd = &mockReader{} - err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) { + err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) { return rd, nil }, func(ird io.Reader) error { return errors.Errorf("consumer error") diff --git a/internal/backend/utils.go b/internal/backend/utils.go index 64af705ac..db71c070f 100644 --- a/internal/backend/utils.go +++ b/internal/backend/utils.go @@ -11,7 +11,6 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) func verifyContentMatchesName(s string, data []byte) (bool, error) { @@ -33,7 +32,7 @@ func verifyContentMatchesName(s string, data []byte) (bool, error) { // LoadAll reads all data stored in the backend for the handle into the given // buffer, which is truncated. If the buffer is not large enough or nil, a new // one is allocated. -func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle) ([]byte, error) { +func LoadAll(ctx context.Context, buf []byte, be Backend, h Handle) ([]byte, error) { retriedInvalidData := false err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error { // make sure this is idempotent, in case an error occurs this function may be called multiple times! @@ -47,7 +46,7 @@ func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle // retry loading damaged data only once. If a file fails to download correctly // the second time, then it is likely corrupted at the backend. Return the data // to the caller in that case to let it decide what to do with the data. - if !retriedInvalidData && h.Type != restic.ConfigFile { + if !retriedInvalidData && h.Type != ConfigFile { if matches, err := verifyContentMatchesName(h.Name, buf); err == nil && !matches { debug.Log("retry loading broken blob %v", h) retriedInvalidData = true @@ -77,11 +76,11 @@ func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser { } type memorizedLister struct { - fileInfos []restic.FileInfo - tpe restic.FileType + fileInfos []FileInfo + tpe FileType } -func (m *memorizedLister) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (m *memorizedLister) List(ctx context.Context, t FileType, fn func(FileInfo) error) error { if t != m.tpe { return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t) } @@ -97,13 +96,13 @@ func (m *memorizedLister) List(ctx context.Context, t restic.FileType, fn func(r return ctx.Err() } -func MemorizeList(ctx context.Context, be restic.Lister, t restic.FileType) (restic.Lister, error) { +func MemorizeList(ctx context.Context, be Lister, t FileType) (Lister, error) { if _, ok := be.(*memorizedLister); ok { return be, nil } - var fileInfos []restic.FileInfo - err := be.List(ctx, t, func(fi restic.FileInfo) error { + var fileInfos []FileInfo + err := be.List(ctx, t, func(fi FileInfo) error { fileInfos = append(fileInfos, fi) return nil }) diff --git a/internal/backend/utils_test.go b/internal/backend/utils_test.go index b3676d0b6..92afc74d9 100644 --- a/internal/backend/utils_test.go +++ b/internal/backend/utils_test.go @@ -26,11 +26,11 @@ func TestLoadAll(t *testing.T) { data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB) id := restic.Hash(data) - h := restic.Handle{Name: id.String(), Type: restic.PackFile} - err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) + h := backend.Handle{Name: id.String(), Type: backend.PackFile} + err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher())) rtest.OK(t, err) - buf, err := backend.LoadAll(context.TODO(), buf, b, restic.Handle{Type: restic.PackFile, Name: id.String()}) + buf, err := backend.LoadAll(context.TODO(), buf, b, backend.Handle{Type: backend.PackFile, Name: id.String()}) rtest.OK(t, err) if len(buf) != len(data) { @@ -45,10 +45,10 @@ func TestLoadAll(t *testing.T) { } } -func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle { +func save(t testing.TB, be backend.Backend, buf []byte) backend.Handle { id := restic.Hash(buf) - h := restic.Handle{Name: id.String(), Type: restic.PackFile} - err := be.Save(context.TODO(), h, restic.NewByteReader(buf, be.Hasher())) + h := backend.Handle{Name: id.String(), Type: backend.PackFile} + err := be.Save(context.TODO(), h, backend.NewByteReader(buf, be.Hasher())) if err != nil { t.Fatal(err) } @@ -56,10 +56,10 @@ func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle { } type quickRetryBackend struct { - restic.Backend + backend.Backend } -func (be *quickRetryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *quickRetryBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { err := be.Backend.Load(ctx, h, length, offset, fn) if err != nil { // retry @@ -76,19 +76,19 @@ func TestLoadAllBroken(t *testing.T) { // damage buffer data[0] ^= 0xff - b.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + b.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(data)), nil } // must fail on first try - _, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.PackFile, Name: id.String()}) + _, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.PackFile, Name: id.String()}) if err == nil { t.Fatalf("missing expected error") } // must return the broken data after a retry be := &quickRetryBackend{Backend: b} - buf, err := backend.LoadAll(context.TODO(), nil, be, restic.Handle{Type: restic.PackFile, Name: id.String()}) + buf, err := backend.LoadAll(context.TODO(), nil, be, backend.Handle{Type: backend.PackFile, Name: id.String()}) rtest.OK(t, err) if !bytes.Equal(buf, data) { @@ -104,7 +104,7 @@ func TestLoadAllAppend(t *testing.T) { h2 := save(t, b, randomData) var tests = []struct { - handle restic.Handle + handle backend.Handle buf []byte want []byte }{ @@ -152,11 +152,11 @@ func TestLoadAllAppend(t *testing.T) { func TestMemoizeList(t *testing.T) { // setup backend to serve as data source for memoized list be := mock.NewBackend() - files := []restic.FileInfo{ + files := []backend.FileInfo{ {Size: 42, Name: restic.NewRandomID().String()}, {Size: 45, Name: restic.NewRandomID().String()}, } - be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { + be.ListFn = func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { for _, fi := range files { if err := fn(fi); err != nil { return err @@ -165,17 +165,17 @@ func TestMemoizeList(t *testing.T) { return nil } - mem, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile) + mem, err := backend.MemorizeList(context.TODO(), be, backend.SnapshotFile) rtest.OK(t, err) - err = mem.List(context.TODO(), restic.IndexFile, func(fi restic.FileInfo) error { + err = mem.List(context.TODO(), backend.IndexFile, func(fi backend.FileInfo) error { t.Fatal("file type mismatch") return nil // the memoized lister must return an error by itself }) rtest.Assert(t, err != nil, "missing error on file typ mismatch") - var memFiles []restic.FileInfo - err = mem.List(context.TODO(), restic.SnapshotFile, func(fi restic.FileInfo) error { + var memFiles []backend.FileInfo + err = mem.List(context.TODO(), backend.SnapshotFile, func(fi backend.FileInfo) error { memFiles = append(memFiles, fi) return nil }) @@ -186,9 +186,9 @@ func TestMemoizeList(t *testing.T) { func TestMemoizeListError(t *testing.T) { // setup backend to serve as data source for memoized list be := mock.NewBackend() - be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { + be.ListFn = func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { return fmt.Errorf("list error") } - _, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile) + _, err := backend.MemorizeList(context.TODO(), be, backend.SnapshotFile) rtest.Assert(t, err != nil, "missing error on list error") } diff --git a/internal/cache/backend.go b/internal/cache/backend.go index e76bcaa1b..5cbdb5444 100644 --- a/internal/cache/backend.go +++ b/internal/cache/backend.go @@ -5,35 +5,35 @@ import ( "io" "sync" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" ) // Backend wraps a restic.Backend and adds a cache. type Backend struct { - restic.Backend + backend.Backend *Cache // inProgress contains the handle for all files that are currently // downloaded. The channel in the value is closed as soon as the download // is finished. inProgressMutex sync.Mutex - inProgress map[restic.Handle]chan struct{} + inProgress map[backend.Handle]chan struct{} } -// ensure Backend implements restic.Backend -var _ restic.Backend = &Backend{} +// ensure Backend implements backend.Backend +var _ backend.Backend = &Backend{} -func newBackend(be restic.Backend, c *Cache) *Backend { +func newBackend(be backend.Backend, c *Cache) *Backend { return &Backend{ Backend: be, Cache: c, - inProgress: make(map[restic.Handle]chan struct{}), + inProgress: make(map[backend.Handle]chan struct{}), } } // Remove deletes a file from the backend and the cache if it has been cached. -func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (b *Backend) Remove(ctx context.Context, h backend.Handle) error { debug.Log("cache Remove(%v)", h) err := b.Backend.Remove(ctx, h) if err != nil { @@ -43,18 +43,18 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { return b.Cache.remove(h) } -func autoCacheTypes(h restic.Handle) bool { +func autoCacheTypes(h backend.Handle) bool { switch h.Type { - case restic.IndexFile, restic.SnapshotFile: + case backend.IndexFile, backend.SnapshotFile: return true - case restic.PackFile: + case backend.PackFile: return h.IsMetadata } return false } // Save stores a new file in the backend and the cache. -func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if !autoCacheTypes(h) { return b.Backend.Save(ctx, h, rd) } @@ -89,7 +89,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea return nil } -func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error { +func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error { finish := make(chan struct{}) b.inProgressMutex.Lock() @@ -133,7 +133,7 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error { } // loadFromCache will try to load the file from the cache. -func (b *Backend) loadFromCache(h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) { +func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) { rd, err := b.Cache.load(h, length, offset) if err != nil { return false, err @@ -148,7 +148,7 @@ func (b *Backend) loadFromCache(h restic.Handle, length int, offset int64, consu } // Load loads a file from the cache or the backend. -func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { +func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { b.inProgressMutex.Lock() waitForFinish, inProgress := b.inProgress[h] b.inProgressMutex.Unlock() @@ -194,7 +194,7 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset // Stat tests whether the backend has a file. If it does not exist but still // exists in the cache, it is removed from the cache. -func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { debug.Log("cache Stat(%v)", h) fi, err := b.Backend.Stat(ctx, h) @@ -215,6 +215,6 @@ func (b *Backend) IsNotExist(err error) bool { return b.Backend.IsNotExist(err) } -func (b *Backend) Unwrap() restic.Backend { +func (b *Backend) Unwrap() backend.Backend { return b.Backend } diff --git a/internal/cache/backend_test.go b/internal/cache/backend_test.go index 930d853b2..68fbb02b3 100644 --- a/internal/cache/backend_test.go +++ b/internal/cache/backend_test.go @@ -16,7 +16,7 @@ import ( "github.com/restic/restic/internal/test" ) -func loadAndCompare(t testing.TB, be restic.Backend, h restic.Handle, data []byte) { +func loadAndCompare(t testing.TB, be backend.Backend, h backend.Handle, data []byte) { buf, err := backend.LoadAll(context.TODO(), nil, be, h) if err != nil { t.Fatal(err) @@ -31,25 +31,25 @@ func loadAndCompare(t testing.TB, be restic.Backend, h restic.Handle, data []byt } } -func save(t testing.TB, be restic.Backend, h restic.Handle, data []byte) { - err := be.Save(context.TODO(), h, restic.NewByteReader(data, be.Hasher())) +func save(t testing.TB, be backend.Backend, h backend.Handle, data []byte) { + err := be.Save(context.TODO(), h, backend.NewByteReader(data, be.Hasher())) if err != nil { t.Fatal(err) } } -func remove(t testing.TB, be restic.Backend, h restic.Handle) { +func remove(t testing.TB, be backend.Backend, h backend.Handle) { err := be.Remove(context.TODO(), h) if err != nil { t.Fatal(err) } } -func randomData(n int) (restic.Handle, []byte) { +func randomData(n int) (backend.Handle, []byte) { data := test.Random(rand.Int(), n) id := restic.Hash(data) - h := restic.Handle{ - Type: restic.IndexFile, + h := backend.Handle{ + Type: backend.IndexFile, Name: id.String(), } return h, data @@ -114,11 +114,11 @@ func TestBackend(t *testing.T) { } type loadErrorBackend struct { - restic.Backend + backend.Backend loadError error } -func (be loadErrorBackend) Load(_ context.Context, _ restic.Handle, _ int, _ int64, _ func(rd io.Reader) error) error { +func (be loadErrorBackend) Load(_ context.Context, _ backend.Handle, _ int, _ int64, _ func(rd io.Reader) error) error { time.Sleep(10 * time.Millisecond) return be.loadError } @@ -137,7 +137,7 @@ func TestErrorBackend(t *testing.T) { loadError: testErr, } - loadTest := func(wg *sync.WaitGroup, be restic.Backend) { + loadTest := func(wg *sync.WaitGroup, be backend.Backend) { defer wg.Done() buf, err := backend.LoadAll(context.TODO(), nil, be, h) diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 5b3601741..19b3182df 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -9,6 +9,7 @@ import ( "time" "github.com/pkg/errors" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" @@ -234,7 +235,7 @@ func IsOld(t time.Time, maxAge time.Duration) bool { } // Wrap returns a backend with a cache. -func (c *Cache) Wrap(be restic.Backend) restic.Backend { +func (c *Cache) Wrap(be backend.Backend) backend.Backend { return newBackend(be, c) } diff --git a/internal/cache/file.go b/internal/cache/file.go index c315be19f..48a38c1d3 100644 --- a/internal/cache/file.go +++ b/internal/cache/file.go @@ -14,7 +14,7 @@ import ( "github.com/restic/restic/internal/restic" ) -func (c *Cache) filename(h restic.Handle) string { +func (c *Cache) filename(h backend.Handle) string { if len(h.Name) < 2 { panic("Name is empty or too short") } @@ -22,7 +22,7 @@ func (c *Cache) filename(h restic.Handle) string { return filepath.Join(c.path, cacheLayoutPaths[h.Type], subdir, h.Name) } -func (c *Cache) canBeCached(t restic.FileType) bool { +func (c *Cache) canBeCached(t backend.FileType) bool { if c == nil { return false } @@ -34,7 +34,7 @@ func (c *Cache) canBeCached(t restic.FileType) bool { // Load returns a reader that yields the contents of the file with the // given handle. rd must be closed after use. If an error is returned, the // ReadCloser is nil. -func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (c *Cache) load(h backend.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load(%v, %v, %v) from cache", h, length, offset) if !c.canBeCached(h.Type) { return nil, errors.New("cannot be cached") @@ -78,7 +78,7 @@ func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser, } // Save saves a file in the cache. -func (c *Cache) Save(h restic.Handle, rd io.Reader) error { +func (c *Cache) Save(h backend.Handle, rd io.Reader) error { debug.Log("Save to cache: %v", h) if rd == nil { return errors.New("Save() called with nil reader") @@ -139,7 +139,7 @@ func (c *Cache) Save(h restic.Handle, rd io.Reader) error { } // Remove deletes a file. When the file is not cache, no error is returned. -func (c *Cache) remove(h restic.Handle) error { +func (c *Cache) remove(h backend.Handle) error { if !c.Has(h) { return nil } @@ -165,7 +165,7 @@ func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error { continue } - if err = fs.Remove(c.filename(restic.Handle{Type: t, Name: id.String()})); err != nil { + if err = fs.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil { return err } } @@ -207,7 +207,7 @@ func (c *Cache) list(t restic.FileType) (restic.IDSet, error) { } // Has returns true if the file is cached. -func (c *Cache) Has(h restic.Handle) bool { +func (c *Cache) Has(h backend.Handle) bool { if !c.canBeCached(h.Type) { return false } diff --git a/internal/cache/file_test.go b/internal/cache/file_test.go index e72133cd7..7935f9806 100644 --- a/internal/cache/file_test.go +++ b/internal/cache/file_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" @@ -18,12 +19,12 @@ import ( "golang.org/x/sync/errgroup" ) -func generateRandomFiles(t testing.TB, tpe restic.FileType, c *Cache) restic.IDSet { +func generateRandomFiles(t testing.TB, tpe backend.FileType, c *Cache) restic.IDSet { ids := restic.NewIDSet() for i := 0; i < rand.Intn(15)+10; i++ { buf := test.Random(rand.Int(), 1<<19) id := restic.Hash(buf) - h := restic.Handle{Type: tpe, Name: id.String()} + h := backend.Handle{Type: tpe, Name: id.String()} if c.Has(h) { t.Errorf("index %v present before save", id) @@ -46,7 +47,7 @@ func randomID(s restic.IDSet) restic.ID { panic("set is empty") } -func load(t testing.TB, c *Cache, h restic.Handle) []byte { +func load(t testing.TB, c *Cache, h backend.Handle) []byte { rd, err := c.load(h, 0, 0) if err != nil { t.Fatal(err) @@ -101,7 +102,7 @@ func TestFiles(t *testing.T) { ids := generateRandomFiles(t, tpe, c) id := randomID(ids) - h := restic.Handle{Type: tpe, Name: id.String()} + h := backend.Handle{Type: tpe, Name: id.String()} id2 := restic.Hash(load(t, c, h)) if !id.Equal(id2) { @@ -146,7 +147,7 @@ func TestFileLoad(t *testing.T) { data := test.Random(rand.Int(), 5234142) id := restic.ID{} copy(id[:], data) - h := restic.Handle{ + h := backend.Handle{ Type: restic.PackFile, Name: id.String(), } @@ -230,7 +231,7 @@ func TestFileSaveConcurrent(t *testing.T) { ) rand.Read(id[:]) - h := restic.Handle{ + h := backend.Handle{ Type: restic.PackFile, Name: id.String(), } @@ -275,7 +276,7 @@ func TestFileSaveAfterDamage(t *testing.T) { // save a few bytes of data in the cache data := test.Random(123456789, 42) id := restic.Hash(data) - h := restic.Handle{ + h := backend.Handle{ Type: restic.PackFile, Name: id.String(), } diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 59bc20daf..4f3680d63 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -39,7 +39,7 @@ type Checker struct { trackUnused bool masterIndex *index.MasterIndex - snapshots restic.Lister + snapshots backend.Lister repo restic.Repository } @@ -134,7 +134,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e if p != nil { var numIndexFiles uint64 - err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error { + err := indexList.List(ctx, restic.IndexFile, func(fi backend.FileInfo) error { _, err := restic.ParseID(fi.Name) if err != nil { debug.Log("unable to parse %v as an ID", fi.Name) @@ -245,7 +245,7 @@ func IsOrphanedPack(err error) bool { return errors.As(err, &e) && e.Orphaned } -func isS3Legacy(b restic.Backend) bool { +func isS3Legacy(b backend.Backend) bool { // unwrap cache if be, ok := b.(*cache.Backend); ok { b = be.Backend @@ -367,7 +367,7 @@ func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan restic.TreeI } } -func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.Repository) (ids restic.IDs, errs []error) { +func loadSnapshotTreeIDs(ctx context.Context, lister backend.Lister, repo restic.Repository) (ids restic.IDs, errs []error) { err := restic.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { if err != nil { errs = append(errs, err) @@ -563,7 +563,7 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r // calculate hash on-the-fly while reading the pack and capture pack header var hash restic.ID var hdrBuf []byte - hashingLoader := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + hashingLoader := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { return r.Backend().Load(ctx, h, int(size), 0, func(rd io.Reader) error { hrd := hashing.NewReader(rd, sha256.New()) bufRd.Reset(hrd) diff --git a/internal/checker/checker_test.go b/internal/checker/checker_test.go index ee7e2867c..cca5a582c 100644 --- a/internal/checker/checker_test.go +++ b/internal/checker/checker_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/restic/restic/internal/archiver" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/hashing" @@ -96,7 +97,7 @@ func TestMissingPack(t *testing.T) { repo := repository.TestOpenLocal(t, repodir) - packHandle := restic.Handle{ + packHandle := backend.Handle{ Type: restic.PackFile, Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6", } @@ -129,7 +130,7 @@ func TestUnreferencedPack(t *testing.T) { // index 3f1a only references pack 60e0 packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e" - indexHandle := restic.Handle{ + indexHandle := backend.Handle{ Type: restic.IndexFile, Name: "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44", } @@ -160,7 +161,7 @@ func TestUnreferencedBlobs(t *testing.T) { repo := repository.TestOpenLocal(t, repodir) - snapshotHandle := restic.Handle{ + snapshotHandle := backend.Handle{ Type: restic.SnapshotFile, Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02", } @@ -202,7 +203,7 @@ func TestModifiedIndex(t *testing.T) { done := make(chan struct{}) defer close(done) - h := restic.Handle{ + h := backend.Handle{ Type: restic.IndexFile, Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd", } @@ -238,7 +239,7 @@ func TestModifiedIndex(t *testing.T) { // save the index again with a modified name so that the hash doesn't match // the content any more - h2 := restic.Handle{ + h2 := backend.Handle{ Type: restic.IndexFile, Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd", } @@ -247,7 +248,7 @@ func TestModifiedIndex(t *testing.T) { if hw != nil { hash = hw.Sum(nil) } - rd, err := restic.NewFileReader(tmpfile, hash) + rd, err := backend.NewFileReader(tmpfile, hash) if err != nil { t.Fatal(err) } @@ -304,11 +305,11 @@ func TestDuplicatePacksInIndex(t *testing.T) { // errorBackend randomly modifies data after reading. type errorBackend struct { - restic.Backend + backend.Backend ProduceErrors bool } -func (b errorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { +func (b errorBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { if b.ProduceErrors { return consumer(errorReadCloser{rd}) diff --git a/internal/index/index_parallel.go b/internal/index/index_parallel.go index d505d756e..b2cdb9a09 100644 --- a/internal/index/index_parallel.go +++ b/internal/index/index_parallel.go @@ -5,13 +5,14 @@ import ( "runtime" "sync" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" ) // ForAllIndexes loads all index files in parallel and calls the given callback. // It is guaranteed that the function is not run concurrently. If the callback // returns an error, this function is cancelled and also returns that error. -func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Repository, +func ForAllIndexes(ctx context.Context, lister backend.Lister, repo restic.Repository, fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error { // decoding an index can take quite some time such that this can be both CPU- or IO-bound diff --git a/internal/index/master_index_test.go b/internal/index/master_index_test.go index bf8ec3f41..f76feb5fa 100644 --- a/internal/index/master_index_test.go +++ b/internal/index/master_index_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/index" @@ -369,7 +370,7 @@ func testIndexSave(t *testing.T, version uint) { for id := range obsoletes { t.Logf("remove index %v", id.Str()) - h := restic.Handle{Type: restic.IndexFile, Name: id.String()} + h := backend.Handle{Type: restic.IndexFile, Name: id.String()} err = repo.Backend().Remove(context.TODO(), h) if err != nil { t.Errorf("error removing index %v: %v", id, err) diff --git a/internal/migrations/s3_layout.go b/internal/migrations/s3_layout.go index 9effaee70..6b40013ee 100644 --- a/internal/migrations/s3_layout.go +++ b/internal/migrations/s3_layout.go @@ -6,6 +6,7 @@ import ( "os" "path" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/debug" @@ -23,7 +24,7 @@ type S3Layout struct{} // Check tests whether the migration can be applied. func (m *S3Layout) Check(_ context.Context, repo restic.Repository) (bool, string, error) { - be := restic.AsBackend[*s3.Backend](repo.Backend()) + be := backend.AsBackend[*s3.Backend](repo.Backend()) if be == nil { debug.Log("backend is not s3") return false, "backend is not s3", nil @@ -63,8 +64,8 @@ func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layou fmt.Fprintf(os.Stderr, "renaming file returned error: %v\n", err) } - return be.List(ctx, t, func(fi restic.FileInfo) error { - h := restic.Handle{Type: t, Name: fi.Name} + return be.List(ctx, t, func(fi backend.FileInfo) error { + h := backend.Handle{Type: t, Name: fi.Name} debug.Log("move %v", h) return retry(maxErrors, printErr, func() error { @@ -75,7 +76,7 @@ func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layou // Apply runs the migration. func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error { - be := restic.AsBackend[*s3.Backend](repo.Backend()) + be := backend.AsBackend[*s3.Backend](repo.Backend()) if be == nil { debug.Log("backend is not s3") return errors.New("backend is not s3") diff --git a/internal/migrations/upgrade_repo_v2.go b/internal/migrations/upgrade_repo_v2.go index a81abc0e3..585d9e8c7 100644 --- a/internal/migrations/upgrade_repo_v2.go +++ b/internal/migrations/upgrade_repo_v2.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" ) @@ -57,7 +58,7 @@ func (*UpgradeRepoV2) RepoCheck() bool { return true } func (*UpgradeRepoV2) upgrade(ctx context.Context, repo restic.Repository) error { - h := restic.Handle{Type: restic.ConfigFile} + h := backend.Handle{Type: backend.ConfigFile} if !repo.Backend().HasAtomicReplace() { // remove the original file for backends which do not support atomic overwriting @@ -85,7 +86,7 @@ func (m *UpgradeRepoV2) Apply(ctx context.Context, repo restic.Repository) error return fmt.Errorf("create temp dir failed: %w", err) } - h := restic.Handle{Type: restic.ConfigFile} + h := backend.Handle{Type: restic.ConfigFile} // read raw config file and save it to a temp dir, just in case var rawConfigFile []byte @@ -115,7 +116,7 @@ func (m *UpgradeRepoV2) Apply(ctx context.Context, repo restic.Repository) error // try contingency methods, reupload the original file _ = repo.Backend().Remove(ctx, h) - err = repo.Backend().Save(ctx, h, restic.NewByteReader(rawConfigFile, nil)) + err = repo.Backend().Save(ctx, h, backend.NewByteReader(rawConfigFile, nil)) if err != nil { repoError.ReuploadOldConfigError = err } diff --git a/internal/migrations/upgrade_repo_v2_test.go b/internal/migrations/upgrade_repo_v2_test.go index 96fc7788e..40153d3ca 100644 --- a/internal/migrations/upgrade_repo_v2_test.go +++ b/internal/migrations/upgrade_repo_v2_test.go @@ -7,9 +7,9 @@ import ( "sync" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) @@ -37,14 +37,14 @@ func TestUpgradeRepoV2(t *testing.T) { } type failBackend struct { - restic.Backend + backend.Backend mu sync.Mutex ConfigFileSavesUntilError uint } -func (be *failBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - if h.Type != restic.ConfigFile { +func (be *failBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { + if h.Type != backend.ConfigFile { return be.Backend.Save(ctx, h, rd) } diff --git a/internal/pack/pack_test.go b/internal/pack/pack_test.go index 3f7077390..76ff5c127 100644 --- a/internal/pack/pack_test.go +++ b/internal/pack/pack_test.go @@ -127,8 +127,8 @@ func TestUnpackReadSeeker(t *testing.T) { b := mem.New() id := restic.Hash(packData) - handle := restic.Handle{Type: restic.PackFile, Name: id.String()} - rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher()))) + handle := backend.Handle{Type: backend.PackFile, Name: id.String()} + rtest.OK(t, b.Save(context.TODO(), handle, backend.NewByteReader(packData, b.Hasher()))) verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize) } @@ -140,7 +140,7 @@ func TestShortPack(t *testing.T) { b := mem.New() id := restic.Hash(packData) - handle := restic.Handle{Type: restic.PackFile, Name: id.String()} - rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher()))) + handle := backend.Handle{Type: backend.PackFile, Name: id.String()} + rtest.OK(t, b.Save(context.TODO(), handle, backend.NewByteReader(packData, b.Hasher()))) verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize) } diff --git a/internal/repository/key.go b/internal/repository/key.go index fd20b8e5f..a207aef3d 100644 --- a/internal/repository/key.go +++ b/internal/repository/key.go @@ -178,7 +178,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int, // LoadKey loads a key from the backend. func LoadKey(ctx context.Context, s *Repository, id restic.ID) (k *Key, err error) { - h := restic.Handle{Type: restic.KeyFile, Name: id.String()} + h := backend.Handle{Type: restic.KeyFile, Name: id.String()} data, err := backend.LoadAll(ctx, nil, s.be, h) if err != nil { return nil, err @@ -270,12 +270,12 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str id := restic.Hash(buf) // store in repository and return - h := restic.Handle{ + h := backend.Handle{ Type: restic.KeyFile, Name: id.String(), } - err = s.be.Save(ctx, h, restic.NewByteReader(buf, s.be.Hasher())) + err = s.be.Save(ctx, h, backend.NewByteReader(buf, s.be.Hasher())) if err != nil { return nil, err } diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go index dbd2e8427..3b0ae12a7 100644 --- a/internal/repository/packer_manager.go +++ b/internal/repository/packer_manager.go @@ -8,6 +8,7 @@ import ( "runtime" "sync" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/hashing" "github.com/restic/restic/internal/restic" @@ -145,7 +146,7 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe // calculate sha256 hash in a second pass var rd io.Reader - rd, err = restic.NewFileReader(p.tmpfile, nil) + rd, err = backend.NewFileReader(p.tmpfile, nil) if err != nil { return err } @@ -163,12 +164,12 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe } id := restic.IDFromHash(hr.Sum(nil)) - h := restic.Handle{Type: restic.PackFile, Name: id.String(), IsMetadata: t.IsMetadata()} + h := backend.Handle{Type: backend.PackFile, Name: id.String(), IsMetadata: t.IsMetadata()} var beHash []byte if beHr != nil { beHash = beHr.Sum(nil) } - rrd, err := restic.NewFileReader(p.tmpfile, beHash) + rrd, err := backend.NewFileReader(p.tmpfile, beHash) if err != nil { return err } diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index c8570a9d4..5846a4f21 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -157,7 +158,7 @@ func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs rest } for id := range repackedBlobs { - err = repo.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}) + err = repo.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()}) if err != nil { t.Fatal(err) } @@ -191,7 +192,7 @@ func rebuildIndex(t *testing.T, repo restic.Repository) { } err = repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error { - h := restic.Handle{ + h := backend.Handle{ Type: restic.IndexFile, Name: id.String(), } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 0c270c6e0..ce2ef0da9 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -36,7 +36,7 @@ const MaxPackSize = 128 * 1024 * 1024 // Repository is used to access a repository in a backend. type Repository struct { - be restic.Backend + be backend.Backend cfg restic.Config key *crypto.Key keyID restic.ID @@ -109,7 +109,7 @@ func (c *CompressionMode) Type() string { } // New returns a new repository with backend be. -func New(be restic.Backend, opts Options) (*Repository, error) { +func New(be backend.Backend, opts Options) (*Repository, error) { if opts.Compression == CompressionInvalid { return nil, errors.New("invalid compression mode") } @@ -181,7 +181,7 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res ctx, cancel := context.WithCancel(ctx) - h := restic.Handle{Type: t, Name: id.String()} + h := backend.Handle{Type: t, Name: id.String()} retriedInvalidData := false var dataErr error wr := new(bytes.Buffer) @@ -232,7 +232,7 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res } type haver interface { - Has(restic.Handle) bool + Has(backend.Handle) bool } // sortCachedPacksFirst moves all cached pack files to the front of blobs. @@ -250,7 +250,7 @@ func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) { noncached := make([]restic.PackedBlob, 0, len(blobs)/2) for _, blob := range blobs { - if cache.Has(restic.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) { + if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) { cached = append(cached, blob) continue } @@ -284,7 +284,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. } // load blob from pack - h := restic.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: t.IsMetadata()} + h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: t.IsMetadata()} switch { case cap(buf) < int(blob.Length): @@ -494,9 +494,9 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by } else { id = restic.Hash(ciphertext) } - h := restic.Handle{Type: t, Name: id.String()} + h := backend.Handle{Type: t, Name: id.String()} - err = r.be.Save(ctx, h, restic.NewByteReader(ciphertext, r.be.Hasher())) + err = r.be.Save(ctx, h, backend.NewByteReader(ciphertext, r.be.Hasher())) if err != nil { debug.Log("error saving blob %v: %v", h, err) return restic.ID{}, err @@ -561,7 +561,7 @@ func (r *Repository) flushPacks(ctx context.Context) error { } // Backend returns the backend for the repository. -func (r *Repository) Backend() restic.Backend { +func (r *Repository) Backend() backend.Backend { return r.be } @@ -591,7 +591,7 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { if p != nil { var numIndexFiles uint64 - err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error { + err := indexList.List(ctx, restic.IndexFile, func(fi backend.FileInfo) error { _, err := restic.ParseID(fi.Name) if err != nil { debug.Log("unable to parse %v as an ID", fi.Name) @@ -773,7 +773,7 @@ func (r *Repository) Init(ctx context.Context, version uint, password string, ch return fmt.Errorf("repository version %v too low", version) } - _, err := r.be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) + _, err := r.be.Stat(ctx, backend.Handle{Type: restic.ConfigFile}) if err != nil && !r.be.IsNotExist(err) { return err } @@ -818,7 +818,7 @@ func (r *Repository) KeyID() restic.ID { // List runs fn for all files of type t in the repo. func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error { - return r.be.List(ctx, t, func(fi restic.FileInfo) error { + return r.be.List(ctx, t, func(fi backend.FileInfo) error { id, err := restic.ParseID(fi.Name) if err != nil { debug.Log("unable to parse %v as an ID", fi.Name) @@ -831,7 +831,7 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic // ListPack returns the list of blobs saved in the pack id and the length of // the the pack header. func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, uint32, error) { - h := restic.Handle{Type: restic.PackFile, Name: id.String()} + h := backend.Handle{Type: restic.PackFile, Name: id.String()} return pack.List(r.Key(), backend.ReaderAt(ctx, r.Backend(), h), size) } @@ -881,7 +881,7 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte return newID, known, size, err } -type BackendLoadFn func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error +type BackendLoadFn func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error // Skip sections with more than 4MB unused blobs const maxUnusedRange = 4 * 1024 * 1024 @@ -922,7 +922,7 @@ func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, pack } func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { - h := restic.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: false} + h := backend.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: false} dataStart := blobs[0].Offset dataEnd := blobs[len(blobs)-1].Offset + blobs[len(blobs)-1].Length diff --git a/internal/repository/repository_internal_test.go b/internal/repository/repository_internal_test.go index e5ab6e5b7..d8e35b993 100644 --- a/internal/repository/repository_internal_test.go +++ b/internal/repository/repository_internal_test.go @@ -5,13 +5,14 @@ import ( "sort" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) -type mapcache map[restic.Handle]bool +type mapcache map[backend.Handle]bool -func (c mapcache) Has(h restic.Handle) bool { return c[h] } +func (c mapcache) Has(h backend.Handle) bool { return c[h] } func TestSortCachedPacksFirst(t *testing.T) { var ( @@ -27,15 +28,15 @@ func TestSortCachedPacksFirst(t *testing.T) { blobs[i] = restic.PackedBlob{PackID: id} if i%3 == 0 { - h := restic.Handle{Name: id.String(), Type: restic.PackFile} + h := backend.Handle{Name: id.String(), Type: backend.PackFile} cache[h] = true } } copy(sorted[:], blobs[:]) sort.SliceStable(sorted[:], func(i, j int) bool { - hi := restic.Handle{Type: restic.PackFile, Name: sorted[i].PackID.String()} - hj := restic.Handle{Type: restic.PackFile, Name: sorted[j].PackID.String()} + hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID.String()} + hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID.String()} return cache.Has(hi) && !cache.Has(hj) }) @@ -58,7 +59,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { blobs[i] = restic.PackedBlob{PackID: id} if i%3 == 0 { - h := restic.Handle{Name: id.String(), Type: restic.PackFile} + h := backend.Handle{Name: id.String(), Type: backend.PackFile} cache[h] = true } } diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index d362b0b5e..5dcf21ebe 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -16,6 +16,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/klauspost/compress/zstd" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/index" @@ -278,13 +279,13 @@ func TestRepositoryLoadUnpackedBroken(t *testing.T) { data := rtest.Random(23, 12345) id := restic.Hash(data) - h := restic.Handle{Type: restic.IndexFile, Name: id.String()} + h := backend.Handle{Type: restic.IndexFile, Name: id.String()} // damage buffer data[0] ^= 0xff repo := repository.TestOpenLocal(t, repodir) // store broken file - err := repo.Backend().Save(context.TODO(), h, restic.NewByteReader(data, nil)) + err := repo.Backend().Save(context.TODO(), h, backend.NewByteReader(data, nil)) rtest.OK(t, err) // without a retry backend this will just return an error that the file is broken @@ -296,10 +297,10 @@ func TestRepositoryLoadUnpackedBroken(t *testing.T) { } type damageOnceBackend struct { - restic.Backend + backend.Backend } -func (be *damageOnceBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *damageOnceBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { // don't break the config file as we can't retry it if h.Type == restic.ConfigFile { return be.Backend.Load(ctx, h, length, offset, fn) @@ -352,7 +353,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) { rtest.OK(b, err) b.Logf("index saved as %v", id.Str()) - fi, err := repo.Backend().Stat(context.TODO(), restic.Handle{Type: restic.IndexFile, Name: id.String()}) + fi, err := repo.Backend().Stat(context.TODO(), backend.Handle{Type: restic.IndexFile, Name: id.String()}) rtest.OK(b, err) b.Logf("filesize is %v", fi.Size) @@ -528,7 +529,7 @@ func testStreamPack(t *testing.T, version uint) { packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress) loadCalls := 0 - load := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + load := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { data := packfile if offset > int64(len(data)) { diff --git a/internal/repository/testing.go b/internal/repository/testing.go index 4936cc368..d79137425 100644 --- a/internal/repository/testing.go +++ b/internal/repository/testing.go @@ -6,6 +6,7 @@ import ( "os" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/backend/retry" @@ -34,7 +35,7 @@ func TestUseLowSecurityKDFParameters(t logger) { } // TestBackend returns a fully configured in-memory backend. -func TestBackend(_ testing.TB) restic.Backend { +func TestBackend(_ testing.TB) backend.Backend { return mem.New() } @@ -43,7 +44,7 @@ const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173) // TestRepositoryWithBackend returns a repository initialized with a test // password. If be is nil, an in-memory backend is used. A constant polynomial // is used for the chunker and low-security test parameters. -func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) restic.Repository { +func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint) restic.Repository { t.Helper() TestUseLowSecurityKDFParameters(t) restic.TestDisableCheckPolynomial(t) @@ -98,7 +99,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { // TestOpenLocal opens a local repository. func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) { - var be restic.Backend + var be backend.Backend be, err := local.Open(context.TODO(), local.Config{Path: dir, Connections: 2}) if err != nil { t.Fatal(err) diff --git a/internal/restic/backend_find.go b/internal/restic/backend_find.go index 7c78b3355..7c04e168d 100644 --- a/internal/restic/backend_find.go +++ b/internal/restic/backend_find.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" ) @@ -26,13 +27,13 @@ func (e *NoIDByPrefixError) Error() string { // Find loads the list of all files of type t and searches for names which // start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned. // If more than one is found, nil and ErrMultipleIDMatches is returned. -func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error) { +func Find(ctx context.Context, be backend.Lister, t FileType, prefix string) (ID, error) { match := ID{} ctx, cancel := context.WithCancel(ctx) defer cancel() - err := be.List(ctx, t, func(fi FileInfo) error { + err := be.List(ctx, t, func(fi backend.FileInfo) error { // ignore filename which are not an id id, err := ParseID(fi.Name) if err != nil { diff --git a/internal/restic/backend_find_test.go b/internal/restic/backend_find_test.go index cbd5e7f48..833c29386 100644 --- a/internal/restic/backend_find_test.go +++ b/internal/restic/backend_find_test.go @@ -4,13 +4,15 @@ import ( "context" "strings" "testing" + + "github.com/restic/restic/internal/backend" ) type mockBackend struct { - list func(context.Context, FileType, func(FileInfo) error) error + list func(context.Context, FileType, func(backend.FileInfo) error) error } -func (m mockBackend) List(ctx context.Context, t FileType, fn func(FileInfo) error) error { +func (m mockBackend) List(ctx context.Context, t FileType, fn func(backend.FileInfo) error) error { return m.list(ctx, t, fn) } @@ -29,9 +31,9 @@ func TestFind(t *testing.T) { list := samples m := mockBackend{} - m.list = func(ctx context.Context, t FileType, fn func(FileInfo) error) error { + m.list = func(ctx context.Context, t FileType, fn func(backend.FileInfo) error) error { for _, id := range list { - err := fn(FileInfo{Name: id.String()}) + err := fn(backend.FileInfo{Name: id.String()}) if err != nil { return err } diff --git a/internal/restic/backend_test.go b/internal/restic/backend_test.go deleted file mode 100644 index a970eb5b3..000000000 --- a/internal/restic/backend_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package restic_test - -import ( - "testing" - - "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/test" -) - -type testBackend struct { - restic.Backend -} - -func (t *testBackend) Unwrap() restic.Backend { - return nil -} - -type otherTestBackend struct { - restic.Backend -} - -func (t *otherTestBackend) Unwrap() restic.Backend { - return t.Backend -} - -func TestAsBackend(t *testing.T) { - other := otherTestBackend{} - test.Assert(t, restic.AsBackend[*testBackend](other) == nil, "otherTestBackend is not a testBackend backend") - - testBe := &testBackend{} - test.Assert(t, restic.AsBackend[*testBackend](testBe) == testBe, "testBackend was not returned") - - wrapper := &otherTestBackend{Backend: testBe} - test.Assert(t, restic.AsBackend[*testBackend](wrapper) == testBe, "failed to unwrap testBackend backend") - - wrapper.Backend = other - test.Assert(t, restic.AsBackend[*testBackend](wrapper) == nil, "a wrapped otherTestBackend is not a testBackend") -} diff --git a/internal/restic/lock.go b/internal/restic/lock.go index a65ed6b5c..6b80635ac 100644 --- a/internal/restic/lock.go +++ b/internal/restic/lock.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/debug" @@ -213,7 +214,7 @@ func (l *Lock) Unlock() error { return nil } - return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: l.lockID.String()}) + return l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: l.lockID.String()}) } var StaleLockTimeout = 30 * time.Minute @@ -273,7 +274,7 @@ func (l *Lock) Refresh(ctx context.Context) error { oldLockID := l.lockID l.lockID = &id - return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: oldLockID.String()}) + return l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: oldLockID.String()}) } // RefreshStaleLock is an extended variant of Refresh that can also refresh stale lock files. @@ -302,13 +303,13 @@ func (l *Lock) RefreshStaleLock(ctx context.Context) error { exists, err = l.checkExistence(ctx) if err != nil { // cleanup replacement lock - _ = l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: id.String()}) + _ = l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: id.String()}) return err } if !exists { // cleanup replacement lock - _ = l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: id.String()}) + _ = l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: id.String()}) return ErrRemovedLock } @@ -319,7 +320,7 @@ func (l *Lock) RefreshStaleLock(ctx context.Context) error { oldLockID := l.lockID l.lockID = &id - return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: oldLockID.String()}) + return l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: oldLockID.String()}) } func (l *Lock) checkExistence(ctx context.Context) (bool, error) { @@ -328,7 +329,7 @@ func (l *Lock) checkExistence(ctx context.Context) (bool, error) { exists := false - err := l.repo.Backend().List(ctx, LockFile, func(fi FileInfo) error { + err := l.repo.Backend().List(ctx, LockFile, func(fi backend.FileInfo) error { if fi.Name == l.lockID.String() { exists = true } @@ -387,7 +388,7 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) { } if lock.Stale() { - err = repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()}) + err = repo.Backend().Remove(ctx, backend.Handle{Type: LockFile, Name: id.String()}) if err == nil { processed++ } @@ -403,7 +404,7 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) { func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) { var processed uint32 err := ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error { - err := repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()}) + err := repo.Backend().Remove(ctx, backend.Handle{Type: LockFile, Name: id.String()}) if err == nil { atomic.AddUint32(&processed, 1) } diff --git a/internal/restic/lock_test.go b/internal/restic/lock_test.go index f3c405c9c..faf3f3593 100644 --- a/internal/restic/lock_test.go +++ b/internal/restic/lock_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -53,10 +54,10 @@ func TestMultipleLock(t *testing.T) { } type failLockLoadingBackend struct { - restic.Backend + backend.Backend } -func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *failLockLoadingBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { if h.Type == restic.LockFile { return fmt.Errorf("error loading lock") } @@ -130,7 +131,7 @@ func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, er } func removeLock(repo restic.Repository, id restic.ID) error { - h := restic.Handle{Type: restic.LockFile, Name: id.String()} + h := backend.Handle{Type: restic.LockFile, Name: id.String()} return repo.Backend().Remove(context.TODO(), h) } @@ -191,7 +192,7 @@ func TestLockStale(t *testing.T) { } func lockExists(repo restic.Repository, t testing.TB, id restic.ID) bool { - h := restic.Handle{Type: restic.LockFile, Name: id.String()} + h := backend.Handle{Type: restic.LockFile, Name: id.String()} _, err := repo.Backend().Stat(context.TODO(), h) if err != nil && !repo.Backend().IsNotExist(err) { t.Fatal(err) @@ -317,7 +318,7 @@ func TestLockRefreshStaleMissing(t *testing.T) { lockID := checkSingleLock(t, repo) // refresh must fail if lock was removed - rtest.OK(t, repo.Backend().Remove(context.TODO(), restic.Handle{Type: restic.LockFile, Name: lockID.String()})) + rtest.OK(t, repo.Backend().Remove(context.TODO(), backend.Handle{Type: restic.LockFile, Name: lockID.String()})) time.Sleep(time.Millisecond) err = lock.RefreshStaleLock(context.TODO()) rtest.Assert(t, err == restic.ErrRemovedLock, "unexpected error, expected %v, got %v", restic.ErrRemovedLock, err) diff --git a/internal/restic/parallel.go b/internal/restic/parallel.go index 34a2a019c..ee525d83b 100644 --- a/internal/restic/parallel.go +++ b/internal/restic/parallel.go @@ -3,11 +3,12 @@ package restic import ( "context" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "golang.org/x/sync/errgroup" ) -func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, fn func(context.Context, ID, int64) error) error { +func ParallelList(ctx context.Context, r backend.Lister, t FileType, parallelism uint, fn func(context.Context, ID, int64) error) error { type FileIDInfo struct { ID @@ -22,7 +23,7 @@ func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, f // send list of index files through ch, which is closed afterwards wg.Go(func() error { defer close(ch) - return r.List(ctx, t, func(fi FileInfo) error { + return r.List(ctx, t, func(fi backend.FileInfo) error { id, err := ParseID(fi.Name) if err != nil { debug.Log("unable to parse %v as an ID", fi.Name) diff --git a/internal/restic/repository.go b/internal/restic/repository.go index a651f9906..772b0bdfe 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -3,6 +3,7 @@ package restic import ( "context" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/ui/progress" @@ -17,7 +18,7 @@ var ErrInvalidData = errors.New("invalid data returned") type Repository interface { // Backend returns the backend used by the repository - Backend() Backend + Backend() backend.Backend // Connections returns the maximum number of concurrent backend operations Connections() uint @@ -56,10 +57,17 @@ type Repository interface { SaveUnpacked(context.Context, FileType, []byte) (ID, error) } -// Lister allows listing files in a backend. -type Lister interface { - List(context.Context, FileType, func(FileInfo) error) error -} +type FileType = backend.FileType + +// These are the different data types a backend can store. +const ( + PackFile FileType = backend.PackFile + KeyFile FileType = backend.KeyFile + LockFile FileType = backend.LockFile + SnapshotFile FileType = backend.SnapshotFile + IndexFile FileType = backend.IndexFile + ConfigFile FileType = backend.ConfigFile +) // LoaderUnpacked allows loading a blob not stored in a pack file type LoaderUnpacked interface { diff --git a/internal/restic/snapshot.go b/internal/restic/snapshot.go index 13e795ec8..76f872068 100644 --- a/internal/restic/snapshot.go +++ b/internal/restic/snapshot.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" ) @@ -79,7 +80,7 @@ func SaveSnapshot(ctx context.Context, repo SaverUnpacked, sn *Snapshot) (ID, er // If the called function returns an error, this function is cancelled and // also returns this error. // If a snapshot ID is in excludeIDs, it will be ignored. -func ForAllSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, excludeIDs IDSet, fn func(ID, *Snapshot, error) error) error { +func ForAllSnapshots(ctx context.Context, be backend.Lister, loader LoaderUnpacked, excludeIDs IDSet, fn func(ID, *Snapshot, error) error) error { var m sync.Mutex // For most snapshots decoding is nearly for free, thus just assume were only limited by IO diff --git a/internal/restic/snapshot_find.go b/internal/restic/snapshot_find.go index cb761aee3..210040787 100644 --- a/internal/restic/snapshot_find.go +++ b/internal/restic/snapshot_find.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" ) @@ -34,7 +35,7 @@ func (f *SnapshotFilter) matches(sn *Snapshot) bool { // findLatest finds the latest snapshot with optional target/directory, // tags, hostname, and timestamp filters. -func (f *SnapshotFilter) findLatest(ctx context.Context, be Lister, loader LoaderUnpacked) (*Snapshot, error) { +func (f *SnapshotFilter) findLatest(ctx context.Context, be backend.Lister, loader LoaderUnpacked) (*Snapshot, error) { var err error absTargets := make([]string, 0, len(f.Paths)) @@ -90,7 +91,7 @@ func splitSnapshotID(s string) (id, subfolder string) { // FindSnapshot takes a string and tries to find a snapshot whose ID matches // the string as closely as possible. -func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s string) (*Snapshot, string, error) { +func FindSnapshot(ctx context.Context, be backend.Lister, loader LoaderUnpacked, s string) (*Snapshot, string, error) { s, subfolder := splitSnapshotID(s) // no need to list snapshots if `s` is already a full id @@ -108,7 +109,7 @@ func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s strin // FindLatest returns either the latest of a filtered list of all snapshots // or a snapshot specified by `snapshotID`. -func (f *SnapshotFilter) FindLatest(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, string, error) { +func (f *SnapshotFilter) FindLatest(ctx context.Context, be backend.Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, string, error) { id, subfolder := splitSnapshotID(snapshotID) if id == "latest" { sn, err := f.findLatest(ctx, be, loader) @@ -126,7 +127,7 @@ type SnapshotFindCb func(string, *Snapshot, error) error var ErrInvalidSnapshotSyntax = errors.New(": syntax not allowed") // FindAll yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. -func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotIDs []string, fn SnapshotFindCb) error { +func (f *SnapshotFilter) FindAll(ctx context.Context, be backend.Lister, loader LoaderUnpacked, snapshotIDs []string, fn SnapshotFindCb) error { if len(snapshotIDs) != 0 { var err error usedFilter := false diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index e798f2b8b..ba642ace4 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -8,6 +8,7 @@ import ( "os" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" @@ -135,7 +136,7 @@ func newTestRepo(content []TestFile) *TestRepo { files: files, filesPathToContent: filesPathToContent, } - repo.loader = func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + repo.loader = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { packID, err := restic.ParseID(h.Name) if err != nil { return err @@ -261,7 +262,7 @@ func TestErrorRestoreFiles(t *testing.T) { loadError := errors.New("load error") // loader always returns an error - repo.loader = func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + repo.loader = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { return loadError } @@ -294,7 +295,7 @@ func testPartialDownloadError(t *testing.T, part int) { // loader always returns an error loader := repo.loader - repo.loader = func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + repo.loader = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { // only load partial data to execise fault handling in different places err := loader(ctx, h, length*part/100, offset, fn) if err == nil {