octoleo
/
restic
mirror of https://github.com/octoleo/restic.git
2
2
Fork 0
Code Issues Releases Wiki Activity

move Backend interface to backend package

This commit is contained in:
Michael Eischer 2023-10-01 11:40:12 +02:00
parent ceb0774af1
commit 1b8a67fe76
105 changed files with 822 additions and 775 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/restic/cmd_backup_integration_test.go
View File

@ -9,6 +9,7 @@ import (
"runtime"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
@ -265,7 +266,7 @@ func TestBackupTreeLoadError(t *testing.T) {
// delete the subdirectory pack first
for id := range treePacks {
rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}))
rtest.OK(t, r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()}))
}
testRunRebuildIndex(t, env.gopts)
// now the repo is missing the tree blob in the index; check should report this

2
cmd/restic/cmd_cat.go
View File

@ -154,7 +154,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
return nil
case "pack":
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
h := backend.Handle{Type: restic.PackFile, Name: id.String()}
buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
if err != nil {
return err

4
cmd/restic/cmd_debug.go
View File

@ -321,7 +321,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li
panic(err)
}
be := repo.Backend()
h := restic.Handle{
h := backend.Handle{
Name: packID.String(),
Type: restic.PackFile,
}
@ -490,7 +490,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error {
Printf("examine %v\n", id)
h := restic.Handle{
h := backend.Handle{
Type: restic.PackFile,
Name: id.String(),
}

2
cmd/restic/cmd_diff.go
View File

@ -58,7 +58,7 @@ func init() {
f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
}
func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) {
func loadSnapshot(ctx context.Context, be backend.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) {
sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc)
if err != nil {
return nil, "", errors.Fatal(err.Error())

7
cmd/restic/cmd_key.go
View File

@ -7,6 +7,7 @@ import (
"strings"
"sync"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -149,7 +150,7 @@ func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) e
return errors.Fatal("refusing to remove key currently used to access repository")
}
h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
h := backend.Handle{Type: restic.KeyFile, Name: id.String()}
err := repo.Backend().Remove(ctx, h)
if err != nil {
return err
@ -176,7 +177,7 @@ func changePassword(ctx context.Context, repo *repository.Repository, gopts Glob
return err
}
h := restic.Handle{Type: restic.KeyFile, Name: oldID.String()}
h := backend.Handle{Type: restic.KeyFile, Name: oldID.String()}
err = repo.Backend().Remove(ctx, h)
if err != nil {
return err
@ -193,7 +194,7 @@ func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repos
err := repo.SearchKey(ctx, pw, 0, key.ID().String())
if err != nil {
// the key is invalid, try to remove it
h := restic.Handle{Type: restic.KeyFile, Name: key.ID().String()}
h := backend.Handle{Type: restic.KeyFile, Name: key.ID().String()}
_ = repo.Backend().Remove(ctx, h)
return errors.Fatalf("failed to access repository with new key: %v", err)
}

10
cmd/restic/cmd_key_integration_test.go
View File

@ -6,8 +6,8 @@ import (
"regexp"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -110,11 +110,11 @@ func TestKeyAddRemove(t *testing.T) {
}
type emptySaveBackend struct {
restic.Backend
backend.Backend
}
func (b *emptySaveBackend) Save(ctx context.Context, h restic.Handle, _ restic.RewindReader) error {
return b.Backend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
func (b *emptySaveBackend) Save(ctx context.Context, h backend.Handle, _ backend.RewindReader) error {
return b.Backend.Save(ctx, h, backend.NewByteReader([]byte{}, nil))
}
func TestKeyProblems(t *testing.T) {
@ -122,7 +122,7 @@ func TestKeyProblems(t *testing.T) {
defer cleanup()
testRunInit(t, env.gopts)
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return &emptySaveBackend{r}, nil
}

6
cmd/restic/cmd_prune_integration_test.go
View File

@ -6,13 +6,13 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test"
)
func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
oldHook := gopts.backendTestHook
gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil }
gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
defer func() {
gopts.backendTestHook = oldHook
}()
@ -130,7 +130,7 @@ func TestPruneWithDamagedRepository(t *testing.T) {
removePacksExcept(env.gopts, t, oldPacks, false)
oldHook := env.gopts.backendTestHook
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil }
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
defer func() {
env.gopts.backendTestHook = oldHook
}()

13
cmd/restic/cmd_repair_index_integration_test.go
View File

@ -8,6 +8,7 @@ import (
"sync"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/restic"
@ -70,12 +71,12 @@ func TestRebuildIndexAlwaysFull(t *testing.T) {
// indexErrorBackend modifies the first index after reading.
type indexErrorBackend struct {
restic.Backend
backend.Backend
lock sync.Mutex
hasErred bool
}
func (b *indexErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
func (b *indexErrorBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
// protect hasErred
b.lock.Lock()
@ -101,7 +102,7 @@ func (erd errorReadCloser) Read(p []byte) (int, error) {
}
func TestRebuildIndexDamage(t *testing.T) {
testRebuildIndex(t, func(r restic.Backend) (restic.Backend, error) {
testRebuildIndex(t, func(r backend.Backend) (backend.Backend, error) {
return &indexErrorBackend{
Backend: r,
}, nil
@ -109,11 +110,11 @@ func TestRebuildIndexDamage(t *testing.T) {
}
type appendOnlyBackend struct {
restic.Backend
backend.Backend
}
// called via repo.Backend().Remove()
func (b *appendOnlyBackend) Remove(_ context.Context, h restic.Handle) error {
func (b *appendOnlyBackend) Remove(_ context.Context, h backend.Handle) error {
return errors.Errorf("Failed to remove %v", h)
}
@ -127,7 +128,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
err := withRestoreGlobalOptions(func() error {
globalOptions.stdout = io.Discard
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return &appendOnlyBackend{r}, nil
}
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts)

3
cmd/restic/cmd_repair_packs.go
View File

@ -5,6 +5,7 @@ import (
"io"
"os"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -84,7 +85,7 @@ func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repo
return errors.Fatalf("%s", err)
}
err = repo.Backend().Load(ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error {
err = repo.Backend().Load(ctx, backend.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error {
_, err := f.Seek(0, 0)
if err != nil {
return err

4
cmd/restic/cmd_rewrite.go
View File

@ -128,7 +128,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
if dryRun {
Verbosef("would delete empty snapshot\n")
} else {
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
if err = repo.Backend().Remove(ctx, h); err != nil {
return false, err
}
@ -170,7 +170,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
Verbosef("saved new snapshot %v\n", id.Str())
if forget {
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
if err = repo.Backend().Remove(ctx, h); err != nil {
return false, err
}

3
cmd/restic/cmd_tag.go
View File

@ -5,6 +5,7 @@ import (
"github.com/spf13/cobra"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
@ -85,7 +86,7 @@ func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Sna
debug.Log("new snapshot saved as %v", id)
// Remove the old snapshot.
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
if err = repo.Backend().Remove(ctx, h); err != nil {
return false, err
}

3
cmd/restic/delete.go
View File

@ -5,6 +5,7 @@ import (
"golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
)
@ -45,7 +46,7 @@ func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, rep
for i := 0; i < int(workerCount); i++ {
wg.Go(func() error {
for id := range fileChan {
h := restic.Handle{Type: fileType, Name: id.String()}
h := backend.Handle{Type: fileType, Name: id.String()}
err := repo.Backend().Remove(ctx, h)
if err != nil {
if !gopts.JSON {

2
cmd/restic/find.go
View File

@ -29,7 +29,7 @@ func initSingleSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter)
}
// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, f *restic.SnapshotFilter, snapshotIDs []string) <-chan *restic.Snapshot {
func FindFilteredSnapshots(ctx context.Context, be backend.Lister, loader restic.LoaderUnpacked, f *restic.SnapshotFilter, snapshotIDs []string) <-chan *restic.Snapshot {
out := make(chan *restic.Snapshot)
go func() {
defer close(out)

12
cmd/restic/global.go
View File

@ -48,7 +48,7 @@ var version = "0.16.1-dev (compiled manually)"
// TimeFormat is the format used for all timestamps printed by restic.
const TimeFormat = "2006-01-02 15:04:05"
type backendWrapper func(r restic.Backend) (restic.Backend, error)
type backendWrapper func(r backend.Backend) (backend.Backend, error)
// GlobalOptions hold all global options for restic.
type GlobalOptions struct {
@ -553,7 +553,7 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi
func parseConfig(loc location.Location, opts options.Options) (interface{}, error) {
cfg := loc.Config
if cfg, ok := cfg.(restic.ApplyEnvironmenter); ok {
if cfg, ok := cfg.(backend.ApplyEnvironmenter); ok {
cfg.ApplyEnvironment("")
}
@ -568,14 +568,14 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
}
// Open the backend specified by a location config.
func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) {
func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) {
debug.Log("parsing location %v", location.StripPassword(gopts.backends, s))
loc, err := location.Parse(gopts.backends, s)
if err != nil {
return nil, errors.Fatalf("parsing repository location failed: %v", err)
}
var be restic.Backend
var be backend.Backend
cfg, err := parseConfig(loc, opts)
if err != nil {
@ -613,7 +613,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
}
// check if config is there
fi, err := be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
fi, err := be.Stat(ctx, backend.Handle{Type: restic.ConfigFile})
if err != nil {
return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(gopts.backends, s))
}
@ -626,7 +626,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
}
// Create the backend specified by URI.
func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) {
func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) {
debug.Log("parsing location %v", location.StripPassword(gopts.backends, s))
loc, err := location.Parse(gopts.backends, s)
if err != nil {

7
cmd/restic/integration_helpers_test.go
View File

@ -12,6 +12,7 @@ import (
"sync"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/retry"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
@ -205,7 +206,7 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
extended: make(options.Options),
// replace this hook with "nil" if listing a filetype more than once is necessary
backendTestHook: func(r restic.Backend) (restic.Backend, error) { return newOrderedListOnceBackend(r), nil },
backendTestHook: func(r backend.Backend) (backend.Backend, error) { return newOrderedListOnceBackend(r), nil },
// start with default set of backends
backends: globalOptions.backends,
}
@ -249,7 +250,7 @@ func removePacks(gopts GlobalOptions, t testing.TB, remove restic.IDSet) {
rtest.OK(t, err)
for id := range remove {
rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}))
rtest.OK(t, r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()}))
}
}
@ -272,7 +273,7 @@ func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, rem
if treePacks.Has(id) != removeTreePacks || keep.Has(id) {
return nil
}
return r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})
return r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()})
}))
}

19
cmd/restic/integration_test.go
View File

@ -8,6 +8,7 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
@ -41,12 +42,12 @@ func TestCheckRestoreNoLock(t *testing.T) {
// backends (like e.g. Amazon S3) as the second listing may be inconsistent to what
// is expected by the first listing + some operations.
type listOnceBackend struct {
restic.Backend
backend.Backend
listedFileType map[restic.FileType]bool
strictOrder bool
}
func newListOnceBackend(be restic.Backend) *listOnceBackend {
func newListOnceBackend(be backend.Backend) *listOnceBackend {
return &listOnceBackend{
Backend: be,
listedFileType: make(map[restic.FileType]bool),
@ -54,7 +55,7 @@ func newListOnceBackend(be restic.Backend) *listOnceBackend {
}
}
func newOrderedListOnceBackend(be restic.Backend) *listOnceBackend {
func newOrderedListOnceBackend(be backend.Backend) *listOnceBackend {
return &listOnceBackend{
Backend: be,
listedFileType: make(map[restic.FileType]bool),
@ -62,7 +63,7 @@ func newOrderedListOnceBackend(be restic.Backend) *listOnceBackend {
}
}
func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func(backend.FileInfo) error) error {
if t != restic.LockFile && be.listedFileType[t] {
return errors.Errorf("tried listing type %v the second time", t)
}
@ -77,7 +78,7 @@ func TestListOnce(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return newListOnceBackend(r), nil
}
pruneOpts := PruneOptions{MaxUnused: "0"}
@ -104,10 +105,10 @@ func (r *writeToOnly) WriteTo(w io.Writer) (int64, error) {
}
type onlyLoadWithWriteToBackend struct {
restic.Backend
backend.Backend
}
func (be *onlyLoadWithWriteToBackend) Load(ctx context.Context, h restic.Handle,
func (be *onlyLoadWithWriteToBackend) Load(ctx context.Context, h backend.Handle,
length int, offset int64, fn func(rd io.Reader) error) error {
return be.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
@ -120,7 +121,7 @@ func TestBackendLoadWriteTo(t *testing.T) {
defer cleanup()
// setup backend which only works if it's WriteTo method is correctly propagated upwards
env.gopts.backendInnerTestHook = func(r restic.Backend) (restic.Backend, error) {
env.gopts.backendInnerTestHook = func(r backend.Backend) (backend.Backend, error) {
return &onlyLoadWithWriteToBackend{Backend: r}, nil
}
@ -140,7 +141,7 @@ func TestFindListOnce(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return newListOnceBackend(r), nil
}

7
cmd/restic/lock.go
View File

@ -6,6 +6,7 @@ import (
"sync"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -131,7 +132,7 @@ type refreshLockRequest struct {
result chan bool
}
func refreshLocks(ctx context.Context, backend restic.Backend, lockInfo *lockContext, refreshed chan<- struct{}, forceRefresh <-chan refreshLockRequest) {
func refreshLocks(ctx context.Context, backend backend.Backend, lockInfo *lockContext, refreshed chan<- struct{}, forceRefresh <-chan refreshLockRequest) {
debug.Log("start")
lock := lockInfo.lock
ticker := time.NewTicker(refreshInterval)
@ -257,8 +258,8 @@ func monitorLockRefresh(ctx context.Context, lockInfo *lockContext, refreshed <-
}
}
func tryRefreshStaleLock(ctx context.Context, backend restic.Backend, lock *restic.Lock, cancel context.CancelFunc) bool {
freeze := restic.AsBackend[restic.FreezeBackend](backend)
func tryRefreshStaleLock(ctx context.Context, be backend.Backend, lock *restic.Lock, cancel context.CancelFunc) bool {
freeze := backend.AsBackend[backend.FreezeBackend](be)
if freeze != nil {
debug.Log("freezing backend")
freeze.Freeze()

19
cmd/restic/lock_test.go
View File

@ -9,6 +9,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/debug"
@ -104,11 +105,11 @@ func TestLockConflict(t *testing.T) {
}
type writeOnceBackend struct {
restic.Backend
backend.Backend
written bool
}
func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (b *writeOnceBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if b.written {
return fmt.Errorf("fail after first write")
}
@ -117,7 +118,7 @@ func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic.
}
func TestLockFailedRefresh(t *testing.T) {
repo, cleanup, env := openLockTestRepo(t, func(r restic.Backend) (restic.Backend, error) {
repo, cleanup, env := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) {
return &writeOnceBackend{Backend: r}, nil
})
defer cleanup()
@ -143,11 +144,11 @@ func TestLockFailedRefresh(t *testing.T) {
}
type loggingBackend struct {
restic.Backend
backend.Backend
t *testing.T
}
func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (b *loggingBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
b.t.Logf("save %v @ %v", h, time.Now())
err := b.Backend.Save(ctx, h, rd)
b.t.Logf("save finished %v @ %v", h, time.Now())
@ -155,7 +156,7 @@ func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re
}
func TestLockSuccessfulRefresh(t *testing.T) {
repo, cleanup, env := openLockTestRepo(t, func(r restic.Backend) (restic.Backend, error) {
repo, cleanup, env := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) {
return &loggingBackend{
Backend: r,
t: t,
@ -193,12 +194,12 @@ func TestLockSuccessfulRefresh(t *testing.T) {
}
type slowBackend struct {
restic.Backend
backend.Backend
m sync.Mutex
sleep time.Duration
}
func (b *slowBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (b *slowBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
b.m.Lock()
sleep := b.sleep
b.m.Unlock()
@ -208,7 +209,7 @@ func (b *slowBackend) Save(ctx context.Context, h restic.Handle, rd restic.Rewin
func TestLockSuccessfulStaleRefresh(t *testing.T) {
var sb *slowBackend
repo, cleanup, env := openLockTestRepo(t, func(r restic.Backend) (restic.Backend, error) {
repo, cleanup, env := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) {
sb = &slowBackend{Backend: r}
return sb, nil
})

13
internal/archiver/archiver_test.go
View File

@ -15,6 +15,7 @@ import (
"time"
"github.com/google/go-cmp/cmp"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/errors"
@ -1842,26 +1843,26 @@ func TestArchiverErrorReporting(t *testing.T) {
}
type noCancelBackend struct {
restic.Backend
backend.Backend
}
func (c *noCancelBackend) Remove(_ context.Context, h restic.Handle) error {
func (c *noCancelBackend) Remove(_ context.Context, h backend.Handle) error {
return c.Backend.Remove(context.Background(), h)
}
func (c *noCancelBackend) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) error {
func (c *noCancelBackend) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) error {
return c.Backend.Save(context.Background(), h, rd)
}
func (c *noCancelBackend) Load(_ context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (c *noCancelBackend) Load(_ context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return c.Backend.Load(context.Background(), h, length, offset, fn)
}
func (c *noCancelBackend) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) {
func (c *noCancelBackend) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) {
return c.Backend.Stat(context.Background(), h)
}
func (c *noCancelBackend) List(_ context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (c *noCancelBackend) List(_ context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
return c.Backend.List(context.Background(), t, fn)
}

26
internal/backend/azure/azure.go
View File

@ -12,12 +12,12 @@ import (
"path"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
@ -43,7 +43,7 @@ const saveLargeSize = 256 * 1024 * 1024
const defaultListMaxItems = 5000
// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}
var _ backend.Backend = &Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("azure", ParseConfig, location.NoPassword, Create, Open)
@ -197,7 +197,7 @@ func (be *Backend) Path() string {
}
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
debug.Log("InsertObject(%v, %v)", be.cfg.AccountName, objName)
@ -214,7 +214,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
return err
}
func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.RewindReader) error {
func (be *Backend) saveSmall(ctx context.Context, objName string, rd backend.RewindReader) error {
blockBlobClient := be.container.NewBlockBlobClient(objName)
// upload it as a new "block", use the base64 hash for the ID
@ -239,7 +239,7 @@ func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.Rewi
return errors.Wrap(err, "CommitBlockList")
}
func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.RewindReader) error {
func (be *Backend) saveLarge(ctx context.Context, objName string, rd backend.RewindReader) error {
blockBlobClient := be.container.NewBlockBlobClient(objName)
buf := make([]byte, 100*1024*1024)
@ -294,11 +294,11 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
objName := be.Filename(h)
blockBlobClient := be.container.NewBlobClient(objName)
@ -317,17 +317,17 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int,
}
// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
objName := be.Filename(h)
blobClient := be.container.NewBlobClient(objName)
props, err := blobClient.GetProperties(ctx, nil)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "blob.GetProperties")
return backend.FileInfo{}, errors.Wrap(err, "blob.GetProperties")
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Size: *props.ContentLength,
Name: h.Name,
}
@ -335,7 +335,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo,
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
blob := be.container.NewBlobClient(objName)
@ -350,7 +350,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, _ := be.Basedir(t)
// make sure prefix ends with a slash
@ -381,7 +381,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: path.Base(m),
Size: *item.Properties.ContentLength,
}

4
internal/backend/azure/azure_test.go
View File

@ -122,11 +122,11 @@ func TestUploadLargeFile(t *testing.T) {
data := rtest.Random(23, 300*1024*1024)
id := restic.Hash(data)
h := restic.Handle{Name: id.String(), Type: restic.PackFile}
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
t.Logf("hash of %d bytes: %v", len(data), id)
err = be.Save(ctx, h, restic.NewByteReader(data, be.Hasher()))
err = be.Save(ctx, h, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatal(err)
}

4
internal/backend/azure/config.go
View File

@ -5,9 +5,9 @@ import (
"path"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to an azure compatible
@ -57,7 +57,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {

28
internal/backend/b2/b2.go
View File

@ -9,12 +9,12 @@ import (
"sync"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/Backblaze/blazer/b2"
"github.com/Backblaze/blazer/base"
@ -34,8 +34,8 @@ type b2Backend struct {
// Billing happens in 1000 item granlarity, but we are more interested in reducing the number of network round trips
const defaultListMaxItems = 10 * 1000
// ensure statically that *b2Backend implements restic.Backend.
var _ restic.Backend = &b2Backend{}
// ensure statically that *b2Backend implements backend.Backend.
var _ backend.Backend = &b2Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("b2", ParseConfig, location.NoPassword, Create, Open)
@ -85,7 +85,7 @@ func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Clien
}
// Open opens a connection to the B2 service.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
debug.Log("cfg %#v", cfg)
ctx, cancel := context.WithCancel(ctx)
@ -118,7 +118,7 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend
// Create opens a connection to the B2 service. If the bucket does not exist yet,
// it is created.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
debug.Log("cfg %#v", cfg)
ctx, cancel := context.WithCancel(ctx)
@ -188,14 +188,14 @@ func (be *b2Backend) IsNotExist(err error) bool {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *b2Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *b2Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
name := be.Layout.Filename(h)
obj := be.bucket.Object(name)
@ -213,7 +213,7 @@ func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int
}
// Save stores data in the backend at the handle.
func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *b2Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
@ -237,18 +237,18 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind
}
// Stat returns information about a blob.
func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
func (be *b2Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
name := be.Filename(h)
obj := be.bucket.Object(name)
info, err := obj.Attrs(ctx)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Stat")
return backend.FileInfo{}, errors.Wrap(err, "Stat")
}
return restic.FileInfo{Size: info.Size, Name: h.Name}, nil
return backend.FileInfo{Size: info.Size, Name: h.Name}, nil
}
// Remove removes the blob with the given name and type.
func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
func (be *b2Backend) Remove(ctx context.Context, h backend.Handle) error {
// the retry backend will also repeat the remove method up to 10 times
for i := 0; i < 3; i++ {
obj := be.bucket.Object(be.Filename(h))
@ -284,7 +284,7 @@ func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
}
// List returns a channel that yields all names of blobs of type t.
func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *b2Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
@ -299,7 +299,7 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic
return err
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: path.Base(obj.Name()),
Size: attrs.Size,
}

4
internal/backend/b2/config.go
View File

@ -6,9 +6,9 @@ import (
"regexp"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to an b2 compatible
@ -82,7 +82,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {

11
internal/restic/backend.go → internal/backend/backend.go
View File

@ -1,4 +1,4 @@
package restic
package backend
import (
"context"
@ -70,7 +70,7 @@ type Backend interface {
Delete(ctx context.Context) error
}
type BackendUnwrapper interface {
type Unwrapper interface {
// Unwrap returns the underlying backend or nil if there is none.
Unwrap() Backend
}
@ -81,7 +81,7 @@ func AsBackend[B Backend](b Backend) B {
return be
}
if be, ok := b.(BackendUnwrapper); ok {
if be, ok := b.(Unwrapper); ok {
b = be.Unwrap()
} else {
// not the backend we're looking for
@ -110,3 +110,8 @@ type FileInfo struct {
type ApplyEnvironmenter interface {
ApplyEnvironment(prefix string)
}
// Lister allows listing files in a backend.
type Lister interface {
List(context.Context, FileType, func(FileInfo) error) error
}

38
internal/backend/backend_test.go Normal file
View File

@ -0,0 +1,38 @@
package backend_test
import (
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/test"
)
type testBackend struct {
backend.Backend
}
func (t *testBackend) Unwrap() backend.Backend {
return nil
}
type otherTestBackend struct {
backend.Backend
}
func (t *otherTestBackend) Unwrap() backend.Backend {
return t.Backend
}
func TestAsBackend(t *testing.T) {
other := otherTestBackend{}
test.Assert(t, backend.AsBackend[*testBackend](other) == nil, "otherTestBackend is not a testBackend backend")
testBe := &testBackend{}
test.Assert(t, backend.AsBackend[*testBackend](testBe) == testBe, "testBackend was not returned")
wrapper := &otherTestBackend{Backend: testBe}
test.Assert(t, backend.AsBackend[*testBackend](wrapper) == testBe, "failed to unwrap testBackend backend")
wrapper.Backend = other
test.Assert(t, backend.AsBackend[*testBackend](wrapper) == nil, "a wrapped otherTestBackend is not a testBackend")
}

20
internal/backend/dryrun/dry_backend.go
View File

@ -5,8 +5,8 @@ import (
"hash"
"io"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
// Backend passes reads through to an underlying layer and accepts writes, but
@ -15,20 +15,20 @@ import (
// the repo and does normal operations else.
// This is used for `backup --dry-run`.
type Backend struct {
b restic.Backend
b backend.Backend
}
// statically ensure that Backend implements restic.Backend.
var _ restic.Backend = &Backend{}
// statically ensure that Backend implements backend.Backend.
var _ backend.Backend = &Backend{}
func New(be restic.Backend) *Backend {
func New(be backend.Backend) *Backend {
b := &Backend{b: be}
debug.Log("created new dry backend")
return b
}
// Save adds new Data to the backend.
func (be *Backend) Save(_ context.Context, h restic.Handle, _ restic.RewindReader) error {
func (be *Backend) Save(_ context.Context, h backend.Handle, _ backend.RewindReader) error {
if err := h.Valid(); err != nil {
return err
}
@ -38,7 +38,7 @@ func (be *Backend) Save(_ context.Context, h restic.Handle, _ restic.RewindReade
}
// Remove deletes a file from the backend.
func (be *Backend) Remove(_ context.Context, _ restic.Handle) error {
func (be *Backend) Remove(_ context.Context, _ backend.Handle) error {
return nil
}
@ -72,14 +72,14 @@ func (be *Backend) IsNotExist(err error) bool {
return be.b.IsNotExist(err)
}
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
return be.b.List(ctx, t, fn)
}
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(io.Reader) error) error {
return be.b.Load(ctx, h, length, offset, fn)
}
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
return be.b.Stat(ctx, h)
}

16
internal/backend/dryrun/dry_backend_test.go
View File

@ -8,16 +8,16 @@ import (
"strings"
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/dryrun"
"github.com/restic/restic/internal/backend/mem"
)
// make sure that Backend implements backend.Backend
var _ restic.Backend = &dryrun.Backend{}
var _ backend.Backend = &dryrun.Backend{}
func newBackends() (*dryrun.Backend, restic.Backend) {
func newBackends() (*dryrun.Backend, backend.Backend) {
m := mem.New()
return dryrun.New(m), m
}
@ -30,7 +30,7 @@ func TestDry(t *testing.T) {
// won't pass. Instead, perform a series of operations over the backend, testing the state
// at each step.
steps := []struct {
be restic.Backend
be backend.Backend
op string
fname string
content string
@ -61,13 +61,13 @@ func TestDry(t *testing.T) {
for i, step := range steps {
var err error
handle := restic.Handle{Type: restic.PackFile, Name: step.fname}
handle := backend.Handle{Type: backend.PackFile, Name: step.fname}
switch step.op {
case "save":
err = step.be.Save(ctx, handle, restic.NewByteReader([]byte(step.content), step.be.Hasher()))
err = step.be.Save(ctx, handle, backend.NewByteReader([]byte(step.content), step.be.Hasher()))
case "list":
fileList := []string{}
err = step.be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
err = step.be.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
fileList = append(fileList, fi.Name)
return nil
})
@ -86,7 +86,7 @@ func TestDry(t *testing.T) {
case "remove":
err = step.be.Remove(ctx, handle)
case "stat":
var fi restic.FileInfo
var fi backend.FileInfo
fi, err = step.be.Stat(ctx, handle)
if err == nil {
fis := fmt.Sprintf("%s %d", fi.Name, fi.Size)

2
internal/restic/file.go → internal/backend/file.go
View File

@ -1,4 +1,4 @@
package restic
package backend
import (
"fmt"

2
internal/restic/file_test.go → internal/backend/file_test.go
View File

@ -1,4 +1,4 @@
package restic
package backend
import (
"testing"

4
internal/backend/gs/config.go
View File

@ -5,9 +5,9 @@ import (
"path"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to a Google Cloud Storage
@ -59,7 +59,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {

28
internal/backend/gs/gs.go
View File

@ -13,11 +13,11 @@ import (
"cloud.google.com/go/storage"
"github.com/pkg/errors"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
@ -45,8 +45,8 @@ type Backend struct {
layout.Layout
}
// Ensure that *Backend implements restic.Backend.
var _ restic.Backend = &Backend{}
// Ensure that *Backend implements backend.Backend.
var _ backend.Backend = &Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open)
@ -122,7 +122,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
}
// Open opens the gs backend at the specified bucket.
func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Open(_ context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
return open(cfg, rt)
}
@ -131,7 +131,7 @@ func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend,
//
// The service account must have the "storage.buckets.create" permission to
// create a bucket the does not yet exist.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
be, err := open(cfg, rt)
if err != nil {
return nil, errors.Wrap(err, "open")
@ -203,7 +203,7 @@ func (be *Backend) Path() string {
}
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
// Set chunk size to zero to disable resumable uploads.
@ -253,14 +253,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
if length == 0 {
// negative length indicates read till end to GCS lib
length = -1
@ -277,20 +277,20 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int,
}
// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h)
attr, err := be.bucket.Object(objName).Attrs(ctx)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "service.Objects.Get")
return backend.FileInfo{}, errors.Wrap(err, "service.Objects.Get")
}
return restic.FileInfo{Size: attr.Size, Name: h.Name}, nil
return backend.FileInfo{Size: attr.Size, Name: h.Name}, nil
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
err := be.bucket.Object(objName).Delete(ctx)
@ -304,7 +304,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, _ := be.Basedir(t)
// make sure prefix ends with a slash
@ -330,7 +330,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: path.Base(m),
Size: int64(attrs.Size),
}

11
internal/backend/layout/layout.go
View File

@ -7,6 +7,7 @@ import (
"path/filepath"
"regexp"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
@ -15,9 +16,9 @@ import (
// Layout computes paths for file name storage.
type Layout interface {
Filename(restic.Handle) string
Dirname(restic.Handle) string
Basedir(restic.FileType) (dir string, subdirs bool)
Filename(backend.Handle) string
Dirname(backend.Handle) string
Basedir(backend.FileType) (dir string, subdirs bool)
Paths() []string
Name() string
}
@ -102,13 +103,13 @@ func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, err
}
// key file in the "keys" dir (DefaultLayout)
foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile]))
foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[backend.KeyFile]))
if err != nil {
return nil, err
}
// key file in the "key" dir (S3LegacyLayout)
foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile]))
foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[backend.KeyFile]))
if err != nil {
return nil, err
}

28
internal/backend/layout/layout_default.go
View File

@ -3,7 +3,7 @@ package layout
import (
"encoding/hex"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
)
// DefaultLayout implements the default layout for local and sftp backends, as
@ -15,12 +15,12 @@ type DefaultLayout struct {
Join func(...string) string
}
var defaultLayoutPaths = map[restic.FileType]string{
restic.PackFile: "data",
restic.SnapshotFile: "snapshots",
restic.IndexFile: "index",
restic.LockFile: "locks",
restic.KeyFile: "keys",
var defaultLayoutPaths = map[backend.FileType]string{
backend.PackFile: "data",
backend.SnapshotFile: "snapshots",
backend.IndexFile: "index",
backend.LockFile: "locks",
backend.KeyFile: "keys",
}
func (l *DefaultLayout) String() string {
@ -33,10 +33,10 @@ func (l *DefaultLayout) Name() string {
}
// Dirname returns the directory path for a given file type and name.
func (l *DefaultLayout) Dirname(h restic.Handle) string {
func (l *DefaultLayout) Dirname(h backend.Handle) string {
p := defaultLayoutPaths[h.Type]
if h.Type == restic.PackFile && len(h.Name) > 2 {
if h.Type == backend.PackFile && len(h.Name) > 2 {
p = l.Join(p, h.Name[:2]) + "/"
}
@ -44,9 +44,9 @@ func (l *DefaultLayout) Dirname(h restic.Handle) string {
}
// Filename returns a path to a file, including its name.
func (l *DefaultLayout) Filename(h restic.Handle) string {
func (l *DefaultLayout) Filename(h backend.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
if h.Type == backend.ConfigFile {
return l.Join(l.Path, "config")
}
@ -62,15 +62,15 @@ func (l *DefaultLayout) Paths() (dirs []string) {
// also add subdirs
for i := 0; i < 256; i++ {
subdir := hex.EncodeToString([]byte{byte(i)})
dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[restic.PackFile], subdir))
dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[backend.PackFile], subdir))
}
return dirs
}
// Basedir returns the base dir name for type t.
func (l *DefaultLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) {
if t == restic.PackFile {
func (l *DefaultLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) {
if t == backend.PackFile {
subdirs = true
}

14
internal/backend/layout/layout_rest.go
View File

@ -1,6 +1,8 @@
package layout
import "github.com/restic/restic/internal/restic"
import (
"github.com/restic/restic/internal/backend"
)
// RESTLayout implements the default layout for the REST protocol.
type RESTLayout struct {
@ -21,8 +23,8 @@ func (l *RESTLayout) Name() string {
}
// Dirname returns the directory path for a given file type and name.
func (l *RESTLayout) Dirname(h restic.Handle) string {
if h.Type == restic.ConfigFile {
func (l *RESTLayout) Dirname(h backend.Handle) string {
if h.Type == backend.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
@ -30,10 +32,10 @@ func (l *RESTLayout) Dirname(h restic.Handle) string {
}
// Filename returns a path to a file, including its name.
func (l *RESTLayout) Filename(h restic.Handle) string {
func (l *RESTLayout) Filename(h backend.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
if h.Type == backend.ConfigFile {
name = "config"
}
@ -49,6 +51,6 @@ func (l *RESTLayout) Paths() (dirs []string) {
}
// Basedir returns the base dir name for files of type t.
func (l *RESTLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) {
func (l *RESTLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) {
return l.URL + l.Join(l.Path, restLayoutPaths[t]), false
}

26
internal/backend/layout/layout_s3legacy.go
View File

@ -1,6 +1,8 @@
package layout
import "github.com/restic/restic/internal/restic"
import (
"github.com/restic/restic/internal/backend"
)
// S3LegacyLayout implements the old layout used for s3 cloud storage backends, as
// described in the Design document.
@ -10,12 +12,12 @@ type S3LegacyLayout struct {
Join func(...string) string
}
var s3LayoutPaths = map[restic.FileType]string{
restic.PackFile: "data",
restic.SnapshotFile: "snapshot",
restic.IndexFile: "index",
restic.LockFile: "lock",
restic.KeyFile: "key",
var s3LayoutPaths = map[backend.FileType]string{
backend.PackFile: "data",
backend.SnapshotFile: "snapshot",
backend.IndexFile: "index",
backend.LockFile: "lock",
backend.KeyFile: "key",
}
func (l *S3LegacyLayout) String() string {
@ -44,8 +46,8 @@ func (l *S3LegacyLayout) join(url string, items ...string) string {
}
// Dirname returns the directory path for a given file type and name.
func (l *S3LegacyLayout) Dirname(h restic.Handle) string {
if h.Type == restic.ConfigFile {
func (l *S3LegacyLayout) Dirname(h backend.Handle) string {
if h.Type == backend.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
@ -53,10 +55,10 @@ func (l *S3LegacyLayout) Dirname(h restic.Handle) string {
}
// Filename returns a path to a file, including its name.
func (l *S3LegacyLayout) Filename(h restic.Handle) string {
func (l *S3LegacyLayout) Filename(h backend.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
if h.Type == backend.ConfigFile {
name = "config"
}
@ -72,6 +74,6 @@ func (l *S3LegacyLayout) Paths() (dirs []string) {
}
// Basedir returns the base dir name for type t.
func (l *S3LegacyLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) {
func (l *S3LegacyLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) {
return l.Join(l.Path, s3LayoutPaths[t]), false
}

80
internal/backend/layout/layout_test.go
View File

@ -9,7 +9,7 @@ import (
"sort"
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test"
)
@ -19,79 +19,79 @@ func TestDefaultLayout(t *testing.T) {
var tests = []struct {
path string
join func(...string) string
restic.Handle
backend.Handle
filename string
}{
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.PackFile, Name: "0123456"},
backend.Handle{Type: backend.PackFile, Name: "0123456"},
filepath.Join(tempdir, "data", "01", "0123456"),
},
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
filepath.Join(tempdir, "config"),
},
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
filepath.Join(tempdir, "snapshots", "123456"),
},
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.IndexFile, Name: "123456"},
backend.Handle{Type: backend.IndexFile, Name: "123456"},
filepath.Join(tempdir, "index", "123456"),
},
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.LockFile, Name: "123456"},
backend.Handle{Type: backend.LockFile, Name: "123456"},
filepath.Join(tempdir, "locks", "123456"),
},
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.KeyFile, Name: "123456"},
backend.Handle{Type: backend.KeyFile, Name: "123456"},
filepath.Join(tempdir, "keys", "123456"),
},
{
"",
path.Join,
restic.Handle{Type: restic.PackFile, Name: "0123456"},
backend.Handle{Type: backend.PackFile, Name: "0123456"},
"data/01/0123456",
},
{
"",
path.Join,
restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
"config",
},
{
"",
path.Join,
restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
"snapshots/123456",
},
{
"",
path.Join,
restic.Handle{Type: restic.IndexFile, Name: "123456"},
backend.Handle{Type: backend.IndexFile, Name: "123456"},
"index/123456",
},
{
"",
path.Join,
restic.Handle{Type: restic.LockFile, Name: "123456"},
backend.Handle{Type: backend.LockFile, Name: "123456"},
"locks/123456",
},
{
"",
path.Join,
restic.Handle{Type: restic.KeyFile, Name: "123456"},
backend.Handle{Type: backend.KeyFile, Name: "123456"},
"keys/123456",
},
}
@ -143,31 +143,31 @@ func TestRESTLayout(t *testing.T) {
path := rtest.TempDir(t)
var tests = []struct {
restic.Handle
backend.Handle
filename string
}{
{
restic.Handle{Type: restic.PackFile, Name: "0123456"},
backend.Handle{Type: backend.PackFile, Name: "0123456"},
filepath.Join(path, "data", "0123456"),
},
{
restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
filepath.Join(path, "config"),
},
{
restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
filepath.Join(path, "snapshots", "123456"),
},
{
restic.Handle{Type: restic.IndexFile, Name: "123456"},
backend.Handle{Type: backend.IndexFile, Name: "123456"},
filepath.Join(path, "index", "123456"),
},
{
restic.Handle{Type: restic.LockFile, Name: "123456"},
backend.Handle{Type: backend.LockFile, Name: "123456"},
filepath.Join(path, "locks", "123456"),
},
{
restic.Handle{Type: restic.KeyFile, Name: "123456"},
backend.Handle{Type: backend.KeyFile, Name: "123456"},
filepath.Join(path, "keys", "123456"),
},
}
@ -209,61 +209,61 @@ func TestRESTLayout(t *testing.T) {
func TestRESTLayoutURLs(t *testing.T) {
var tests = []struct {
l Layout
h restic.Handle
h backend.Handle
fn string
dir string
}{
{
&RESTLayout{URL: "https://hostname.foo", Path: "", Join: path.Join},
restic.Handle{Type: restic.PackFile, Name: "foobar"},
backend.Handle{Type: backend.PackFile, Name: "foobar"},
"https://hostname.foo/data/foobar",
"https://hostname.foo/data/",
},
{
&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
backend.Handle{Type: backend.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/locks/foobar",
"https://hostname.foo:1234/prefix/repo/locks/",
},
{
&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
{
&S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.PackFile, Name: "foobar"},
backend.Handle{Type: backend.PackFile, Name: "foobar"},
"https://hostname.foo/data/foobar",
"https://hostname.foo/data/",
},
{
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
backend.Handle{Type: backend.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/lock/foobar",
"https://hostname.foo:1234/prefix/repo/lock/",
},
{
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
{
&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
restic.Handle{Type: restic.PackFile, Name: "foobar"},
backend.Handle{Type: backend.PackFile, Name: "foobar"},
"data/foobar",
"data/",
},
{
&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
backend.Handle{Type: backend.LockFile, Name: "foobar"},
"lock/foobar",
"lock/",
},
{
&S3LegacyLayout{URL: "", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"/config",
"/",
},
@ -288,31 +288,31 @@ func TestS3LegacyLayout(t *testing.T) {
path := rtest.TempDir(t)
var tests = []struct {
restic.Handle
backend.Handle
filename string
}{
{
restic.Handle{Type: restic.PackFile, Name: "0123456"},
backend.Handle{Type: backend.PackFile, Name: "0123456"},
filepath.Join(path, "data", "0123456"),
},
{
restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
filepath.Join(path, "config"),
},
{
restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
filepath.Join(path, "snapshot", "123456"),
},
{
restic.Handle{Type: restic.IndexFile, Name: "123456"},
backend.Handle{Type: backend.IndexFile, Name: "123456"},
filepath.Join(path, "index", "123456"),
},
{
restic.Handle{Type: restic.LockFile, Name: "123456"},
backend.Handle{Type: backend.LockFile, Name: "123456"},
filepath.Join(path, "lock", "123456"),
},
{
restic.Handle{Type: restic.KeyFile, Name: "123456"},
backend.Handle{Type: backend.KeyFile, Name: "123456"},
filepath.Join(path, "key", "123456"),
},
}
@ -415,8 +415,8 @@ func TestParseLayout(t *testing.T) {
}
// test that the functions work (and don't panic)
_ = layout.Dirname(restic.Handle{Type: restic.PackFile})
_ = layout.Filename(restic.Handle{Type: restic.PackFile, Name: "1234"})
_ = layout.Dirname(backend.Handle{Type: backend.PackFile})
_ = layout.Filename(backend.Handle{Type: backend.PackFile, Name: "1234"})
_ = layout.Paths()
layoutName := fmt.Sprintf("%T", layout)

22
internal/backend/limiter/limiter_backend.go
View File

@ -4,12 +4,12 @@ import (
"context"
"io"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
)
func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) {
return func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) {
var be restic.Backend
func WrapBackendConstructor[B backend.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (backend.Backend, error) {
return func(ctx context.Context, cfg C, lim Limiter) (backend.Backend, error) {
var be backend.Backend
be, err := constructor(ctx, cfg)
if err != nil {
return nil, err
@ -24,7 +24,7 @@ func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx contex
// LimitBackend wraps a Backend and applies rate limiting to Load() and Save()
// calls on the backend.
func LimitBackend(be restic.Backend, l Limiter) restic.Backend {
func LimitBackend(be backend.Backend, l Limiter) backend.Backend {
return rateLimitedBackend{
Backend: be,
limiter: l,
@ -32,11 +32,11 @@ func LimitBackend(be restic.Backend, l Limiter) restic.Backend {
}
type rateLimitedBackend struct {
restic.Backend
backend.Backend
limiter Limiter
}
func (r rateLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (r rateLimitedBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
limited := limitedRewindReader{
RewindReader: rd,
limited: r.limiter.Upstream(rd),
@ -46,7 +46,7 @@ func (r rateLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic
}
type limitedRewindReader struct {
restic.RewindReader
backend.RewindReader
limited io.Reader
}
@ -55,13 +55,13 @@ func (l limitedRewindReader) Read(b []byte) (int, error) {
return l.limited.Read(b)
}
func (r rateLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
func (r rateLimitedBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
return r.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
return consumer(newDownstreamLimitedReader(rd, r.limiter))
})
}
func (r rateLimitedBackend) Unwrap() restic.Backend { return r.Backend }
func (r rateLimitedBackend) Unwrap() backend.Backend { return r.Backend }
type limitedReader struct {
io.Reader
@ -85,4 +85,4 @@ func (l *limitedReader) WriteTo(w io.Writer) (int64, error) {
return l.writerTo.WriteTo(l.limiter.DownstreamWriter(w))
}
var _ restic.Backend = (*rateLimitedBackend)(nil)
var _ backend.Backend = (*rateLimitedBackend)(nil)

12
internal/backend/limiter/limiter_backend_test.go
View File

@ -8,8 +8,8 @@ import (
"io"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -21,11 +21,11 @@ func randomBytes(t *testing.T, size int) []byte {
}
func TestLimitBackendSave(t *testing.T) {
testHandle := restic.Handle{Type: restic.PackFile, Name: "test"}
testHandle := backend.Handle{Type: backend.PackFile, Name: "test"}
data := randomBytes(t, 1234)
be := mock.NewBackend()
be.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
be.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
buf := new(bytes.Buffer)
_, err := io.Copy(buf, rd)
if err != nil {
@ -39,7 +39,7 @@ func TestLimitBackendSave(t *testing.T) {
limiter := NewStaticLimiter(Limits{42 * 1024, 42 * 1024})
limbe := LimitBackend(be, limiter)
rd := restic.NewByteReader(data, nil)
rd := backend.NewByteReader(data, nil)
err := limbe.Save(context.TODO(), testHandle, rd)
rtest.OK(t, err)
}
@ -64,7 +64,7 @@ func (r *tracedReadWriteToCloser) Close() error {
}
func TestLimitBackendLoad(t *testing.T) {
testHandle := restic.Handle{Type: restic.PackFile, Name: "test"}
testHandle := backend.Handle{Type: backend.PackFile, Name: "test"}
data := randomBytes(t, 1234)
for _, test := range []struct {
@ -72,7 +72,7 @@ func TestLimitBackendLoad(t *testing.T) {
}{{false, false}, {false, true}, {true, false}, {true, true}} {
be := mock.NewBackend()
src := newTracedReadWriteToCloser(bytes.NewReader(data))
be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
if length != 0 || offset != 0 {
return nil, fmt.Errorf("Not supported")
}

4
internal/backend/local/layout_test.go
View File

@ -5,7 +5,7 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test"
)
@ -49,7 +49,7 @@ func TestLayout(t *testing.T) {
}
packs := make(map[string]bool)
err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
packs[fi.Name] = false
return nil
})

31
internal/backend/local/local.go
View File

@ -16,7 +16,6 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
"github.com/cenkalti/backoff/v4"
)
@ -28,8 +27,8 @@ type Local struct {
util.Modes
}
// ensure statically that *Local implements restic.Backend.
var _ restic.Backend = &Local{}
// ensure statically that *Local implements backend.Backend.
var _ backend.Backend = &Local{}
func NewFactory() location.Factory {
return location.NewLimitedBackendFactory("local", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open))
@ -43,7 +42,7 @@ func open(ctx context.Context, cfg Config) (*Local, error) {
return nil, err
}
fi, err := fs.Stat(l.Filename(restic.Handle{Type: restic.ConfigFile}))
fi, err := fs.Stat(l.Filename(backend.Handle{Type: backend.ConfigFile}))
m := util.DeriveModesFromFileInfo(fi, err)
debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir)
@ -71,7 +70,7 @@ func Create(ctx context.Context, cfg Config) (*Local, error) {
}
// test if config file already exists
_, err = fs.Lstat(be.Filename(restic.Handle{Type: restic.ConfigFile}))
_, err = fs.Lstat(be.Filename(backend.Handle{Type: backend.ConfigFile}))
if err == nil {
return nil, errors.New("config file already exists")
}
@ -112,7 +111,7 @@ func (b *Local) IsNotExist(err error) bool {
}
// Save stores data in the backend at the handle.
func (b *Local) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) (err error) {
func (b *Local) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) (err error) {
finalname := b.Filename(h)
dir := filepath.Dir(finalname)
@ -210,11 +209,11 @@ var tempFile = os.CreateTemp // Overridden by test.
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (b *Local) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, b.openReader, fn)
}
func (b *Local) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (b *Local) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
f, err := fs.Open(b.Filename(h))
if err != nil {
return nil, err
@ -236,17 +235,17 @@ func (b *Local) openReader(_ context.Context, h restic.Handle, length int, offse
}
// Stat returns information about a blob.
func (b *Local) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) {
func (b *Local) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) {
fi, err := fs.Stat(b.Filename(h))
if err != nil {
return restic.FileInfo{}, errors.WithStack(err)
return backend.FileInfo{}, errors.WithStack(err)
}
return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil
return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil
}
// Remove removes the blob with the given name and type.
func (b *Local) Remove(_ context.Context, h restic.Handle) error {
func (b *Local) Remove(_ context.Context, h backend.Handle) error {
fn := b.Filename(h)
// reset read-only flag
@ -260,7 +259,7 @@ func (b *Local) Remove(_ context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) (err error) {
func (b *Local) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) (err error) {
basedir, subdirs := b.Basedir(t)
if subdirs {
err = visitDirs(ctx, basedir, fn)
@ -280,7 +279,7 @@ func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.File
// two levels of directory structure (including dir itself as the first level).
// Also, visitDirs assumes it sees a directory full of directories, while
// visitFiles wants a directory full or regular files.
func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error) error {
func visitDirs(ctx context.Context, dir string, fn func(backend.FileInfo) error) error {
d, err := fs.Open(dir)
if err != nil {
return err
@ -307,7 +306,7 @@ func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error)
return ctx.Err()
}
func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error, ignoreNotADirectory bool) error {
func visitFiles(ctx context.Context, dir string, fn func(backend.FileInfo) error, ignoreNotADirectory bool) error {
d, err := fs.Open(dir)
if err != nil {
return err
@ -341,7 +340,7 @@ func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error,
default:
}
err := fn(restic.FileInfo{
err := fn(backend.FileInfo{
Name: fi.Name(),
Size: fi.Size(),
})

4
internal/backend/local/local_internal_test.go
View File

@ -8,7 +8,7 @@ import (
"syscall"
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test"
"github.com/cenkalti/backoff/v4"
@ -32,7 +32,7 @@ func TestNoSpacePermanent(t *testing.T) {
rtest.OK(t, be.Close())
}()
h := restic.Handle{Type: restic.ConfigFile}
h := backend.Handle{Type: backend.ConfigFile}
err = be.Save(context.Background(), h, nil)
_, ok := err.(*backoff.PermanentError)
rtest.Assert(t, ok,

4
internal/backend/location/display_location_test.go
View File

@ -3,15 +3,15 @@ package location_test
import (
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
func TestStripPassword(t *testing.T) {
registry := location.NewRegistry()
registry.Register(
location.NewHTTPBackendFactory[any, restic.Backend]("test", nil,
location.NewHTTPBackendFactory[any, backend.Backend]("test", nil,
func(s string) string {
return "cleaned"
}, nil, nil,

4
internal/backend/location/location_test.go
View File

@ -3,8 +3,8 @@ package location_test
import (
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
@ -13,7 +13,7 @@ type testConfig struct {
}
func testFactory() location.Factory {
return location.NewHTTPBackendFactory[testConfig, restic.Backend](
return location.NewHTTPBackendFactory[testConfig, backend.Backend](
"local",
func(s string) (*testConfig, error) {
return &testConfig{loc: s}, nil

16
internal/backend/location/registry.go
View File

@ -4,8 +4,8 @@ import (
"context"
"net/http"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/restic"
)
type Registry struct {
@ -33,11 +33,11 @@ type Factory interface {
Scheme() string
ParseConfig(s string) (interface{}, error)
StripPassword(s string) string
Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error)
Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error)
Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error)
Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error)
}
type genericBackendFactory[C any, T restic.Backend] struct {
type genericBackendFactory[C any, T backend.Backend] struct {
scheme string
parseConfigFn func(s string) (*C, error)
stripPasswordFn func(s string) string
@ -58,14 +58,14 @@ func (f *genericBackendFactory[C, T]) StripPassword(s string) string {
}
return s
}
func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) {
func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) {
return f.createFn(ctx, *cfg.(*C), rt, lim)
}
func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) {
func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) {
return f.openFn(ctx, *cfg.(*C), rt, lim)
}
func NewHTTPBackendFactory[C any, T restic.Backend](
func NewHTTPBackendFactory[C any, T backend.Backend](
scheme string,
parseConfigFn func(s string) (*C, error),
stripPasswordFn func(s string) string,
@ -85,7 +85,7 @@ func NewHTTPBackendFactory[C any, T restic.Backend](
}
}
func NewLimitedBackendFactory[C any, T restic.Backend](
func NewLimitedBackendFactory[C any, T backend.Backend](
scheme string,
parseConfigFn func(s string) (*C, error),
stripPasswordFn func(s string) string,

22
internal/backend/logger/log.go
View File

@ -4,18 +4,18 @@ import (
"context"
"io"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
type Backend struct {
restic.Backend
backend.Backend
}
// statically ensure that Backend implements restic.Backend.
var _ restic.Backend = &Backend{}
// statically ensure that Backend implements backend.Backend.
var _ backend.Backend = &Backend{}
func New(be restic.Backend) *Backend {
func New(be backend.Backend) *Backend {
return &Backend{Backend: be}
}
@ -26,7 +26,7 @@ func (be *Backend) IsNotExist(err error) bool {
}
// Save adds new Data to the backend.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
debug.Log("Save(%v, %v)", h, rd.Length())
err := be.Backend.Save(ctx, h, rd)
debug.Log(" save err %v", err)
@ -34,28 +34,28 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
}
// Remove deletes a file from the backend.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
debug.Log("Remove(%v)", h)
err := be.Backend.Remove(ctx, h)
debug.Log(" remove err %v", err)
return err
}
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(io.Reader) error) error {
debug.Log("Load(%v, length %v, offset %v)", h, length, offset)
err := be.Backend.Load(ctx, h, length, offset, fn)
debug.Log(" load err %v", err)
return err
}
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
debug.Log("Stat(%v)", h)
fi, err := be.Backend.Stat(ctx, h)
debug.Log(" stat err %v", err)
return fi, err
}
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
debug.Log("List(%v)", t)
err := be.Backend.List(ctx, t, fn)
debug.Log(" list err %v", err)
@ -76,4 +76,4 @@ func (be *Backend) Close() error {
return err
}
func (be *Backend) Unwrap() restic.Backend { return be.Backend }
func (be *Backend) Unwrap() backend.Backend { return be.Backend }

30
internal/backend/mem/mem_backend.go
View File

@ -10,17 +10,17 @@ import (
"sync"
"github.com/cespare/xxhash/v2"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
type memMap map[restic.Handle][]byte
type memMap map[backend.Handle][]byte
// make sure that MemoryBackend implements backend.Backend
var _ restic.Backend = &MemoryBackend{}
var _ backend.Backend = &MemoryBackend{}
// NewFactory creates a persistent mem backend
func NewFactory() location.Factory {
@ -69,12 +69,12 @@ func (be *MemoryBackend) IsNotExist(err error) bool {
}
// Save adds new Data to the backend.
func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *MemoryBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
be.m.Lock()
defer be.m.Unlock()
h.IsMetadata = false
if h.Type == restic.ConfigFile {
if h.Type == backend.ConfigFile {
h.Name = ""
}
@ -112,16 +112,16 @@ func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *MemoryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *MemoryBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *MemoryBackend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
be.m.Lock()
defer be.m.Unlock()
h.IsMetadata = false
if h.Type == restic.ConfigFile {
if h.Type == backend.ConfigFile {
h.Name = ""
}
@ -143,25 +143,25 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length
}
// Stat returns information about a file in the backend.
func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (be *MemoryBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
be.m.Lock()
defer be.m.Unlock()
h.IsMetadata = false
if h.Type == restic.ConfigFile {
if h.Type == backend.ConfigFile {
h.Name = ""
}
e, ok := be.data[h]
if !ok {
return restic.FileInfo{}, errNotFound
return backend.FileInfo{}, errNotFound
}
return restic.FileInfo{Size: int64(len(e)), Name: h.Name}, ctx.Err()
return backend.FileInfo{Size: int64(len(e)), Name: h.Name}, ctx.Err()
}
// Remove deletes a file from the backend.
func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error {
func (be *MemoryBackend) Remove(ctx context.Context, h backend.Handle) error {
be.m.Lock()
defer be.m.Unlock()
@ -176,7 +176,7 @@ func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error {
}
// List returns a channel which yields entries from the backend.
func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *MemoryBackend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
entries := make(map[string]int64)
be.m.Lock()
@ -190,7 +190,7 @@ func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(re
be.m.Unlock()
for name, size := range entries {
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: name,
Size: size,
}

28
internal/backend/mock/backend.go
View File

@ -5,19 +5,19 @@ import (
"hash"
"io"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
// Backend implements a mock backend.
type Backend struct {
CloseFn func() error
IsNotExistFn func(err error) bool
SaveFn func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error
OpenReaderFn func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error)
StatFn func(ctx context.Context, h restic.Handle) (restic.FileInfo, error)
ListFn func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error
RemoveFn func(ctx context.Context, h restic.Handle) error
SaveFn func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error
OpenReaderFn func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error)
StatFn func(ctx context.Context, h backend.Handle) (backend.FileInfo, error)
ListFn func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error
RemoveFn func(ctx context.Context, h backend.Handle) error
DeleteFn func(ctx context.Context) error
ConnectionsFn func() uint
LocationFn func() string
@ -84,7 +84,7 @@ func (m *Backend) IsNotExist(err error) bool {
}
// Save data in the backend.
func (m *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (m *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if m.SaveFn == nil {
return errors.New("not implemented")
}
@ -94,7 +94,7 @@ func (m *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (m *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (m *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
rd, err := m.openReader(ctx, h, length, offset)
if err != nil {
return err
@ -107,7 +107,7 @@ func (m *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
return rd.Close()
}
func (m *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (m *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
if m.OpenReaderFn == nil {
return nil, errors.New("not implemented")
}
@ -116,16 +116,16 @@ func (m *Backend) openReader(ctx context.Context, h restic.Handle, length int, o
}
// Stat an object in the backend.
func (m *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (m *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
if m.StatFn == nil {
return restic.FileInfo{}, errors.New("not implemented")
return backend.FileInfo{}, errors.New("not implemented")
}
return m.StatFn(ctx, h)
}
// List items of type t.
func (m *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (m *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
if m.ListFn == nil {
return nil
}
@ -134,7 +134,7 @@ func (m *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi
}
// Remove data from the backend.
func (m *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (m *Backend) Remove(ctx context.Context, h backend.Handle) error {
if m.RemoveFn == nil {
return errors.New("not implemented")
}
@ -152,4 +152,4 @@ func (m *Backend) Delete(ctx context.Context) error {
}
// Make sure that Backend implements the backend interface.
var _ restic.Backend = &Backend{}
var _ backend.Backend = &Backend{}

6
internal/backend/rclone/internal_test.go
View File

@ -5,8 +5,8 @@ import (
"os/exec"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -32,9 +32,9 @@ func TestRcloneExit(t *testing.T) {
t.Log("killed rclone")
for i := 0; i < 10; i++ {
_, err = be.Stat(context.TODO(), restic.Handle{
_, err = be.Stat(context.TODO(), backend.Handle{
Name: "foo",
Type: restic.PackFile,
Type: backend.PackFile,
})
rtest.Assert(t, err != nil, "expected an error")
}

9
internal/backend/readerat.go
View File

@ -6,13 +6,12 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
type backendReaderAt struct {
ctx context.Context
be restic.Backend
h restic.Handle
be Backend
h Handle
}
func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
@ -22,12 +21,12 @@ func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
// ReaderAt returns an io.ReaderAt for a file in the backend. The returned reader
// should not escape the caller function to avoid unexpected interactions with the
// embedded context
func ReaderAt(ctx context.Context, be restic.Backend, h restic.Handle) io.ReaderAt {
func ReaderAt(ctx context.Context, be Backend, h Handle) io.ReaderAt {
return backendReaderAt{ctx: ctx, be: be, h: h}
}
// ReadAt reads from the backend handle h at the given position.
func ReadAt(ctx context.Context, be restic.Backend, h restic.Handle, offset int64, p []byte) (n int, err error) {
func ReadAt(ctx context.Context, be Backend, h Handle, offset int64, p []byte) (n int, err error) {
debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p))
err = be.Load(ctx, h, len(p), offset, func(rd io.Reader) (ierr error) {

4
internal/backend/rest/config.go
View File

@ -5,9 +5,9 @@ import (
"os"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to a REST server.
@ -73,7 +73,7 @@ func prepareURL(s string) string {
return s
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {

46
internal/backend/rest/rest.go
View File

@ -11,16 +11,16 @@ import (
"path"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
// make sure the rest backend implements restic.Backend
var _ restic.Backend = &Backend{}
// make sure the rest backend implements backend.Backend
var _ backend.Backend = &Backend{}
// Backend uses the REST protocol to access data stored on a server.
type Backend struct {
@ -65,7 +65,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, er
return nil, err
}
_, err = be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
_, err = be.Stat(ctx, backend.Handle{Type: backend.ConfigFile})
if err == nil {
return nil, errors.New("config file already exists")
}
@ -118,7 +118,7 @@ func (b *Backend) HasAtomicReplace() bool {
}
// Save stores data in the backend at the handle.
func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
@ -157,7 +157,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
// notExistError is returned whenever the requested file does not exist on the
// server.
type notExistError struct {
restic.Handle
backend.Handle
}
func (e *notExistError) Error() string {
@ -172,7 +172,7 @@ func (b *Backend) IsNotExist(err error) bool {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
r, err := b.openReader(ctx, h, length, offset)
if err != nil {
return err
@ -201,7 +201,7 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
return err
}
func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
req, err := http.NewRequestWithContext(ctx, "GET", b.Filename(h), nil)
if err != nil {
return nil, errors.WithStack(err)
@ -238,37 +238,37 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o
}
// Stat returns information about a blob.
func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodHead, b.Filename(h), nil)
if err != nil {
return restic.FileInfo{}, errors.WithStack(err)
return backend.FileInfo{}, errors.WithStack(err)
}
req.Header.Set("Accept", ContentTypeV2)
resp, err := b.client.Do(req)
if err != nil {
return restic.FileInfo{}, errors.WithStack(err)
return backend.FileInfo{}, errors.WithStack(err)
}
_, _ = io.Copy(io.Discard, resp.Body)
if err = resp.Body.Close(); err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Close")
return backend.FileInfo{}, errors.Wrap(err, "Close")
}
if resp.StatusCode == http.StatusNotFound {
_ = resp.Body.Close()
return restic.FileInfo{}, &notExistError{h}
return backend.FileInfo{}, &notExistError{h}
}
if resp.StatusCode != http.StatusOK {
return restic.FileInfo{}, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status)
return backend.FileInfo{}, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status)
}
if resp.ContentLength < 0 {
return restic.FileInfo{}, errors.New("negative content length")
return backend.FileInfo{}, errors.New("negative content length")
}
bi := restic.FileInfo{
bi := backend.FileInfo{
Size: resp.ContentLength,
Name: h.Name,
}
@ -277,7 +277,7 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e
}
// Remove removes the blob with the given name and type.
func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (b *Backend) Remove(ctx context.Context, h backend.Handle) error {
req, err := http.NewRequestWithContext(ctx, "DELETE", b.Filename(h), nil)
if err != nil {
return errors.WithStack(err)
@ -309,8 +309,8 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
url := b.Dirname(restic.Handle{Type: t})
func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
url := b.Dirname(backend.Handle{Type: t})
if !strings.HasSuffix(url, "/") {
url += "/"
}
@ -346,7 +346,7 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi
// listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET
// /data/`) only returns the names of the files, so we need to issue an HTTP
// HEAD request for each file.
func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
func (b *Backend) listv1(ctx context.Context, t backend.FileType, resp *http.Response, fn func(backend.FileInfo) error) error {
debug.Log("parsing API v1 response")
dec := json.NewDecoder(resp.Body)
var list []string
@ -355,7 +355,7 @@ func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Resp
}
for _, m := range list {
fi, err := b.Stat(ctx, restic.Handle{Name: m, Type: t})
fi, err := b.Stat(ctx, backend.Handle{Name: m, Type: t})
if err != nil {
return err
}
@ -380,7 +380,7 @@ func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Resp
// listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET
// /data/`) returns the names and sizes of all files.
func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(restic.FileInfo) error) error {
func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(backend.FileInfo) error) error {
debug.Log("parsing API v2 response")
dec := json.NewDecoder(resp.Body)
@ -397,7 +397,7 @@ func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(resti
return ctx.Err()
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: item.Name,
Size: item.Size,
}

14
internal/backend/rest/rest_int_test.go
View File

@ -10,8 +10,8 @@ import (
"strconv"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/restic"
)
func TestListAPI(t *testing.T) {
@ -22,7 +22,7 @@ func TestListAPI(t *testing.T) {
Data string // response data
Requests int
Result []restic.FileInfo
Result []backend.FileInfo
}{
{
Name: "content-type-unknown",
@ -32,7 +32,7 @@ func TestListAPI(t *testing.T) {
"3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352",
"8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b"
]`,
Result: []restic.FileInfo{
Result: []backend.FileInfo{
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386},
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214},
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393},
@ -47,7 +47,7 @@ func TestListAPI(t *testing.T) {
"3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352",
"8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b"
]`,
Result: []restic.FileInfo{
Result: []backend.FileInfo{
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386},
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214},
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393},
@ -62,7 +62,7 @@ func TestListAPI(t *testing.T) {
{"name": "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "size": 1002},
{"name": "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", "size": 1003}
]`,
Result: []restic.FileInfo{
Result: []backend.FileInfo{
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 1001},
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 1002},
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 1003},
@ -122,8 +122,8 @@ func TestListAPI(t *testing.T) {
t.Fatal(err)
}
var list []restic.FileInfo
err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
var list []backend.FileInfo
err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
list = append(list, fi)
return nil
})

24
internal/backend/retry/backend_retry.go
View File

@ -7,27 +7,27 @@ import (
"time"
"github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
// Backend retries operations on the backend in case of an error with a
// backoff.
type Backend struct {
restic.Backend
backend.Backend
MaxTries int
Report func(string, error, time.Duration)
Success func(string, int)
}
// statically ensure that RetryBackend implements restic.Backend.
var _ restic.Backend = &Backend{}
// statically ensure that RetryBackend implements backend.Backend.
var _ backend.Backend = &Backend{}
// New wraps be with a backend that retries operations after a
// backoff. report is called with a description and the error, if one occurred.
// success is called with the number of retries before a successful operation
// (it is not called if it succeeded on the first try)
func New(be restic.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend {
func New(be backend.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend {
return &Backend{
Backend: be,
MaxTries: maxTries,
@ -92,7 +92,7 @@ func (be *Backend) retry(ctx context.Context, msg string, f func() error) error
}
// Save stores the data in the backend under the given handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
return be.retry(ctx, fmt.Sprintf("Save(%v)", h), func() error {
err := rd.Rewind()
if err != nil {
@ -125,7 +125,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
// given offset. If length is larger than zero, only a portion of the file
// is returned. rd must be closed after use. If an error is returned, the
// ReadCloser must be nil.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) {
return be.retry(ctx, fmt.Sprintf("Load(%v, %v, %v)", h, length, offset),
func() error {
err := be.Backend.Load(ctx, h, length, offset, consumer)
@ -137,7 +137,7 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
}
// Stat returns information about the File identified by h.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (fi backend.FileInfo, err error) {
err = be.retry(ctx, fmt.Sprintf("Stat(%v)", h),
func() error {
var innerError error
@ -153,7 +153,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInf
}
// Remove removes a File with type t and name.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) (err error) {
return be.retry(ctx, fmt.Sprintf("Remove(%v)", h), func() error {
return be.Backend.Remove(ctx, h)
})
@ -163,7 +163,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) {
// error is returned by the underlying backend, the request is retried. When fn
// returns an error, the operation is aborted and the error is returned to the
// caller.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
// create a new context that we can cancel when fn returns an error, so
// that listing is aborted
listCtx, cancel := context.WithCancel(ctx)
@ -173,7 +173,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
var innerErr error // remember when fn returned an error, so we can return that to the caller
err := be.retry(listCtx, fmt.Sprintf("List(%v)", t), func() error {
return be.Backend.List(ctx, t, func(fi restic.FileInfo) error {
return be.Backend.List(ctx, t, func(fi backend.FileInfo) error {
if _, ok := listed[fi.Name]; ok {
return nil
}
@ -196,6 +196,6 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
return err
}
func (be *Backend) Unwrap() restic.Backend {
func (be *Backend) Unwrap() backend.Backend {
return be.Backend
}

57
internal/backend/retry/backend_retry_test.go
View File

@ -8,6 +8,7 @@ import (
"time"
"github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -18,7 +19,7 @@ func TestBackendSaveRetry(t *testing.T) {
buf := bytes.NewBuffer(nil)
errcount := 0
be := &mock.Backend{
SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if errcount == 0 {
errcount++
_, err := io.CopyN(io.Discard, rd, 120)
@ -38,7 +39,7 @@ func TestBackendSaveRetry(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
data := test.Random(23, 5*1024*1024+11241)
err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher()))
err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatal(err)
}
@ -56,14 +57,14 @@ func TestBackendSaveRetryAtomic(t *testing.T) {
errcount := 0
calledRemove := false
be := &mock.Backend{
SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if errcount == 0 {
errcount++
return errors.New("injected error")
}
return nil
},
RemoveFn: func(ctx context.Context, h restic.Handle) error {
RemoveFn: func(ctx context.Context, h backend.Handle) error {
calledRemove = true
return nil
},
@ -74,7 +75,7 @@ func TestBackendSaveRetryAtomic(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
data := test.Random(23, 5*1024*1024+11241)
err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher()))
err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatal(err)
}
@ -91,15 +92,15 @@ func TestBackendListRetry(t *testing.T) {
retry := 0
be := &mock.Backend{
ListFn: func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
ListFn: func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
// fail during first retry, succeed during second
retry++
if retry == 1 {
_ = fn(restic.FileInfo{Name: ID1})
_ = fn(backend.FileInfo{Name: ID1})
return errors.New("test list error")
}
_ = fn(restic.FileInfo{Name: ID1})
_ = fn(restic.FileInfo{Name: ID2})
_ = fn(backend.FileInfo{Name: ID1})
_ = fn(backend.FileInfo{Name: ID2})
return nil
},
}
@ -108,7 +109,7 @@ func TestBackendListRetry(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
var listed []string
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
listed = append(listed, fi.Name)
return nil
})
@ -121,10 +122,10 @@ func TestBackendListRetryErrorFn(t *testing.T) {
var names = []string{"id1", "id2", "foo", "bar"}
be := &mock.Backend{
ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error {
ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error {
t.Logf("List called for %v", tpe)
for _, name := range names {
err := fn(restic.FileInfo{Name: name})
err := fn(backend.FileInfo{Name: name})
if err != nil {
return err
}
@ -141,7 +142,7 @@ func TestBackendListRetryErrorFn(t *testing.T) {
var listed []string
run := 0
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
t.Logf("fn called for %v", fi.Name)
run++
// return an error for the third item in the list
@ -172,7 +173,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
retries := 0
be := &mock.Backend{
ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error {
ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error {
t.Logf("List called for %v, retries %v", tpe, retries)
retries++
for i, name := range names {
@ -180,7 +181,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
return ErrBackendTest
}
err := fn(restic.FileInfo{Name: name})
err := fn(backend.FileInfo{Name: name})
if err != nil {
return err
}
@ -195,7 +196,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
retryBackend := New(be, maxRetries, nil, nil)
var listed []string
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
t.Logf("fn called for %v", fi.Name)
listed = append(listed, fi.Name)
return nil
@ -252,7 +253,7 @@ func TestBackendLoadRetry(t *testing.T) {
attempt := 0
be := mock.NewBackend()
be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
// returns failing reader on first invocation, good reader on subsequent invocations
attempt++
if attempt > 1 {
@ -265,7 +266,7 @@ func TestBackendLoadRetry(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
var buf []byte
err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) {
err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
buf, err = io.ReadAll(rd)
return err
})
@ -280,7 +281,7 @@ func TestBackendLoadNotExists(t *testing.T) {
attempt := 0
be := mock.NewBackend()
be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
attempt++
if attempt > 1 {
t.Fail()
@ -295,7 +296,7 @@ func TestBackendLoadNotExists(t *testing.T) {
TestFastRetries(t)
retryBackend := New(be, 10, nil, nil)
err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) {
err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
return nil
})
test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err)
@ -308,13 +309,13 @@ func TestBackendStatNotExists(t *testing.T) {
attempt := 0
be := mock.NewBackend()
be.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
be.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
attempt++
if attempt > 1 {
t.Fail()
return restic.FileInfo{}, errors.New("must not retry")
return backend.FileInfo{}, errors.New("must not retry")
}
return restic.FileInfo{}, notFound
return backend.FileInfo{}, notFound
}
be.IsNotExistFn = func(err error) bool {
return errors.Is(err, notFound)
@ -323,7 +324,7 @@ func TestBackendStatNotExists(t *testing.T) {
TestFastRetries(t)
retryBackend := New(be, 10, nil, nil)
_, err := retryBackend.Stat(context.TODO(), restic.Handle{})
_, err := retryBackend.Stat(context.TODO(), backend.Handle{})
test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err)
test.Equals(t, 1, attempt)
}
@ -337,7 +338,7 @@ func TestBackendCanceledContext(t *testing.T) {
// check that we received the expected context canceled error instead
TestFastRetries(t)
retryBackend := New(mock.NewBackend(), 2, nil, nil)
h := restic.Handle{Type: restic.PackFile, Name: restic.NewRandomID().String()}
h := backend.Handle{Type: backend.PackFile, Name: restic.NewRandomID().String()}
// create an already canceled context
ctx, cancel := context.WithCancel(context.Background())
@ -346,15 +347,15 @@ func TestBackendCanceledContext(t *testing.T) {
_, err := retryBackend.Stat(ctx, h)
assertIsCanceled(t, err)
err = retryBackend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
err = retryBackend.Save(ctx, h, backend.NewByteReader([]byte{}, nil))
assertIsCanceled(t, err)
err = retryBackend.Remove(ctx, h)
assertIsCanceled(t, err)
err = retryBackend.Load(ctx, restic.Handle{}, 0, 0, func(rd io.Reader) (err error) {
err = retryBackend.Load(ctx, backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
return nil
})
assertIsCanceled(t, err)
err = retryBackend.List(ctx, restic.PackFile, func(restic.FileInfo) error {
err = retryBackend.List(ctx, backend.PackFile, func(backend.FileInfo) error {
return nil
})
assertIsCanceled(t, err)

2
internal/restic/rewind_reader.go → internal/backend/rewind_reader.go
View File

@ -1,4 +1,4 @@
package restic
package backend
import (
"bytes"

2
internal/restic/rewind_reader_test.go → internal/backend/rewind_reader_test.go
View File

@ -1,4 +1,4 @@
package restic
package backend
import (
"bytes"

4
internal/backend/s3/config.go
View File

@ -6,9 +6,9 @@ import (
"path"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to an s3 compatible
@ -94,7 +94,7 @@ func createConfig(endpoint, bucket, prefix string, useHTTP bool) (*Config, error
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {

30
internal/backend/s3/s3.go
View File

@ -11,12 +11,12 @@ import (
"strings"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
@ -30,7 +30,7 @@ type Backend struct {
}
// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}
var _ backend.Backend = &Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open)
@ -127,13 +127,13 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro
// Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
return open(ctx, cfg, rt)
}
// Create opens the S3 backend at bucket and region and creates the bucket if
// it does not exist yet.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
be, err := open(ctx, cfg, rt)
if err != nil {
return nil, errors.Wrap(err, "open")
@ -272,7 +272,7 @@ func (be *Backend) Path() string {
}
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
opts := minio.PutObjectOptions{StorageClass: be.cfg.StorageClass}
@ -294,14 +294,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
objName := be.Filename(h)
opts := minio.GetObjectOptions{}
@ -326,7 +326,7 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int,
}
// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h)
var obj *minio.Object
@ -334,7 +334,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
obj, err = be.client.GetObject(ctx, be.cfg.Bucket, objName, opts)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "client.GetObject")
return backend.FileInfo{}, errors.Wrap(err, "client.GetObject")
}
// make sure that the object is closed properly.
@ -347,14 +347,14 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
fi, err := obj.Stat()
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Stat")
return backend.FileInfo{}, errors.Wrap(err, "Stat")
}
return restic.FileInfo{Size: fi.Size, Name: h.Name}, nil
return backend.FileInfo{Size: fi.Size, Name: h.Name}, nil
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{})
@ -368,7 +368,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, recursive := be.Basedir(t)
// make sure prefix ends with a slash
@ -400,7 +400,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: path.Base(m),
Size: obj.Size,
}
@ -431,7 +431,7 @@ func (be *Backend) Delete(ctx context.Context) error {
func (be *Backend) Close() error { return nil }
// Rename moves a file based on the new layout l.
func (be *Backend) Rename(ctx context.Context, h restic.Handle, l layout.Layout) error {
func (be *Backend) Rename(ctx context.Context, h backend.Handle, l layout.Layout) error {
debug.Log("Rename %v to %v", h, l)
oldname := be.Filename(h)
newname := l.Filename(h)

4
internal/backend/s3/s3_test.go
View File

@ -14,11 +14,11 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -117,7 +117,7 @@ func newMinioTestSuite(t testing.TB) (*test.Suite[s3.Config], func()) {
return &cfg, nil
},
Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be restic.Backend, err error) {
Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be backend.Backend, err error) {
for i := 0; i < 10; i++ {
be, err = s3.Create(ctx, cfg, rt)
if err != nil {

28
internal/backend/sema/backend.go
View File

@ -6,22 +6,22 @@ import (
"sync"
"github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
// make sure that connectionLimitedBackend implements restic.Backend
var _ restic.Backend = &connectionLimitedBackend{}
// make sure that connectionLimitedBackend implements backend.Backend
var _ backend.Backend = &connectionLimitedBackend{}
// connectionLimitedBackend limits the number of concurrent operations.
type connectionLimitedBackend struct {
restic.Backend
backend.Backend
sem semaphore
freezeLock sync.Mutex
}
// NewBackend creates a backend that limits the concurrent operations on the underlying backend
func NewBackend(be restic.Backend) restic.Backend {
func NewBackend(be backend.Backend) backend.Backend {
sem, err := newSemaphore(be.Connections())
if err != nil {
panic(err)
@ -35,9 +35,9 @@ func NewBackend(be restic.Backend) restic.Backend {
// typeDependentLimit acquire a token unless the FileType is a lock file. The returned function
// must be called to release the token.
func (be *connectionLimitedBackend) typeDependentLimit(t restic.FileType) func() {
func (be *connectionLimitedBackend) typeDependentLimit(t backend.FileType) func() {
// allow concurrent lock file operations to ensure that the lock refresh is always possible
if t == restic.LockFile {
if t == backend.LockFile {
return func() {}
}
be.sem.GetToken()
@ -59,7 +59,7 @@ func (be *connectionLimitedBackend) Unfreeze() {
}
// Save adds new Data to the backend.
func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *connectionLimitedBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if err := h.Valid(); err != nil {
return backoff.Permanent(err)
}
@ -75,7 +75,7 @@ func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, r
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *connectionLimitedBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
if err := h.Valid(); err != nil {
return backoff.Permanent(err)
}
@ -96,22 +96,22 @@ func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, l
}
// Stat returns information about a file in the backend.
func (be *connectionLimitedBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (be *connectionLimitedBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
if err := h.Valid(); err != nil {
return restic.FileInfo{}, backoff.Permanent(err)
return backend.FileInfo{}, backoff.Permanent(err)
}
defer be.typeDependentLimit(h.Type)()
if ctx.Err() != nil {
return restic.FileInfo{}, ctx.Err()
return backend.FileInfo{}, ctx.Err()
}
return be.Backend.Stat(ctx, h)
}
// Remove deletes a file from the backend.
func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle) error {
func (be *connectionLimitedBackend) Remove(ctx context.Context, h backend.Handle) error {
if err := h.Valid(); err != nil {
return backoff.Permanent(err)
}
@ -125,6 +125,6 @@ func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle)
return be.Backend.Remove(ctx, h)
}
func (be *connectionLimitedBackend) Unwrap() restic.Backend {
func (be *connectionLimitedBackend) Unwrap() backend.Backend {
return be.Backend
}

64
internal/backend/sema/backend_test.go
View File

@ -8,37 +8,37 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
"golang.org/x/sync/errgroup"
)
func TestParameterValidationSave(t *testing.T) {
m := mock.NewBackend()
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
return nil
}
be := sema.NewBackend(m)
err := be.Save(context.TODO(), restic.Handle{}, nil)
err := be.Save(context.TODO(), backend.Handle{}, nil)
test.Assert(t, err != nil, "Save() with invalid handle did not return an error")
}
func TestParameterValidationLoad(t *testing.T) {
m := mock.NewBackend()
m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
m.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return io.NopCloser(nil), nil
}
be := sema.NewBackend(m)
nilCb := func(rd io.Reader) error { return nil }
err := be.Load(context.TODO(), restic.Handle{}, 10, 0, nilCb)
err := be.Load(context.TODO(), backend.Handle{}, 10, 0, nilCb)
test.Assert(t, err != nil, "Load() with invalid handle did not return an error")
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
err = be.Load(context.TODO(), h, 10, -1, nilCb)
test.Assert(t, err != nil, "Save() with negative offset did not return an error")
err = be.Load(context.TODO(), h, -1, 0, nilCb)
@ -47,23 +47,23 @@ func TestParameterValidationLoad(t *testing.T) {
func TestParameterValidationStat(t *testing.T) {
m := mock.NewBackend()
m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
return restic.FileInfo{}, nil
m.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
return backend.FileInfo{}, nil
}
be := sema.NewBackend(m)
_, err := be.Stat(context.TODO(), restic.Handle{})
_, err := be.Stat(context.TODO(), backend.Handle{})
test.Assert(t, err != nil, "Stat() with invalid handle did not return an error")
}
func TestParameterValidationRemove(t *testing.T) {
m := mock.NewBackend()
m.RemoveFn = func(ctx context.Context, h restic.Handle) error {
m.RemoveFn = func(ctx context.Context, h backend.Handle) error {
return nil
}
be := sema.NewBackend(m)
err := be.Remove(context.TODO(), restic.Handle{})
err := be.Remove(context.TODO(), backend.Handle{})
test.Assert(t, err != nil, "Remove() with invalid handle did not return an error")
}
@ -71,7 +71,7 @@ func TestUnwrap(t *testing.T) {
m := mock.NewBackend()
be := sema.NewBackend(m)
unwrapper := be.(restic.BackendUnwrapper)
unwrapper := be.(backend.Unwrapper)
test.Assert(t, unwrapper.Unwrap() == m, "Unwrap() returned wrong backend")
}
@ -100,7 +100,7 @@ func countingBlocker() (func(), func(int) int) {
return wait, unblock
}
func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be restic.Backend) func() error, unblock func(int) int, isUnlimited bool) {
func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be backend.Backend) func() error, unblock func(int) int, isUnlimited bool) {
expectBlocked := int(2)
workerCount := expectBlocked + 1
@ -125,13 +125,13 @@ func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(b
func TestConcurrencyLimitSave(t *testing.T) {
wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) {
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
wait()
return nil
}
}, func(be restic.Backend) func() error {
}, func(be backend.Backend) func() error {
return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
return be.Save(context.TODO(), h, nil)
}
}, unblock, false)
@ -140,13 +140,13 @@ func TestConcurrencyLimitSave(t *testing.T) {
func TestConcurrencyLimitLoad(t *testing.T) {
wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) {
m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
m.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
wait()
return io.NopCloser(nil), nil
}
}, func(be restic.Backend) func() error {
}, func(be backend.Backend) func() error {
return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
nilCb := func(rd io.Reader) error { return nil }
return be.Load(context.TODO(), h, 10, 0, nilCb)
}
@ -156,13 +156,13 @@ func TestConcurrencyLimitLoad(t *testing.T) {
func TestConcurrencyLimitStat(t *testing.T) {
wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) {
m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
m.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
wait()
return restic.FileInfo{}, nil
return backend.FileInfo{}, nil
}
}, func(be restic.Backend) func() error {
}, func(be backend.Backend) func() error {
return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
_, err := be.Stat(context.TODO(), h)
return err
}
@ -172,13 +172,13 @@ func TestConcurrencyLimitStat(t *testing.T) {
func TestConcurrencyLimitDelete(t *testing.T) {
wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) {
m.RemoveFn = func(ctx context.Context, h restic.Handle) error {
m.RemoveFn = func(ctx context.Context, h backend.Handle) error {
wait()
return nil
}
}, func(be restic.Backend) func() error {
}, func(be backend.Backend) func() error {
return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
return be.Remove(context.TODO(), h)
}
}, unblock, false)
@ -187,13 +187,13 @@ func TestConcurrencyLimitDelete(t *testing.T) {
func TestConcurrencyUnlimitedLockSave(t *testing.T) {
wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) {
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
wait()
return nil
}
}, func(be restic.Backend) func() error {
}, func(be backend.Backend) func() error {
return func() error {
h := restic.Handle{Type: restic.LockFile, Name: "foobar"}
h := backend.Handle{Type: backend.LockFile, Name: "foobar"}
return be.Save(context.TODO(), h, nil)
}
}, unblock, true)
@ -202,13 +202,13 @@ func TestConcurrencyUnlimitedLockSave(t *testing.T) {
func TestFreeze(t *testing.T) {
var counter int64
m := mock.NewBackend()
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
atomic.AddInt64(&counter, 1)
return nil
}
m.ConnectionsFn = func() uint { return 2 }
be := sema.NewBackend(m)
fb := be.(restic.FreezeBackend)
fb := be.(backend.FreezeBackend)
// Freeze backend
fb.Freeze()
@ -218,7 +218,7 @@ func TestFreeze(t *testing.T) {
wg.Add(1)
go func() {
defer wg.Done()
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
test.OK(t, be.Save(context.TODO(), h, nil))
}()

4
internal/backend/sftp/layout_test.go
View File

@ -6,8 +6,8 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/sftp"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -56,7 +56,7 @@ func TestLayout(t *testing.T) {
}
packs := make(map[string]bool)
err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
packs[fi.Name] = false
return nil
})

27
internal/backend/sftp/sftp.go
View File

@ -20,7 +20,6 @@ import (
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/cenkalti/backoff/v4"
"github.com/pkg/sftp"
@ -42,7 +41,7 @@ type SFTP struct {
util.Modes
}
var _ restic.Backend = &SFTP{}
var _ backend.Backend = &SFTP{}
func NewFactory() location.Factory {
return location.NewLimitedBackendFactory("sftp", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open))
@ -153,7 +152,7 @@ func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) {
debug.Log("layout: %v\n", sftp.Layout)
fi, err := sftp.c.Stat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile}))
fi, err := sftp.c.Stat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile}))
m := util.DeriveModesFromFileInfo(fi, err)
debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir)
@ -263,7 +262,7 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) {
sftp.Modes = util.DefaultModes
// test if config file already exists
_, err = sftp.c.Lstat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile}))
_, err = sftp.c.Lstat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile}))
if err == nil {
return nil, errors.New("config file already exists")
}
@ -314,7 +313,7 @@ func tempSuffix() string {
}
// Save stores data in the backend at the handle.
func (r *SFTP) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) error {
func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) error {
if err := r.clientError(); err != nil {
return err
}
@ -414,11 +413,11 @@ func (r *SFTP) checkNoSpace(dir string, size int64, origErr error) error {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (r *SFTP) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (r *SFTP) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, r.openReader, fn)
}
func (r *SFTP) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (r *SFTP) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
f, err := r.c.Open(r.Filename(h))
if err != nil {
return nil, err
@ -442,21 +441,21 @@ func (r *SFTP) openReader(_ context.Context, h restic.Handle, length int, offset
}
// Stat returns information about a blob.
func (r *SFTP) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) {
func (r *SFTP) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) {
if err := r.clientError(); err != nil {
return restic.FileInfo{}, err
return backend.FileInfo{}, err
}
fi, err := r.c.Lstat(r.Filename(h))
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Lstat")
return backend.FileInfo{}, errors.Wrap(err, "Lstat")
}
return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil
return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil
}
// Remove removes the content stored at name.
func (r *SFTP) Remove(_ context.Context, h restic.Handle) error {
func (r *SFTP) Remove(_ context.Context, h backend.Handle) error {
if err := r.clientError(); err != nil {
return err
}
@ -466,7 +465,7 @@ func (r *SFTP) Remove(_ context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (r *SFTP) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
basedir, subdirs := r.Basedir(t)
walker := r.c.Walk(basedir)
for {
@ -499,7 +498,7 @@ func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileI
debug.Log("send %v\n", path.Base(walker.Path()))
rfi := restic.FileInfo{
rfi := backend.FileInfo{
Name: path.Base(walker.Path()),
Size: fi.Size(),
}

4
internal/backend/swift/config.go
View File

@ -4,9 +4,9 @@ import (
"os"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains basic configuration needed to specify swift location for a swift server
@ -74,7 +74,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {

26
internal/backend/swift/swift.go
View File

@ -13,12 +13,12 @@ import (
"strings"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/ncw/swift/v2"
)
@ -32,8 +32,8 @@ type beSwift struct {
layout.Layout
}
// ensure statically that *beSwift implements restic.Backend.
var _ restic.Backend = &beSwift{}
// ensure statically that *beSwift implements backend.Backend.
var _ backend.Backend = &beSwift{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("swift", ParseConfig, location.NoPassword, Open, Open)
@ -41,7 +41,7 @@ func NewFactory() location.Factory {
// Open opens the swift backend at a container in region. The container is
// created if it does not exist yet.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
debug.Log("config %#v", cfg)
be := &beSwift{
@ -134,11 +134,11 @@ func (be *beSwift) HasAtomicReplace() bool {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *beSwift) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *beSwift) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
objName := be.Filename(h)
@ -160,7 +160,7 @@ func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int,
}
// Save stores data in the backend at the handle.
func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *beSwift) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
encoding := "binary/octet-stream"
@ -174,19 +174,19 @@ func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
}
// Stat returns information about a blob.
func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
func (be *beSwift) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h)
obj, _, err := be.conn.Object(ctx, be.container, objName)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "conn.Object")
return backend.FileInfo{}, errors.Wrap(err, "conn.Object")
}
return restic.FileInfo{Size: obj.Bytes, Name: h.Name}, nil
return backend.FileInfo{Size: obj.Bytes, Name: h.Name}, nil
}
// Remove removes the blob with the given name and type.
func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error {
func (be *beSwift) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
err := be.conn.ObjectDelete(ctx, be.container, objName)
@ -195,7 +195,7 @@ func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *beSwift) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, _ := be.Basedir(t)
prefix += "/"
@ -212,7 +212,7 @@ func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: m,
Size: obj.Bytes,
}

4
internal/backend/swift/swift_test.go
View File

@ -6,9 +6,9 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/swift"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -20,7 +20,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] {
// wait for removals for at least 5m
WaitForDelayedRemoval: 5 * time.Minute,
ErrorHandler: func(t testing.TB, be restic.Backend, err error) error {
ErrorHandler: func(t testing.TB, be backend.Backend, err error) error {
if err == nil {
return nil
}

13
internal/backend/test/benchmarks.go
View File

@ -6,22 +6,23 @@ import (
"io"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
func saveRandomFile(t testing.TB, be restic.Backend, length int) ([]byte, restic.Handle) {
func saveRandomFile(t testing.TB, be backend.Backend, length int) ([]byte, backend.Handle) {
data := test.Random(23, length)
id := restic.Hash(data)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := be.Save(context.TODO(), handle, restic.NewByteReader(data, be.Hasher()))
handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := be.Save(context.TODO(), handle, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatalf("Save() error: %+v", err)
}
return data, handle
}
func remove(t testing.TB, be restic.Backend, h restic.Handle) {
func remove(t testing.TB, be backend.Backend, h backend.Handle) {
if err := be.Remove(context.TODO(), h); err != nil {
t.Fatalf("Remove() returned error: %v", err)
}
@ -146,9 +147,9 @@ func (s *Suite[C]) BenchmarkSave(t *testing.B) {
length := 1<<24 + 2123
data := test.Random(23, length)
id := restic.Hash(data)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
rd := restic.NewByteReader(data, be.Hasher())
rd := backend.NewByteReader(data, be.Hasher())
t.SetBytes(int64(length))
t.ResetTimer()

2
internal/backend/test/doc.go
View File

@ -17,7 +17,7 @@
//
// func newTestSuite(t testing.TB) *test.Suite {
// return &test.Suite{
// Create: func(cfg interface{}) (restic.Backend, error) {
// Create: func(cfg interface{}) (backend.Backend, error) {
// [...]
// },
// [...]

13
internal/backend/test/suite.go
View File

@ -11,7 +11,6 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
@ -35,7 +34,7 @@ type Suite[C any] struct {
WaitForDelayedRemoval time.Duration
// ErrorHandler allows ignoring certain errors.
ErrorHandler func(testing.TB, restic.Backend, error) error
ErrorHandler func(testing.TB, backend.Backend, error) error
}
// RunTests executes all defined tests as subtests of t.
@ -156,7 +155,7 @@ func (s *Suite[C]) RunBenchmarks(b *testing.B) {
s.cleanup(b)
}
func (s *Suite[C]) createOrError() (restic.Backend, error) {
func (s *Suite[C]) createOrError() (backend.Backend, error) {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
return nil, fmt.Errorf("cannot create transport for tests: %v", err)
@ -167,7 +166,7 @@ func (s *Suite[C]) createOrError() (restic.Backend, error) {
return nil, err
}
_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
_, err = be.Stat(context.TODO(), backend.Handle{Type: backend.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
@ -179,7 +178,7 @@ func (s *Suite[C]) createOrError() (restic.Backend, error) {
return be, nil
}
func (s *Suite[C]) create(t testing.TB) restic.Backend {
func (s *Suite[C]) create(t testing.TB) backend.Backend {
be, err := s.createOrError()
if err != nil {
t.Fatal(err)
@ -187,7 +186,7 @@ func (s *Suite[C]) create(t testing.TB) restic.Backend {
return be
}
func (s *Suite[C]) open(t testing.TB) restic.Backend {
func (s *Suite[C]) open(t testing.TB) backend.Backend {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
@ -208,7 +207,7 @@ func (s *Suite[C]) cleanup(t testing.TB) {
s.close(t, be)
}
func (s *Suite[C]) close(t testing.TB, be restic.Backend) {
func (s *Suite[C]) close(t testing.TB, be backend.Backend) {
err := be.Close()
if err != nil {
t.Fatal(err)

100
internal/backend/test/tests.go
View File

@ -27,7 +27,7 @@ func seedRand(t testing.TB) {
t.Logf("rand initialized with seed %d", seed)
}
func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, error) {
func beTest(ctx context.Context, be backend.Backend, h backend.Handle) (bool, error) {
_, err := be.Stat(ctx, h)
if err != nil && be.IsNotExist(err) {
return false, nil
@ -49,7 +49,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
defer s.close(t, b)
// remove a config if present
cfgHandle := restic.Handle{Type: restic.ConfigFile}
cfgHandle := backend.Handle{Type: backend.ConfigFile}
cfgPresent, err := beTest(context.TODO(), b, cfgHandle)
if err != nil {
t.Fatalf("unable to test for config: %+v", err)
@ -60,7 +60,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
}
// save a config
store(t, b, restic.ConfigFile, []byte("test config"))
store(t, b, backend.ConfigFile, []byte("test config"))
// now create the backend again, this must fail
_, err = s.createOrError()
@ -69,7 +69,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
}
// remove config
err = b.Remove(context.TODO(), restic.Handle{Type: restic.ConfigFile, Name: ""})
err = b.Remove(context.TODO(), backend.Handle{Type: backend.ConfigFile, Name: ""})
if err != nil {
t.Fatalf("unexpected error removing config: %+v", err)
}
@ -94,13 +94,13 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
var testString = "Config"
// create config and read it back
_, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.ConfigFile})
_, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.ConfigFile})
if err == nil {
t.Fatalf("did not get expected error for non-existing config")
}
test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize error from LoadAll(): %v", err)
err = b.Save(context.TODO(), restic.Handle{Type: restic.ConfigFile}, restic.NewByteReader([]byte(testString), b.Hasher()))
err = b.Save(context.TODO(), backend.Handle{Type: backend.ConfigFile}, backend.NewByteReader([]byte(testString), b.Hasher()))
if err != nil {
t.Fatalf("Save() error: %+v", err)
}
@ -108,7 +108,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
// try accessing the config with different names, should all return the
// same config
for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
h := restic.Handle{Type: restic.ConfigFile, Name: name}
h := backend.Handle{Type: backend.ConfigFile, Name: name}
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
if err != nil {
t.Fatalf("unable to read config with name %q: %+v", name, err)
@ -120,7 +120,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
}
// remove the config
remove(t, b, restic.Handle{Type: restic.ConfigFile})
remove(t, b, backend.Handle{Type: backend.ConfigFile})
}
// TestLoad tests the backend's Load function.
@ -130,7 +130,7 @@ func (s *Suite[C]) TestLoad(t *testing.T) {
b := s.open(t)
defer s.close(t, b)
err := testLoad(b, restic.Handle{Type: restic.PackFile, Name: "foobar"})
err := testLoad(b, backend.Handle{Type: backend.PackFile, Name: "foobar"})
if err == nil {
t.Fatalf("Load() did not return an error for non-existing blob")
}
@ -141,8 +141,8 @@ func (s *Suite[C]) TestLoad(t *testing.T) {
data := test.Random(23, length)
id := restic.Hash(data)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
err = b.Save(context.TODO(), handle, restic.NewByteReader(data, b.Hasher()))
handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
err = b.Save(context.TODO(), handle, backend.NewByteReader(data, b.Hasher()))
if err != nil {
t.Fatalf("Save() error: %+v", err)
}
@ -243,7 +243,7 @@ func (s *Suite[C]) TestList(t *testing.T) {
// Check that the backend is empty to start with
var found []string
err := b.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
found = append(found, fi.Name)
return nil
})
@ -259,8 +259,8 @@ func (s *Suite[C]) TestList(t *testing.T) {
for i := 0; i < numTestFiles; i++ {
data := test.Random(rand.Int(), rand.Intn(100)+55)
id := restic.Hash(data)
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher()))
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
if err != nil {
t.Fatal(err)
}
@ -284,7 +284,7 @@ func (s *Suite[C]) TestList(t *testing.T) {
s.SetListMaxItems(test.maxItems)
}
err := b.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
id, err := restic.ParseID(fi.Name)
if err != nil {
t.Fatal(err)
@ -320,9 +320,9 @@ func (s *Suite[C]) TestList(t *testing.T) {
}
t.Logf("remove %d files", numTestFiles)
handles := make([]restic.Handle, 0, len(list1))
handles := make([]backend.Handle, 0, len(list1))
for id := range list1 {
handles = append(handles, restic.Handle{Type: restic.PackFile, Name: id.String()})
handles = append(handles, backend.Handle{Type: backend.PackFile, Name: id.String()})
}
err = s.delayedRemove(t, b, handles...)
@ -340,13 +340,13 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
b := s.open(t)
defer s.close(t, b)
testFiles := make([]restic.Handle, 0, numTestFiles)
testFiles := make([]backend.Handle, 0, numTestFiles)
for i := 0; i < numTestFiles; i++ {
data := []byte(fmt.Sprintf("random test blob %v", i))
id := restic.Hash(data)
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher()))
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
if err != nil {
t.Fatal(err)
}
@ -358,7 +358,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
cancel()
// pass in a cancelled context
err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
t.Errorf("got FileInfo %v for cancelled context", fi)
return nil
})
@ -373,7 +373,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
defer cancel()
i := 0
err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
i++
// cancel the context on the first file
if i == 1 {
@ -396,7 +396,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
defer cancel()
i := 0
err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
// cancel the context at the last file
i++
if i == numTestFiles {
@ -423,7 +423,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
i := 0
// pass in a context with a timeout
err := b.List(ctxTimeout, restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(ctxTimeout, backend.PackFile, func(fi backend.FileInfo) error {
i++
// wait until the context is cancelled
@ -494,11 +494,11 @@ func (s *Suite[C]) TestSave(t *testing.T) {
data := test.Random(23, length)
id = sha256.Sum256(data)
h := restic.Handle{
Type: restic.PackFile,
h := backend.Handle{
Type: backend.PackFile,
Name: id.String(),
}
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher()))
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
test.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
@ -546,7 +546,7 @@ func (s *Suite[C]) TestSave(t *testing.T) {
t.Fatal(err)
}
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
// wrap the tempfile in an errorCloser, so we can detect if the backend
// closes the reader
@ -585,7 +585,7 @@ func (s *Suite[C]) TestSave(t *testing.T) {
}
type incompleteByteReader struct {
restic.ByteReader
backend.ByteReader
}
func (r *incompleteByteReader) Length() int64 {
@ -609,8 +609,8 @@ func (s *Suite[C]) TestSaveError(t *testing.T) {
copy(id[:], data)
// test that incomplete uploads fail
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &incompleteByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())})
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &incompleteByteReader{ByteReader: *backend.NewByteReader(data, b.Hasher())})
// try to delete possible leftovers
_ = s.delayedRemove(t, b, h)
if err == nil {
@ -619,7 +619,7 @@ func (s *Suite[C]) TestSaveError(t *testing.T) {
}
type wrongByteReader struct {
restic.ByteReader
backend.ByteReader
}
func (b *wrongByteReader) Hash() []byte {
@ -648,8 +648,8 @@ func (s *Suite[C]) TestSaveWrongHash(t *testing.T) {
copy(id[:], data)
// test that upload with hash mismatch fails
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())})
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *backend.NewByteReader(data, b.Hasher())})
exists, err2 := beTest(context.TODO(), b, h)
if err2 != nil {
t.Fatal(err2)
@ -674,23 +674,23 @@ var testStrings = []struct {
{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
}
func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) restic.Handle {
func store(t testing.TB, b backend.Backend, tpe backend.FileType, data []byte) backend.Handle {
id := restic.Hash(data)
h := restic.Handle{Name: id.String(), Type: tpe}
err := b.Save(context.TODO(), h, restic.NewByteReader([]byte(data), b.Hasher()))
h := backend.Handle{Name: id.String(), Type: tpe}
err := b.Save(context.TODO(), h, backend.NewByteReader([]byte(data), b.Hasher()))
test.OK(t, err)
return h
}
// testLoad loads a blob (but discards its contents).
func testLoad(b restic.Backend, h restic.Handle) error {
func testLoad(b backend.Backend, h backend.Handle) error {
return b.Load(context.TODO(), h, 0, 0, func(rd io.Reader) (ierr error) {
_, ierr = io.Copy(io.Discard, rd)
return ierr
})
}
func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error {
func (s *Suite[C]) delayedRemove(t testing.TB, be backend.Backend, handles ...backend.Handle) error {
// Some backend (swift, I'm looking at you) may implement delayed
// removal of data. Let's wait a bit if this happens.
@ -734,11 +734,11 @@ func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...res
return nil
}
func delayedList(t testing.TB, b restic.Backend, tpe restic.FileType, max int, maxwait time.Duration) restic.IDs {
func delayedList(t testing.TB, b backend.Backend, tpe backend.FileType, max int, maxwait time.Duration) restic.IDs {
list := restic.NewIDSet()
start := time.Now()
for i := 0; i < max; i++ {
err := b.List(context.TODO(), tpe, func(fi restic.FileInfo) error {
err := b.List(context.TODO(), tpe, func(fi backend.FileInfo) error {
id := restic.TestParseID(fi.Name)
list.Insert(id)
return nil
@ -763,9 +763,9 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
test.Assert(t, !b.IsNotExist(nil), "IsNotExist() recognized nil error")
for _, tpe := range []restic.FileType{
restic.PackFile, restic.KeyFile, restic.LockFile,
restic.SnapshotFile, restic.IndexFile,
for _, tpe := range []backend.FileType{
backend.PackFile, backend.KeyFile, backend.LockFile,
backend.SnapshotFile, backend.IndexFile,
} {
// detect non-existing files
for _, ts := range testStrings {
@ -773,7 +773,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
test.OK(t, err)
// test if blob is already in repository
h := restic.Handle{Type: tpe, Name: id.String()}
h := backend.Handle{Type: tpe, Name: id.String()}
ret, err := beTest(context.TODO(), b, h)
test.OK(t, err)
test.Assert(t, !ret, "blob was found to exist before creating")
@ -799,7 +799,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
store(t, b, tpe, []byte(ts.data))
// test Load()
h := restic.Handle{Type: tpe, Name: ts.id}
h := backend.Handle{Type: tpe, Name: ts.id}
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
test.OK(t, err)
test.Equals(t, ts.data, string(buf))
@ -823,7 +823,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
// test adding the first file again
ts := testStrings[0]
h := restic.Handle{Type: tpe, Name: ts.id}
h := backend.Handle{Type: tpe, Name: ts.id}
// remove and recreate
err := s.delayedRemove(t, b, h)
@ -835,7 +835,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
test.Assert(t, !ok, "removed blob still present")
// create blob
err = b.Save(context.TODO(), h, restic.NewByteReader([]byte(ts.data), b.Hasher()))
err = b.Save(context.TODO(), h, backend.NewByteReader([]byte(ts.data), b.Hasher()))
test.OK(t, err)
// list items
@ -859,12 +859,12 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list)
}
var handles []restic.Handle
var handles []backend.Handle
for _, ts := range testStrings {
id, err := restic.ParseID(ts.id)
test.OK(t, err)
h := restic.Handle{Type: tpe, Name: id.String()}
h := backend.Handle{Type: tpe, Name: id.String()}
found, err := beTest(context.TODO(), b, h)
test.OK(t, err)

26
internal/backend/util/defaults.go
View File

@ -4,12 +4,12 @@ import (
"context"
"io"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
)
// DefaultLoad implements Backend.Load using lower-level openReader func
func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64,
openReader func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error),
func DefaultLoad(ctx context.Context, h backend.Handle, length int, offset int64,
openReader func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error),
fn func(rd io.Reader) error) error {
rd, err := openReader(ctx, h, length, offset)
@ -25,23 +25,23 @@ func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64,
}
// DefaultDelete removes all restic keys in the bucket. It will not remove the bucket itself.
func DefaultDelete(ctx context.Context, be restic.Backend) error {
alltypes := []restic.FileType{
restic.PackFile,
restic.KeyFile,
restic.LockFile,
restic.SnapshotFile,
restic.IndexFile}
func DefaultDelete(ctx context.Context, be backend.Backend) error {
alltypes := []backend.FileType{
backend.PackFile,
backend.KeyFile,
backend.LockFile,
backend.SnapshotFile,
backend.IndexFile}
for _, t := range alltypes {
err := be.List(ctx, t, func(fi restic.FileInfo) error {
return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name})
err := be.List(ctx, t, func(fi backend.FileInfo) error {
return be.Remove(ctx, backend.Handle{Type: t, Name: fi.Name})
})
if err != nil {
return nil
}
}
err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
err := be.Remove(ctx, backend.Handle{Type: backend.ConfigFile})
if err != nil && be.IsNotExist(err) {
err = nil
}

10
internal/backend/util/defaults_test.go
View File

@ -5,9 +5,9 @@ import (
"io"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -26,11 +26,11 @@ func (rd *mockReader) Close() error {
func TestDefaultLoad(t *testing.T) {
h := restic.Handle{Name: "id", Type: restic.PackFile}
h := backend.Handle{Name: "id", Type: backend.PackFile}
rd := &mockReader{}
// happy case, assert correct parameters are passed around and content stream is closed
err := util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) {
err := util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) {
rtest.Equals(t, h, ih)
rtest.Equals(t, int(10), length)
rtest.Equals(t, int64(11), offset)
@ -44,7 +44,7 @@ func TestDefaultLoad(t *testing.T) {
rtest.Equals(t, true, rd.closed)
// unhappy case, assert producer errors are handled correctly
err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) {
err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return nil, errors.Errorf("producer error")
}, func(ird io.Reader) error {
t.Fatalf("unexpected consumer invocation")
@ -54,7 +54,7 @@ func TestDefaultLoad(t *testing.T) {
// unhappy case, assert consumer errors are handled correctly
rd = &mockReader{}
err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) {
err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return rd, nil
}, func(ird io.Reader) error {
return errors.Errorf("consumer error")

17
internal/backend/utils.go
View File

@ -11,7 +11,6 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
func verifyContentMatchesName(s string, data []byte) (bool, error) {
@ -33,7 +32,7 @@ func verifyContentMatchesName(s string, data []byte) (bool, error) {
// LoadAll reads all data stored in the backend for the handle into the given
// buffer, which is truncated. If the buffer is not large enough or nil, a new
// one is allocated.
func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle) ([]byte, error) {
func LoadAll(ctx context.Context, buf []byte, be Backend, h Handle) ([]byte, error) {
retriedInvalidData := false
err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
// make sure this is idempotent, in case an error occurs this function may be called multiple times!
@ -47,7 +46,7 @@ func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle
// retry loading damaged data only once. If a file fails to download correctly
// the second time, then it is likely corrupted at the backend. Return the data
// to the caller in that case to let it decide what to do with the data.
if !retriedInvalidData && h.Type != restic.ConfigFile {
if !retriedInvalidData && h.Type != ConfigFile {
if matches, err := verifyContentMatchesName(h.Name, buf); err == nil && !matches {
debug.Log("retry loading broken blob %v", h)
retriedInvalidData = true
@ -77,11 +76,11 @@ func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
}
type memorizedLister struct {
fileInfos []restic.FileInfo
tpe restic.FileType
fileInfos []FileInfo
tpe FileType
}
func (m *memorizedLister) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (m *memorizedLister) List(ctx context.Context, t FileType, fn func(FileInfo) error) error {
if t != m.tpe {
return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t)
}
@ -97,13 +96,13 @@ func (m *memorizedLister) List(ctx context.Context, t restic.FileType, fn func(r
return ctx.Err()
}
func MemorizeList(ctx context.Context, be restic.Lister, t restic.FileType) (restic.Lister, error) {
func MemorizeList(ctx context.Context, be Lister, t FileType) (Lister, error) {
if _, ok := be.(*memorizedLister); ok {
return be, nil
}
var fileInfos []restic.FileInfo
err := be.List(ctx, t, func(fi restic.FileInfo) error {
var fileInfos []FileInfo
err := be.List(ctx, t, func(fi FileInfo) error {
fileInfos = append(fileInfos, fi)
return nil
})

40
internal/backend/utils_test.go
View File

@ -26,11 +26,11 @@ func TestLoadAll(t *testing.T) {
data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB)
id := restic.Hash(data)
h := restic.Handle{Name: id.String(), Type: restic.PackFile}
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher()))
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
rtest.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), buf, b, restic.Handle{Type: restic.PackFile, Name: id.String()})
buf, err := backend.LoadAll(context.TODO(), buf, b, backend.Handle{Type: backend.PackFile, Name: id.String()})
rtest.OK(t, err)
if len(buf) != len(data) {
@ -45,10 +45,10 @@ func TestLoadAll(t *testing.T) {
}
}
func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle {
func save(t testing.TB, be backend.Backend, buf []byte) backend.Handle {
id := restic.Hash(buf)
h := restic.Handle{Name: id.String(), Type: restic.PackFile}
err := be.Save(context.TODO(), h, restic.NewByteReader(buf, be.Hasher()))
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
err := be.Save(context.TODO(), h, backend.NewByteReader(buf, be.Hasher()))
if err != nil {
t.Fatal(err)
}
@ -56,10 +56,10 @@ func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle {
}
type quickRetryBackend struct {
restic.Backend
backend.Backend
}
func (be *quickRetryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *quickRetryBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
err := be.Backend.Load(ctx, h, length, offset, fn)
if err != nil {
// retry
@ -76,19 +76,19 @@ func TestLoadAllBroken(t *testing.T) {
// damage buffer
data[0] ^= 0xff
b.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
b.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(data)), nil
}
// must fail on first try
_, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.PackFile, Name: id.String()})
_, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.PackFile, Name: id.String()})
if err == nil {
t.Fatalf("missing expected error")
}
// must return the broken data after a retry
be := &quickRetryBackend{Backend: b}
buf, err := backend.LoadAll(context.TODO(), nil, be, restic.Handle{Type: restic.PackFile, Name: id.String()})
buf, err := backend.LoadAll(context.TODO(), nil, be, backend.Handle{Type: backend.PackFile, Name: id.String()})
rtest.OK(t, err)
if !bytes.Equal(buf, data) {
@ -104,7 +104,7 @@ func TestLoadAllAppend(t *testing.T) {
h2 := save(t, b, randomData)
var tests = []struct {
handle restic.Handle
handle backend.Handle
buf []byte
want []byte
}{
@ -152,11 +152,11 @@ func TestLoadAllAppend(t *testing.T) {
func TestMemoizeList(t *testing.T) {
// setup backend to serve as data source for memoized list
be := mock.NewBackend()
files := []restic.FileInfo{
files := []backend.FileInfo{
{Size: 42, Name: restic.NewRandomID().String()},
{Size: 45, Name: restic.NewRandomID().String()},
}
be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
be.ListFn = func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
for _, fi := range files {
if err := fn(fi); err != nil {
return err
@ -165,17 +165,17 @@ func TestMemoizeList(t *testing.T) {
return nil
}
mem, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile)
mem, err := backend.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.OK(t, err)
err = mem.List(context.TODO(), restic.IndexFile, func(fi restic.FileInfo) error {
err = mem.List(context.TODO(), backend.IndexFile, func(fi backend.FileInfo) error {
t.Fatal("file type mismatch")
return nil // the memoized lister must return an error by itself
})
rtest.Assert(t, err != nil, "missing error on file typ mismatch")
var memFiles []restic.FileInfo
err = mem.List(context.TODO(), restic.SnapshotFile, func(fi restic.FileInfo) error {
var memFiles []backend.FileInfo
err = mem.List(context.TODO(), backend.SnapshotFile, func(fi backend.FileInfo) error {
memFiles = append(memFiles, fi)
return nil
})
@ -186,9 +186,9 @@ func TestMemoizeList(t *testing.T) {
func TestMemoizeListError(t *testing.T) {
// setup backend to serve as data source for memoized list
be := mock.NewBackend()
be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
be.ListFn = func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
return fmt.Errorf("list error")
}
_, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile)
_, err := backend.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.Assert(t, err != nil, "missing error on list error")
}

34
internal/cache/backend.go vendored
View File

@ -5,35 +5,35 @@ import (
"io"
"sync"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
// Backend wraps a restic.Backend and adds a cache.
type Backend struct {
restic.Backend
backend.Backend
*Cache
// inProgress contains the handle for all files that are currently
// downloaded. The channel in the value is closed as soon as the download
// is finished.
inProgressMutex sync.Mutex
inProgress map[restic.Handle]chan struct{}
inProgress map[backend.Handle]chan struct{}
}
// ensure Backend implements restic.Backend
var _ restic.Backend = &Backend{}
// ensure Backend implements backend.Backend
var _ backend.Backend = &Backend{}
func newBackend(be restic.Backend, c *Cache) *Backend {
func newBackend(be backend.Backend, c *Cache) *Backend {
return &Backend{
Backend: be,
Cache: c,
inProgress: make(map[restic.Handle]chan struct{}),
inProgress: make(map[backend.Handle]chan struct{}),
}
}
// Remove deletes a file from the backend and the cache if it has been cached.
func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (b *Backend) Remove(ctx context.Context, h backend.Handle) error {
debug.Log("cache Remove(%v)", h)
err := b.Backend.Remove(ctx, h)
if err != nil {
@ -43,18 +43,18 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
return b.Cache.remove(h)
}
func autoCacheTypes(h restic.Handle) bool {
func autoCacheTypes(h backend.Handle) bool {
switch h.Type {
case restic.IndexFile, restic.SnapshotFile:
case backend.IndexFile, backend.SnapshotFile:
return true
case restic.PackFile:
case backend.PackFile:
return h.IsMetadata
}
return false
}
// Save stores a new file in the backend and the cache.
func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if !autoCacheTypes(h) {
return b.Backend.Save(ctx, h, rd)
}
@ -89,7 +89,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
return nil
}
func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error {
finish := make(chan struct{})
b.inProgressMutex.Lock()
@ -133,7 +133,7 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
}
// loadFromCache will try to load the file from the cache.
func (b *Backend) loadFromCache(h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
rd, err := b.Cache.load(h, length, offset)
if err != nil {
return false, err
@ -148,7 +148,7 @@ func (b *Backend) loadFromCache(h restic.Handle, length int, offset int64, consu
}
// Load loads a file from the cache or the backend.
func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
b.inProgressMutex.Lock()
waitForFinish, inProgress := b.inProgress[h]
b.inProgressMutex.Unlock()
@ -194,7 +194,7 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
// Stat tests whether the backend has a file. If it does not exist but still
// exists in the cache, it is removed from the cache.
func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
debug.Log("cache Stat(%v)", h)
fi, err := b.Backend.Stat(ctx, h)
@ -215,6 +215,6 @@ func (b *Backend) IsNotExist(err error) bool {
return b.Backend.IsNotExist(err)
}
func (b *Backend) Unwrap() restic.Backend {
func (b *Backend) Unwrap() backend.Backend {
return b.Backend
}

20
internal/cache/backend_test.go vendored
View File

@ -16,7 +16,7 @@ import (
"github.com/restic/restic/internal/test"
)
func loadAndCompare(t testing.TB, be restic.Backend, h restic.Handle, data []byte) {
func loadAndCompare(t testing.TB, be backend.Backend, h backend.Handle, data []byte) {
buf, err := backend.LoadAll(context.TODO(), nil, be, h)
if err != nil {
t.Fatal(err)
@ -31,25 +31,25 @@ func loadAndCompare(t testing.TB, be restic.Backend, h restic.Handle, data []byt
}
}
func save(t testing.TB, be restic.Backend, h restic.Handle, data []byte) {
err := be.Save(context.TODO(), h, restic.NewByteReader(data, be.Hasher()))
func save(t testing.TB, be backend.Backend, h backend.Handle, data []byte) {
err := be.Save(context.TODO(), h, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatal(err)
}
}
func remove(t testing.TB, be restic.Backend, h restic.Handle) {
func remove(t testing.TB, be backend.Backend, h backend.Handle) {
err := be.Remove(context.TODO(), h)
if err != nil {
t.Fatal(err)
}
}
func randomData(n int) (restic.Handle, []byte) {
func randomData(n int) (backend.Handle, []byte) {
data := test.Random(rand.Int(), n)
id := restic.Hash(data)
h := restic.Handle{
Type: restic.IndexFile,
h := backend.Handle{
Type: backend.IndexFile,
Name: id.String(),
}
return h, data
@ -114,11 +114,11 @@ func TestBackend(t *testing.T) {
}
type loadErrorBackend struct {
restic.Backend
backend.Backend
loadError error
}
func (be loadErrorBackend) Load(_ context.Context, _ restic.Handle, _ int, _ int64, _ func(rd io.Reader) error) error {
func (be loadErrorBackend) Load(_ context.Context, _ backend.Handle, _ int, _ int64, _ func(rd io.Reader) error) error {
time.Sleep(10 * time.Millisecond)
return be.loadError
}
@ -137,7 +137,7 @@ func TestErrorBackend(t *testing.T) {
loadError: testErr,
}
loadTest := func(wg *sync.WaitGroup, be restic.Backend) {
loadTest := func(wg *sync.WaitGroup, be backend.Backend) {
defer wg.Done()
buf, err := backend.LoadAll(context.TODO(), nil, be, h)

3
internal/cache/cache.go vendored
View File

@ -9,6 +9,7 @@ import (
"time"
"github.com/pkg/errors"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
@ -234,7 +235,7 @@ func IsOld(t time.Time, maxAge time.Duration) bool {
}
// Wrap returns a backend with a cache.
func (c *Cache) Wrap(be restic.Backend) restic.Backend {
func (c *Cache) Wrap(be backend.Backend) backend.Backend {
return newBackend(be, c)
}

14
internal/cache/file.go vendored
View File

@ -14,7 +14,7 @@ import (
"github.com/restic/restic/internal/restic"
)
func (c *Cache) filename(h restic.Handle) string {
func (c *Cache) filename(h backend.Handle) string {
if len(h.Name) < 2 {
panic("Name is empty or too short")
}
@ -22,7 +22,7 @@ func (c *Cache) filename(h restic.Handle) string {
return filepath.Join(c.path, cacheLayoutPaths[h.Type], subdir, h.Name)
}
func (c *Cache) canBeCached(t restic.FileType) bool {
func (c *Cache) canBeCached(t backend.FileType) bool {
if c == nil {
return false
}
@ -34,7 +34,7 @@ func (c *Cache) canBeCached(t restic.FileType) bool {
// Load returns a reader that yields the contents of the file with the
// given handle. rd must be closed after use. If an error is returned, the
// ReadCloser is nil.
func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (c *Cache) load(h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
debug.Log("Load(%v, %v, %v) from cache", h, length, offset)
if !c.canBeCached(h.Type) {
return nil, errors.New("cannot be cached")
@ -78,7 +78,7 @@ func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser,
}
// Save saves a file in the cache.
func (c *Cache) Save(h restic.Handle, rd io.Reader) error {
func (c *Cache) Save(h backend.Handle, rd io.Reader) error {
debug.Log("Save to cache: %v", h)
if rd == nil {
return errors.New("Save() called with nil reader")
@ -139,7 +139,7 @@ func (c *Cache) Save(h restic.Handle, rd io.Reader) error {
}
// Remove deletes a file. When the file is not cache, no error is returned.
func (c *Cache) remove(h restic.Handle) error {
func (c *Cache) remove(h backend.Handle) error {
if !c.Has(h) {
return nil
}
@ -165,7 +165,7 @@ func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
continue
}
if err = fs.Remove(c.filename(restic.Handle{Type: t, Name: id.String()})); err != nil {
if err = fs.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil {
return err
}
}
@ -207,7 +207,7 @@ func (c *Cache) list(t restic.FileType) (restic.IDSet, error) {
}
// Has returns true if the file is cached.
func (c *Cache) Has(h restic.Handle) bool {
func (c *Cache) Has(h backend.Handle) bool {
if !c.canBeCached(h.Type) {
return false
}

15
internal/cache/file_test.go vendored
View File

@ -10,6 +10,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
@ -18,12 +19,12 @@ import (
"golang.org/x/sync/errgroup"
)
func generateRandomFiles(t testing.TB, tpe restic.FileType, c *Cache) restic.IDSet {
func generateRandomFiles(t testing.TB, tpe backend.FileType, c *Cache) restic.IDSet {
ids := restic.NewIDSet()
for i := 0; i < rand.Intn(15)+10; i++ {
buf := test.Random(rand.Int(), 1<<19)
id := restic.Hash(buf)
h := restic.Handle{Type: tpe, Name: id.String()}
h := backend.Handle{Type: tpe, Name: id.String()}
if c.Has(h) {
t.Errorf("index %v present before save", id)
@ -46,7 +47,7 @@ func randomID(s restic.IDSet) restic.ID {
panic("set is empty")
}
func load(t testing.TB, c *Cache, h restic.Handle) []byte {
func load(t testing.TB, c *Cache, h backend.Handle) []byte {
rd, err := c.load(h, 0, 0)
if err != nil {
t.Fatal(err)
@ -101,7 +102,7 @@ func TestFiles(t *testing.T) {
ids := generateRandomFiles(t, tpe, c)
id := randomID(ids)
h := restic.Handle{Type: tpe, Name: id.String()}
h := backend.Handle{Type: tpe, Name: id.String()}
id2 := restic.Hash(load(t, c, h))
if !id.Equal(id2) {
@ -146,7 +147,7 @@ func TestFileLoad(t *testing.T) {
data := test.Random(rand.Int(), 5234142)
id := restic.ID{}
copy(id[:], data)
h := restic.Handle{
h := backend.Handle{
Type: restic.PackFile,
Name: id.String(),
}
@ -230,7 +231,7 @@ func TestFileSaveConcurrent(t *testing.T) {
)
rand.Read(id[:])
h := restic.Handle{
h := backend.Handle{
Type: restic.PackFile,
Name: id.String(),
}
@ -275,7 +276,7 @@ func TestFileSaveAfterDamage(t *testing.T) {
// save a few bytes of data in the cache
data := test.Random(123456789, 42)
id := restic.Hash(data)
h := restic.Handle{
h := backend.Handle{
Type: restic.PackFile,
Name: id.String(),
}

10
internal/checker/checker.go
View File

@ -39,7 +39,7 @@ type Checker struct {
trackUnused bool
masterIndex *index.MasterIndex
snapshots restic.Lister
snapshots backend.Lister
repo restic.Repository
}
@ -134,7 +134,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
if p != nil {
var numIndexFiles uint64
err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error {
err := indexList.List(ctx, restic.IndexFile, func(fi backend.FileInfo) error {
_, err := restic.ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
@ -245,7 +245,7 @@ func IsOrphanedPack(err error) bool {
return errors.As(err, &e) && e.Orphaned
}
func isS3Legacy(b restic.Backend) bool {
func isS3Legacy(b backend.Backend) bool {
// unwrap cache
if be, ok := b.(*cache.Backend); ok {
b = be.Backend
@ -367,7 +367,7 @@ func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan restic.TreeI
}
}
func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.Repository) (ids restic.IDs, errs []error) {
func loadSnapshotTreeIDs(ctx context.Context, lister backend.Lister, repo restic.Repository) (ids restic.IDs, errs []error) {
err := restic.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil {
errs = append(errs, err)
@ -563,7 +563,7 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r
// calculate hash on-the-fly while reading the pack and capture pack header
var hash restic.ID
var hdrBuf []byte
hashingLoader := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
hashingLoader := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return r.Backend().Load(ctx, h, int(size), 0, func(rd io.Reader) error {
hrd := hashing.NewReader(rd, sha256.New())
bufRd.Reset(hrd)

17
internal/checker/checker_test.go
View File

@ -13,6 +13,7 @@ import (
"time"
"github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/hashing"
@ -96,7 +97,7 @@ func TestMissingPack(t *testing.T) {
repo := repository.TestOpenLocal(t, repodir)
packHandle := restic.Handle{
packHandle := backend.Handle{
Type: restic.PackFile,
Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6",
}
@ -129,7 +130,7 @@ func TestUnreferencedPack(t *testing.T) {
// index 3f1a only references pack 60e0
packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
indexHandle := restic.Handle{
indexHandle := backend.Handle{
Type: restic.IndexFile,
Name: "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44",
}
@ -160,7 +161,7 @@ func TestUnreferencedBlobs(t *testing.T) {
repo := repository.TestOpenLocal(t, repodir)
snapshotHandle := restic.Handle{
snapshotHandle := backend.Handle{
Type: restic.SnapshotFile,
Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02",
}
@ -202,7 +203,7 @@ func TestModifiedIndex(t *testing.T) {
done := make(chan struct{})
defer close(done)
h := restic.Handle{
h := backend.Handle{
Type: restic.IndexFile,
Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
}
@ -238,7 +239,7 @@ func TestModifiedIndex(t *testing.T) {
// save the index again with a modified name so that the hash doesn't match
// the content any more
h2 := restic.Handle{
h2 := backend.Handle{
Type: restic.IndexFile,
Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
}
@ -247,7 +248,7 @@ func TestModifiedIndex(t *testing.T) {
if hw != nil {
hash = hw.Sum(nil)
}
rd, err := restic.NewFileReader(tmpfile, hash)
rd, err := backend.NewFileReader(tmpfile, hash)
if err != nil {
t.Fatal(err)
}
@ -304,11 +305,11 @@ func TestDuplicatePacksInIndex(t *testing.T) {
// errorBackend randomly modifies data after reading.
type errorBackend struct {
restic.Backend
backend.Backend
ProduceErrors bool
}
func (b errorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
func (b errorBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
if b.ProduceErrors {
return consumer(errorReadCloser{rd})

3
internal/index/index_parallel.go
View File

@ -5,13 +5,14 @@ import (
"runtime"
"sync"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
)
// ForAllIndexes loads all index files in parallel and calls the given callback.
// It is guaranteed that the function is not run concurrently. If the callback
// returns an error, this function is cancelled and also returns that error.
func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Repository,
func ForAllIndexes(ctx context.Context, lister backend.Lister, repo restic.Repository,
fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error {
// decoding an index can take quite some time such that this can be both CPU- or IO-bound

3
internal/index/master_index_test.go
View File

@ -8,6 +8,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/index"
@ -369,7 +370,7 @@ func testIndexSave(t *testing.T, version uint) {
for id := range obsoletes {
t.Logf("remove index %v", id.Str())
h := restic.Handle{Type: restic.IndexFile, Name: id.String()}
h := backend.Handle{Type: restic.IndexFile, Name: id.String()}
err = repo.Backend().Remove(context.TODO(), h)
if err != nil {
t.Errorf("error removing index %v: %v", id, err)

9
internal/migrations/s3_layout.go
View File

@ -6,6 +6,7 @@ import (
"os"
"path"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/debug"
@ -23,7 +24,7 @@ type S3Layout struct{}
// Check tests whether the migration can be applied.
func (m *S3Layout) Check(_ context.Context, repo restic.Repository) (bool, string, error) {
be := restic.AsBackend[*s3.Backend](repo.Backend())
be := backend.AsBackend[*s3.Backend](repo.Backend())
if be == nil {
debug.Log("backend is not s3")
return false, "backend is not s3", nil
@ -63,8 +64,8 @@ func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layou
fmt.Fprintf(os.Stderr, "renaming file returned error: %v\n", err)
}
return be.List(ctx, t, func(fi restic.FileInfo) error {
h := restic.Handle{Type: t, Name: fi.Name}
return be.List(ctx, t, func(fi backend.FileInfo) error {
h := backend.Handle{Type: t, Name: fi.Name}
debug.Log("move %v", h)
return retry(maxErrors, printErr, func() error {
@ -75,7 +76,7 @@ func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layou
// Apply runs the migration.
func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error {
be := restic.AsBackend[*s3.Backend](repo.Backend())
be := backend.AsBackend[*s3.Backend](repo.Backend())
if be == nil {
debug.Log("backend is not s3")
return errors.New("backend is not s3")

7
internal/migrations/upgrade_repo_v2.go
View File

@ -7,6 +7,7 @@ import (
"os"
"path/filepath"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
)
@ -57,7 +58,7 @@ func (*UpgradeRepoV2) RepoCheck() bool {
return true
}
func (*UpgradeRepoV2) upgrade(ctx context.Context, repo restic.Repository) error {
h := restic.Handle{Type: restic.ConfigFile}
h := backend.Handle{Type: backend.ConfigFile}
if !repo.Backend().HasAtomicReplace() {
// remove the original file for backends which do not support atomic overwriting
@ -85,7 +86,7 @@ func (m *UpgradeRepoV2) Apply(ctx context.Context, repo restic.Repository) error
return fmt.Errorf("create temp dir failed: %w", err)
}
h := restic.Handle{Type: restic.ConfigFile}
h := backend.Handle{Type: restic.ConfigFile}
// read raw config file and save it to a temp dir, just in case
var rawConfigFile []byte
@ -115,7 +116,7 @@ func (m *UpgradeRepoV2) Apply(ctx context.Context, repo restic.Repository) error
// try contingency methods, reupload the original file
_ = repo.Backend().Remove(ctx, h)
err = repo.Backend().Save(ctx, h, restic.NewByteReader(rawConfigFile, nil))
err = repo.Backend().Save(ctx, h, backend.NewByteReader(rawConfigFile, nil))
if err != nil {
repoError.ReuploadOldConfigError = err
}

8
internal/migrations/upgrade_repo_v2_test.go
View File

@ -7,9 +7,9 @@ import (
"sync"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
@ -37,14 +37,14 @@ func TestUpgradeRepoV2(t *testing.T) {
}
type failBackend struct {
restic.Backend
backend.Backend
mu sync.Mutex
ConfigFileSavesUntilError uint
}
func (be *failBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
if h.Type != restic.ConfigFile {
func (be *failBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if h.Type != backend.ConfigFile {
return be.Backend.Save(ctx, h, rd)
}

8
internal/pack/pack_test.go
View File

@ -127,8 +127,8 @@ func TestUnpackReadSeeker(t *testing.T) {
b := mem.New()
id := restic.Hash(packData)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher())))
handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
rtest.OK(t, b.Save(context.TODO(), handle, backend.NewByteReader(packData, b.Hasher())))
verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize)
}
@ -140,7 +140,7 @@ func TestShortPack(t *testing.T) {
b := mem.New()
id := restic.Hash(packData)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher())))
handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
rtest.OK(t, b.Save(context.TODO(), handle, backend.NewByteReader(packData, b.Hasher())))
verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize)
}

6
internal/repository/key.go
View File

@ -178,7 +178,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int,
// LoadKey loads a key from the backend.
func LoadKey(ctx context.Context, s *Repository, id restic.ID) (k *Key, err error) {
h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
h := backend.Handle{Type: restic.KeyFile, Name: id.String()}
data, err := backend.LoadAll(ctx, nil, s.be, h)
if err != nil {
return nil, err
@ -270,12 +270,12 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str
id := restic.Hash(buf)
// store in repository and return
h := restic.Handle{
h := backend.Handle{
Type: restic.KeyFile,
Name: id.String(),
}
err = s.be.Save(ctx, h, restic.NewByteReader(buf, s.be.Hasher()))
err = s.be.Save(ctx, h, backend.NewByteReader(buf, s.be.Hasher()))
if err != nil {
return nil, err
}

7
internal/repository/packer_manager.go
View File

@ -8,6 +8,7 @@ import (
"runtime"
"sync"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/hashing"
"github.com/restic/restic/internal/restic"
@ -145,7 +146,7 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe
// calculate sha256 hash in a second pass
var rd io.Reader
rd, err = restic.NewFileReader(p.tmpfile, nil)
rd, err = backend.NewFileReader(p.tmpfile, nil)
if err != nil {
return err
}
@ -163,12 +164,12 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe
}
id := restic.IDFromHash(hr.Sum(nil))
h := restic.Handle{Type: restic.PackFile, Name: id.String(), IsMetadata: t.IsMetadata()}
h := backend.Handle{Type: backend.PackFile, Name: id.String(), IsMetadata: t.IsMetadata()}
var beHash []byte
if beHr != nil {
beHash = beHr.Sum(nil)
}
rrd, err := restic.NewFileReader(p.tmpfile, beHash)
rrd, err := backend.NewFileReader(p.tmpfile, beHash)
if err != nil {
return err
}

5
internal/repository/repack_test.go
View File

@ -6,6 +6,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -157,7 +158,7 @@ func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs rest
}
for id := range repackedBlobs {
err = repo.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})
err = repo.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()})
if err != nil {
t.Fatal(err)
}
@ -191,7 +192,7 @@ func rebuildIndex(t *testing.T, repo restic.Repository) {
}
err = repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error {
h := restic.Handle{
h := backend.Handle{
Type: restic.IndexFile,
Name: id.String(),
}

30
internal/repository/repository.go
View File

@ -36,7 +36,7 @@ const MaxPackSize = 128 * 1024 * 1024
// Repository is used to access a repository in a backend.
type Repository struct {
be restic.Backend
be backend.Backend
cfg restic.Config
key *crypto.Key
keyID restic.ID
@ -109,7 +109,7 @@ func (c *CompressionMode) Type() string {
}
// New returns a new repository with backend be.
func New(be restic.Backend, opts Options) (*Repository, error) {
func New(be backend.Backend, opts Options) (*Repository, error) {
if opts.Compression == CompressionInvalid {
return nil, errors.New("invalid compression mode")
}
@ -181,7 +181,7 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res
ctx, cancel := context.WithCancel(ctx)
h := restic.Handle{Type: t, Name: id.String()}
h := backend.Handle{Type: t, Name: id.String()}
retriedInvalidData := false
var dataErr error
wr := new(bytes.Buffer)
@ -232,7 +232,7 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res
}
type haver interface {
Has(restic.Handle) bool
Has(backend.Handle) bool
}
// sortCachedPacksFirst moves all cached pack files to the front of blobs.
@ -250,7 +250,7 @@ func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) {
noncached := make([]restic.PackedBlob, 0, len(blobs)/2)
for _, blob := range blobs {
if cache.Has(restic.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) {
if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) {
cached = append(cached, blob)
continue
}
@ -284,7 +284,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.
}
// load blob from pack
h := restic.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: t.IsMetadata()}
h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: t.IsMetadata()}
switch {
case cap(buf) < int(blob.Length):
@ -494,9 +494,9 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by
} else {
id = restic.Hash(ciphertext)
}
h := restic.Handle{Type: t, Name: id.String()}
h := backend.Handle{Type: t, Name: id.String()}
err = r.be.Save(ctx, h, restic.NewByteReader(ciphertext, r.be.Hasher()))
err = r.be.Save(ctx, h, backend.NewByteReader(ciphertext, r.be.Hasher()))
if err != nil {
debug.Log("error saving blob %v: %v", h, err)
return restic.ID{}, err
@ -561,7 +561,7 @@ func (r *Repository) flushPacks(ctx context.Context) error {
}
// Backend returns the backend for the repository.
func (r *Repository) Backend() restic.Backend {
func (r *Repository) Backend() backend.Backend {
return r.be
}
@ -591,7 +591,7 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
if p != nil {
var numIndexFiles uint64
err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error {
err := indexList.List(ctx, restic.IndexFile, func(fi backend.FileInfo) error {
_, err := restic.ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
@ -773,7 +773,7 @@ func (r *Repository) Init(ctx context.Context, version uint, password string, ch
return fmt.Errorf("repository version %v too low", version)
}
_, err := r.be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
_, err := r.be.Stat(ctx, backend.Handle{Type: restic.ConfigFile})
if err != nil && !r.be.IsNotExist(err) {
return err
}
@ -818,7 +818,7 @@ func (r *Repository) KeyID() restic.ID {
// List runs fn for all files of type t in the repo.
func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error {
return r.be.List(ctx, t, func(fi restic.FileInfo) error {
return r.be.List(ctx, t, func(fi backend.FileInfo) error {
id, err := restic.ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
@ -831,7 +831,7 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic
// ListPack returns the list of blobs saved in the pack id and the length of
// the the pack header.
func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, uint32, error) {
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
h := backend.Handle{Type: restic.PackFile, Name: id.String()}
return pack.List(r.Key(), backend.ReaderAt(ctx, r.Backend(), h), size)
}
@ -881,7 +881,7 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte
return newID, known, size, err
}
type BackendLoadFn func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error
type BackendLoadFn func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error
// Skip sections with more than 4MB unused blobs
const maxUnusedRange = 4 * 1024 * 1024
@ -922,7 +922,7 @@ func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, pack
}
func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
h := restic.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: false}
h := backend.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: false}
dataStart := blobs[0].Offset
dataEnd := blobs[len(blobs)-1].Offset + blobs[len(blobs)-1].Length

13
internal/repository/repository_internal_test.go
View File

@ -5,13 +5,14 @@ import (
"sort"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
type mapcache map[restic.Handle]bool
type mapcache map[backend.Handle]bool
func (c mapcache) Has(h restic.Handle) bool { return c[h] }
func (c mapcache) Has(h backend.Handle) bool { return c[h] }
func TestSortCachedPacksFirst(t *testing.T) {
var (
@ -27,15 +28,15 @@ func TestSortCachedPacksFirst(t *testing.T) {
blobs[i] = restic.PackedBlob{PackID: id}
if i%3 == 0 {
h := restic.Handle{Name: id.String(), Type: restic.PackFile}
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
cache[h] = true
}
}
copy(sorted[:], blobs[:])
sort.SliceStable(sorted[:], func(i, j int) bool {
hi := restic.Handle{Type: restic.PackFile, Name: sorted[i].PackID.String()}
hj := restic.Handle{Type: restic.PackFile, Name: sorted[j].PackID.String()}
hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID.String()}
hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID.String()}
return cache.Has(hi) && !cache.Has(hj)
})
@ -58,7 +59,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) {
blobs[i] = restic.PackedBlob{PackID: id}
if i%3 == 0 {
h := restic.Handle{Name: id.String(), Type: restic.PackFile}
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
cache[h] = true
}
}

13
internal/repository/repository_test.go
View File

@ -16,6 +16,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/klauspost/compress/zstd"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/index"
@ -278,13 +279,13 @@ func TestRepositoryLoadUnpackedBroken(t *testing.T) {
data := rtest.Random(23, 12345)
id := restic.Hash(data)
h := restic.Handle{Type: restic.IndexFile, Name: id.String()}
h := backend.Handle{Type: restic.IndexFile, Name: id.String()}
// damage buffer
data[0] ^= 0xff
repo := repository.TestOpenLocal(t, repodir)
// store broken file
err := repo.Backend().Save(context.TODO(), h, restic.NewByteReader(data, nil))
err := repo.Backend().Save(context.TODO(), h, backend.NewByteReader(data, nil))
rtest.OK(t, err)
// without a retry backend this will just return an error that the file is broken
@ -296,10 +297,10 @@ func TestRepositoryLoadUnpackedBroken(t *testing.T) {
}
type damageOnceBackend struct {
restic.Backend
backend.Backend
}
func (be *damageOnceBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *damageOnceBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
// don't break the config file as we can't retry it
if h.Type == restic.ConfigFile {
return be.Backend.Load(ctx, h, length, offset, fn)
@ -352,7 +353,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) {
rtest.OK(b, err)
b.Logf("index saved as %v", id.Str())
fi, err := repo.Backend().Stat(context.TODO(), restic.Handle{Type: restic.IndexFile, Name: id.String()})
fi, err := repo.Backend().Stat(context.TODO(), backend.Handle{Type: restic.IndexFile, Name: id.String()})
rtest.OK(b, err)
b.Logf("filesize is %v", fi.Size)
@ -528,7 +529,7 @@ func testStreamPack(t *testing.T, version uint) {
packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress)
loadCalls := 0
load := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
load := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
data := packfile
if offset > int64(len(data)) {

7
internal/repository/testing.go
View File

@ -6,6 +6,7 @@ import (
"os"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/backend/retry"
@ -34,7 +35,7 @@ func TestUseLowSecurityKDFParameters(t logger) {
}
// TestBackend returns a fully configured in-memory backend.
func TestBackend(_ testing.TB) restic.Backend {
func TestBackend(_ testing.TB) backend.Backend {
return mem.New()
}
@ -43,7 +44,7 @@ const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173)
// TestRepositoryWithBackend returns a repository initialized with a test
// password. If be is nil, an in-memory backend is used. A constant polynomial
// is used for the chunker and low-security test parameters.
func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) restic.Repository {
func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint) restic.Repository {
t.Helper()
TestUseLowSecurityKDFParameters(t)
restic.TestDisableCheckPolynomial(t)
@ -98,7 +99,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository {
// TestOpenLocal opens a local repository.
func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) {
var be restic.Backend
var be backend.Backend
be, err := local.Open(context.TODO(), local.Config{Path: dir, Connections: 2})
if err != nil {
t.Fatal(err)

5
internal/restic/backend_find.go
View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
)
@ -26,13 +27,13 @@ func (e *NoIDByPrefixError) Error() string {
// Find loads the list of all files of type t and searches for names which
// start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned.
// If more than one is found, nil and ErrMultipleIDMatches is returned.
func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error) {
func Find(ctx context.Context, be backend.Lister, t FileType, prefix string) (ID, error) {
match := ID{}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
err := be.List(ctx, t, func(fi FileInfo) error {
err := be.List(ctx, t, func(fi backend.FileInfo) error {
// ignore filename which are not an id
id, err := ParseID(fi.Name)
if err != nil {

10
internal/restic/backend_find_test.go
View File

@ -4,13 +4,15 @@ import (
"context"
"strings"
"testing"
"github.com/restic/restic/internal/backend"
)
type mockBackend struct {
list func(context.Context, FileType, func(FileInfo) error) error
list func(context.Context, FileType, func(backend.FileInfo) error) error
}
func (m mockBackend) List(ctx context.Context, t FileType, fn func(FileInfo) error) error {
func (m mockBackend) List(ctx context.Context, t FileType, fn func(backend.FileInfo) error) error {
return m.list(ctx, t, fn)
}
@ -29,9 +31,9 @@ func TestFind(t *testing.T) {
list := samples
m := mockBackend{}
m.list = func(ctx context.Context, t FileType, fn func(FileInfo) error) error {
m.list = func(ctx context.Context, t FileType, fn func(backend.FileInfo) error) error {
for _, id := range list {
err := fn(FileInfo{Name: id.String()})
err := fn(backend.FileInfo{Name: id.String()})
if err != nil {
return err
}

38
internal/restic/backend_test.go
View File

@ -1,38 +0,0 @@
package restic_test
import (
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
type testBackend struct {
restic.Backend
}
func (t *testBackend) Unwrap() restic.Backend {
return nil
}
type otherTestBackend struct {
restic.Backend
}
func (t *otherTestBackend) Unwrap() restic.Backend {
return t.Backend
}
func TestAsBackend(t *testing.T) {
other := otherTestBackend{}
test.Assert(t, restic.AsBackend[*testBackend](other) == nil, "otherTestBackend is not a testBackend backend")
testBe := &testBackend{}
test.Assert(t, restic.AsBackend[*testBackend](testBe) == testBe, "testBackend was not returned")
wrapper := &otherTestBackend{Backend: testBe}
test.Assert(t, restic.AsBackend[*testBackend](wrapper) == testBe, "failed to unwrap testBackend backend")
wrapper.Backend = other
test.Assert(t, restic.AsBackend[*testBackend](wrapper) == nil, "a wrapped otherTestBackend is not a testBackend")
}

17
internal/restic/lock.go
View File

@ -12,6 +12,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/debug"
@ -213,7 +214,7 @@ func (l *Lock) Unlock() error {
return nil
}
return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: l.lockID.String()})
return l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: l.lockID.String()})
}
var StaleLockTimeout = 30 * time.Minute
@ -273,7 +274,7 @@ func (l *Lock) Refresh(ctx context.Context) error {
oldLockID := l.lockID
l.lockID = &id
return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: oldLockID.String()})
return l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: oldLockID.String()})
}
// RefreshStaleLock is an extended variant of Refresh that can also refresh stale lock files.
@ -302,13 +303,13 @@ func (l *Lock) RefreshStaleLock(ctx context.Context) error {
exists, err = l.checkExistence(ctx)
if err != nil {
// cleanup replacement lock
_ = l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: id.String()})
_ = l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: id.String()})
return err
}
if !exists {
// cleanup replacement lock
_ = l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: id.String()})
_ = l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: id.String()})
return ErrRemovedLock
}
@ -319,7 +320,7 @@ func (l *Lock) RefreshStaleLock(ctx context.Context) error {
oldLockID := l.lockID
l.lockID = &id
return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: oldLockID.String()})
return l.repo.Backend().Remove(context.TODO(), backend.Handle{Type: LockFile, Name: oldLockID.String()})
}
func (l *Lock) checkExistence(ctx context.Context) (bool, error) {
@ -328,7 +329,7 @@ func (l *Lock) checkExistence(ctx context.Context) (bool, error) {
exists := false
err := l.repo.Backend().List(ctx, LockFile, func(fi FileInfo) error {
err := l.repo.Backend().List(ctx, LockFile, func(fi backend.FileInfo) error {
if fi.Name == l.lockID.String() {
exists = true
}
@ -387,7 +388,7 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) {
}
if lock.Stale() {
err = repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()})
err = repo.Backend().Remove(ctx, backend.Handle{Type: LockFile, Name: id.String()})
if err == nil {
processed++
}
@ -403,7 +404,7 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) {
func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) {
var processed uint32
err := ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error {
err := repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()})
err := repo.Backend().Remove(ctx, backend.Handle{Type: LockFile, Name: id.String()})
if err == nil {
atomic.AddUint32(&processed, 1)
}

11
internal/restic/lock_test.go
View File

@ -8,6 +8,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -53,10 +54,10 @@ func TestMultipleLock(t *testing.T) {
}
type failLockLoadingBackend struct {
restic.Backend
backend.Backend
}
func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *failLockLoadingBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
if h.Type == restic.LockFile {
return fmt.Errorf("error loading lock")
}
@ -130,7 +131,7 @@ func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, er
}
func removeLock(repo restic.Repository, id restic.ID) error {
h := restic.Handle{Type: restic.LockFile, Name: id.String()}
h := backend.Handle{Type: restic.LockFile, Name: id.String()}
return repo.Backend().Remove(context.TODO(), h)
}
@ -191,7 +192,7 @@ func TestLockStale(t *testing.T) {
}
func lockExists(repo restic.Repository, t testing.TB, id restic.ID) bool {
h := restic.Handle{Type: restic.LockFile, Name: id.String()}
h := backend.Handle{Type: restic.LockFile, Name: id.String()}
_, err := repo.Backend().Stat(context.TODO(), h)
if err != nil && !repo.Backend().IsNotExist(err) {
t.Fatal(err)
@ -317,7 +318,7 @@ func TestLockRefreshStaleMissing(t *testing.T) {
lockID := checkSingleLock(t, repo)
// refresh must fail if lock was removed
rtest.OK(t, repo.Backend().Remove(context.TODO(), restic.Handle{Type: restic.LockFile, Name: lockID.String()}))
rtest.OK(t, repo.Backend().Remove(context.TODO(), backend.Handle{Type: restic.LockFile, Name: lockID.String()}))
time.Sleep(time.Millisecond)
err = lock.RefreshStaleLock(context.TODO())
rtest.Assert(t, err == restic.ErrRemovedLock, "unexpected error, expected %v, got %v", restic.ErrRemovedLock, err)

Some files were not shown because too many files have changed in this diff Show More