octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-12-25 12:09:59 +00:00
Code Issues Releases Wiki Activity

Merge pull request #4499 from MichaelEischer/modular-backend-code

Browse Source 
Split backend code from restic package
This commit is contained in:
Michael Eischer 2023-10-27 20:19:20 +02:00 committed by GitHub
commit 45962c2847
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
132 changed files with 1145 additions and 1062 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/restic/cmd_backup.go
View File

@ -453,7 +453,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
f.Tags = []restic.TagList{opts.Tags.Flatten()}
}
sn, _, err := f.FindLatest(ctx, repo.Backend(), repo, snName)
sn, _, err := f.FindLatest(ctx, repo, repo, snName)
// Snapshot not found is ok if no explicit parent was set
if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
err = nil

3
cmd/restic/cmd_backup_integration_test.go
View File

@ -9,6 +9,7 @@ import (
"runtime"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
@ -265,7 +266,7 @@ func TestBackupTreeLoadError(t *testing.T) {
// delete the subdirectory pack first
for id := range treePacks {
rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}))
rtest.OK(t, r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()}))
}
testRunRebuildIndex(t, env.gopts)
// now the repo is missing the tree blob in the index; check should report this

6
cmd/restic/cmd_cat.go
View File

@ -106,7 +106,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "snapshot":
sn, _, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
sn, _, err := restic.FindSnapshot(ctx, repo, repo, args[1])
if err != nil {
return errors.Fatalf("could not find snapshot: %v\n", err)
}
@ -154,7 +154,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
return nil
case "pack":
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
h := backend.Handle{Type: restic.PackFile, Name: id.String()}
buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
if err != nil {
return err
@ -193,7 +193,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
return errors.Fatal("blob not found")
case "tree":
sn, subfolder, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
sn, subfolder, err := restic.FindSnapshot(ctx, repo, repo, args[1])
if err != nil {
return errors.Fatalf("could not find snapshot: %v\n", err)
}

5
cmd/restic/cmd_copy.go
View File

@ -4,7 +4,6 @@ import (
"context"
"fmt"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
@ -88,12 +87,12 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
return err
}
srcSnapshotLister, err := backend.MemorizeList(ctx, srcRepo.Backend(), restic.SnapshotFile)
srcSnapshotLister, err := restic.MemorizeList(ctx, srcRepo, restic.SnapshotFile)
if err != nil {
return err
}
dstSnapshotLister, err := backend.MemorizeList(ctx, dstRepo.Backend(), restic.SnapshotFile)
dstSnapshotLister, err := restic.MemorizeList(ctx, dstRepo, restic.SnapshotFile)
if err != nil {
return err
}

12
cmd/restic/cmd_debug.go
View File

@ -78,7 +78,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
}
func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
return restic.ForAllSnapshots(ctx, repo.Backend(), repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
return restic.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
if err != nil {
return err
}
@ -107,7 +107,7 @@ type Blob struct {
func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
var m sync.Mutex
return restic.ParallelList(ctx, repo.Backend(), restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
blobs, _, err := repo.ListPack(ctx, id, size)
if err != nil {
Warnf("error for pack %v: %v\n", id.Str(), err)
@ -134,7 +134,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
}
func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
Printf("index_id: %v\n", id)
if err != nil {
return err
@ -321,7 +321,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li
panic(err)
}
be := repo.Backend()
h := restic.Handle{
h := backend.Handle{
Name: packID.String(),
Type: restic.PackFile,
}
@ -447,7 +447,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
for _, name := range args {
id, err := restic.ParseID(name)
if err != nil {
id, err = restic.Find(ctx, repo.Backend(), restic.PackFile, name)
id, err = restic.Find(ctx, repo, restic.PackFile, name)
if err != nil {
Warnf("error: %v\n", err)
continue
@ -490,7 +490,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error {
Printf("examine %v\n", id)
h := restic.Handle{
h := backend.Handle{
Type: restic.PackFile,
Name: id.String(),
}

3
cmd/restic/cmd_diff.go
View File

@ -7,7 +7,6 @@ import (
"reflect"
"sort"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -346,7 +345,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
}
// cache snapshots listing
be, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
be, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil {
return err
}

2
cmd/restic/cmd_dump.go
View File

@ -147,7 +147,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
}).FindLatest(ctx, repo.Backend(), repo, snapshotIDString)
}).FindLatest(ctx, repo, repo, snapshotIDString)
if err != nil {
return errors.Fatalf("failed to find snapshot: %v", err)
}

3
cmd/restic/cmd_find.go
View File

@ -9,7 +9,6 @@ import (
"github.com/spf13/cobra"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
@ -584,7 +583,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
}
}
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil {
return err
}

2
cmd/restic/cmd_forget.go
View File

@ -183,7 +183,7 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
var snapshots restic.Snapshots
removeSnIDs := restic.NewIDSet()
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
snapshots = append(snapshots, sn)
}

11
cmd/restic/cmd_key.go
View File

@ -7,6 +7,7 @@ import (
"strings"
"sync"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -59,7 +60,7 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions
var m sync.Mutex
var keys []keyInfo
err := restic.ParallelList(ctx, s.Backend(), restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
err := restic.ParallelList(ctx, s, restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
k, err := repository.LoadKey(ctx, s, id)
if err != nil {
Warnf("LoadKey() failed: %v\n", err)
@ -149,7 +150,7 @@ func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) e
return errors.Fatal("refusing to remove key currently used to access repository")
}
h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
h := backend.Handle{Type: restic.KeyFile, Name: id.String()}
err := repo.Backend().Remove(ctx, h)
if err != nil {
return err
@ -176,7 +177,7 @@ func changePassword(ctx context.Context, repo *repository.Repository, gopts Glob
return err
}
h := restic.Handle{Type: restic.KeyFile, Name: oldID.String()}
h := backend.Handle{Type: restic.KeyFile, Name: oldID.String()}
err = repo.Backend().Remove(ctx, h)
if err != nil {
return err
@ -193,7 +194,7 @@ func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repos
err := repo.SearchKey(ctx, pw, 0, key.ID().String())
if err != nil {
// the key is invalid, try to remove it
h := restic.Handle{Type: restic.KeyFile, Name: key.ID().String()}
h := backend.Handle{Type: restic.KeyFile, Name: key.ID().String()}
_ = repo.Backend().Remove(ctx, h)
return errors.Fatalf("failed to access repository with new key: %v", err)
}
@ -237,7 +238,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error {
return err
}
id, err := restic.Find(ctx, repo.Backend(), restic.KeyFile, args[1])
id, err := restic.Find(ctx, repo, restic.KeyFile, args[1])
if err != nil {
return err
}

10
cmd/restic/cmd_key_integration_test.go
View File

@ -6,8 +6,8 @@ import (
"regexp"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -110,11 +110,11 @@ func TestKeyAddRemove(t *testing.T) {
}
type emptySaveBackend struct {
restic.Backend
backend.Backend
}
func (b *emptySaveBackend) Save(ctx context.Context, h restic.Handle, _ restic.RewindReader) error {
return b.Backend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
func (b *emptySaveBackend) Save(ctx context.Context, h backend.Handle, _ backend.RewindReader) error {
return b.Backend.Save(ctx, h, backend.NewByteReader([]byte{}, nil))
}
func TestKeyProblems(t *testing.T) {
@ -122,7 +122,7 @@ func TestKeyProblems(t *testing.T) {
defer cleanup()
testRunInit(t, env.gopts)
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return &emptySaveBackend{r}, nil
}

2
cmd/restic/cmd_list.go
View File

@ -63,7 +63,7 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args
case "locks":
t = restic.LockFile
case "blobs":
return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil {
return err
}

3
cmd/restic/cmd_ls.go
View File

@ -9,7 +9,6 @@ import (
"github.com/spf13/cobra"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
@ -170,7 +169,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
return err
}
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil {
return err
}

4
cmd/restic/cmd_prune.go
View File

@ -152,7 +152,7 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error
return err
}
if repo.Backend().Connections() < 2 {
if repo.Connections() < 2 {
return errors.Fatal("prune requires a backend connection limit of at least two")
}
@ -810,7 +810,7 @@ func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Rep
func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) {
var snapshotTrees restic.IDs
Verbosef("loading all snapshots...\n")
err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, ignoreSnapshots,
err = restic.ForAllSnapshots(ctx, repo, repo, ignoreSnapshots,
func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil {
debug.Log("failed to load snapshot %v (error %v)", id, err)

6
cmd/restic/cmd_prune_integration_test.go
View File

@ -6,13 +6,13 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test"
)
func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
oldHook := gopts.backendTestHook
gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil }
gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
defer func() {
gopts.backendTestHook = oldHook
}()
@ -130,7 +130,7 @@ func TestPruneWithDamagedRepository(t *testing.T) {
removePacksExcept(env.gopts, t, oldPacks, false)
oldHook := env.gopts.backendTestHook
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil }
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
defer func() {
env.gopts.backendTestHook = oldHook
}()

3
cmd/restic/cmd_recover.go
View File

@ -5,7 +5,6 @@ import (
"os"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/spf13/cobra"
@ -52,7 +51,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
return err
}
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil {
return err
}

2
cmd/restic/cmd_repair_index.go
View File

@ -88,7 +88,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti
} else {
Verbosef("loading indexes...\n")
mi := index.NewMasterIndex()
err := index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
err := index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil {
Warnf("removing invalid index %v: %v\n", id, err)
obsoleteIndexes = append(obsoleteIndexes, id)

13
cmd/restic/cmd_repair_index_integration_test.go
View File

@ -8,6 +8,7 @@ import (
"sync"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/restic"
@ -70,12 +71,12 @@ func TestRebuildIndexAlwaysFull(t *testing.T) {
// indexErrorBackend modifies the first index after reading.
type indexErrorBackend struct {
restic.Backend
backend.Backend
lock sync.Mutex
hasErred bool
}
func (b *indexErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
func (b *indexErrorBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
// protect hasErred
b.lock.Lock()
@ -101,7 +102,7 @@ func (erd errorReadCloser) Read(p []byte) (int, error) {
}
func TestRebuildIndexDamage(t *testing.T) {
testRebuildIndex(t, func(r restic.Backend) (restic.Backend, error) {
testRebuildIndex(t, func(r backend.Backend) (backend.Backend, error) {
return &indexErrorBackend{
Backend: r,
}, nil
@ -109,11 +110,11 @@ func TestRebuildIndexDamage(t *testing.T) {
}
type appendOnlyBackend struct {
restic.Backend
backend.Backend
}
// called via repo.Backend().Remove()
func (b *appendOnlyBackend) Remove(_ context.Context, h restic.Handle) error {
func (b *appendOnlyBackend) Remove(_ context.Context, h backend.Handle) error {
return errors.Errorf("Failed to remove %v", h)
}
@ -127,7 +128,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
err := withRestoreGlobalOptions(func() error {
globalOptions.stdout = io.Discard
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return &appendOnlyBackend{r}, nil
}
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts)

3
cmd/restic/cmd_repair_packs.go
View File

@ -5,6 +5,7 @@ import (
"io"
"os"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -84,7 +85,7 @@ func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repo
return errors.Fatalf("%s", err)
}
err = repo.Backend().Load(ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error {
err = repo.Backend().Load(ctx, backend.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error {
_, err := f.Seek(0, 0)
if err != nil {
return err

3
cmd/restic/cmd_repair_snapshots.go
View File

@ -3,7 +3,6 @@ package main
import (
"context"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/walker"
@ -84,7 +83,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
repo.SetDryRun()
}
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil {
return err
}

2
cmd/restic/cmd_restore.go
View File

@ -168,7 +168,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
}).FindLatest(ctx, repo.Backend(), repo, snapshotIDString)
}).FindLatest(ctx, repo, repo, snapshotIDString)
if err != nil {
return errors.Fatalf("failed to find snapshot: %v", err)
}

6
cmd/restic/cmd_rewrite.go
View File

@ -128,7 +128,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
if dryRun {
Verbosef("would delete empty snapshot\n")
} else {
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
if err = repo.Backend().Remove(ctx, h); err != nil {
return false, err
}
@ -170,7 +170,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
Verbosef("saved new snapshot %v\n", id.Str())
if forget {
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
if err = repo.Backend().Remove(ctx, h); err != nil {
return false, err
}
@ -207,7 +207,7 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
repo.SetDryRun()
}
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil {
return err
}

2
cmd/restic/cmd_snapshots.go
View File

@ -73,7 +73,7 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions
}
var snapshots restic.Snapshots
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
snapshots = append(snapshots, sn)
}
snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy)

3
cmd/restic/cmd_stats.go
View File

@ -8,7 +8,6 @@ import (
"strings"
"github.com/restic/chunker"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -94,7 +93,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args
}
}
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil {
return err
}

5
cmd/restic/cmd_tag.go
View File

@ -5,6 +5,7 @@ import (
"github.com/spf13/cobra"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
@ -85,7 +86,7 @@ func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Sna
debug.Log("new snapshot saved as %v", id)
// Remove the old snapshot.
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
h := backend.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
if err = repo.Backend().Remove(ctx, h); err != nil {
return false, err
}
@ -119,7 +120,7 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st
}
changeCnt := 0
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten())
if err != nil {
Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err)

3
cmd/restic/delete.go
View File

@ -5,6 +5,7 @@ import (
"golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
)
@ -45,7 +46,7 @@ func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, rep
for i := 0; i < int(workerCount); i++ {
wg.Go(func() error {
for id := range fileChan {
h := restic.Handle{Type: fileType, Name: id.String()}
h := backend.Handle{Type: fileType, Name: id.String()}
err := repo.Backend().Remove(ctx, h)
if err != nil {
if !gopts.JSON {

3
cmd/restic/find.go
View File

@ -3,7 +3,6 @@ package main
import (
"context"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
"github.com/spf13/pflag"
)
@ -33,7 +32,7 @@ func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.
out := make(chan *restic.Snapshot)
go func() {
defer close(out)
be, err := backend.MemorizeList(ctx, be, restic.SnapshotFile)
be, err := restic.MemorizeList(ctx, be, restic.SnapshotFile)
if err != nil {
Warnf("could not load snapshots: %v\n", err)
return

12
cmd/restic/global.go
View File

@ -48,7 +48,7 @@ var version = "0.16.1-dev (compiled manually)"
// TimeFormat is the format used for all timestamps printed by restic.
const TimeFormat = "2006-01-02 15:04:05"
type backendWrapper func(r restic.Backend) (restic.Backend, error)
type backendWrapper func(r backend.Backend) (backend.Backend, error)
// GlobalOptions hold all global options for restic.
type GlobalOptions struct {
@ -553,7 +553,7 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi
func parseConfig(loc location.Location, opts options.Options) (interface{}, error) {
cfg := loc.Config
if cfg, ok := cfg.(restic.ApplyEnvironmenter); ok {
if cfg, ok := cfg.(backend.ApplyEnvironmenter); ok {
cfg.ApplyEnvironment("")
}
@ -568,14 +568,14 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
}
// Open the backend specified by a location config.
func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) {
func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) {
debug.Log("parsing location %v", location.StripPassword(gopts.backends, s))
loc, err := location.Parse(gopts.backends, s)
if err != nil {
return nil, errors.Fatalf("parsing repository location failed: %v", err)
}
var be restic.Backend
var be backend.Backend
cfg, err := parseConfig(loc, opts)
if err != nil {
@ -613,7 +613,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
}
// check if config is there
fi, err := be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
fi, err := be.Stat(ctx, backend.Handle{Type: restic.ConfigFile})
if err != nil {
return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(gopts.backends, s))
}
@ -626,7 +626,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
}
// Create the backend specified by URI.
func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) {
func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) {
debug.Log("parsing location %v", location.StripPassword(gopts.backends, s))
loc, err := location.Parse(gopts.backends, s)
if err != nil {

7
cmd/restic/integration_helpers_test.go
View File

@ -12,6 +12,7 @@ import (
"sync"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/retry"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
@ -205,7 +206,7 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
extended: make(options.Options),
// replace this hook with "nil" if listing a filetype more than once is necessary
backendTestHook: func(r restic.Backend) (restic.Backend, error) { return newOrderedListOnceBackend(r), nil },
backendTestHook: func(r backend.Backend) (backend.Backend, error) { return newOrderedListOnceBackend(r), nil },
// start with default set of backends
backends: globalOptions.backends,
}
@ -249,7 +250,7 @@ func removePacks(gopts GlobalOptions, t testing.TB, remove restic.IDSet) {
rtest.OK(t, err)
for id := range remove {
rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}))
rtest.OK(t, r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()}))
}
}
@ -272,7 +273,7 @@ func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, rem
if treePacks.Has(id) != removeTreePacks || keep.Has(id) {
return nil
}
return r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})
return r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()})
}))
}

21
cmd/restic/integration_test.go
View File

@ -8,6 +8,7 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
@ -41,12 +42,12 @@ func TestCheckRestoreNoLock(t *testing.T) {
// backends (like e.g. Amazon S3) as the second listing may be inconsistent to what
// is expected by the first listing + some operations.
type listOnceBackend struct {
restic.Backend
backend.Backend
listedFileType map[restic.FileType]bool
strictOrder bool
}
func newListOnceBackend(be restic.Backend) *listOnceBackend {
func newListOnceBackend(be backend.Backend) *listOnceBackend {
return &listOnceBackend{
Backend: be,
listedFileType: make(map[restic.FileType]bool),
@ -54,7 +55,7 @@ func newListOnceBackend(be restic.Backend) *listOnceBackend {
}
}
func newOrderedListOnceBackend(be restic.Backend) *listOnceBackend {
func newOrderedListOnceBackend(be backend.Backend) *listOnceBackend {
return &listOnceBackend{
Backend: be,
listedFileType: make(map[restic.FileType]bool),
@ -62,7 +63,7 @@ func newOrderedListOnceBackend(be restic.Backend) *listOnceBackend {
}
}
func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func(backend.FileInfo) error) error {
if t != restic.LockFile && be.listedFileType[t] {
return errors.Errorf("tried listing type %v the second time", t)
}
@ -77,7 +78,7 @@ func TestListOnce(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return newListOnceBackend(r), nil
}
pruneOpts := PruneOptions{MaxUnused: "0"}
@ -104,10 +105,10 @@ func (r *writeToOnly) WriteTo(w io.Writer) (int64, error) {
}
type onlyLoadWithWriteToBackend struct {
restic.Backend
backend.Backend
}
func (be *onlyLoadWithWriteToBackend) Load(ctx context.Context, h restic.Handle,
func (be *onlyLoadWithWriteToBackend) Load(ctx context.Context, h backend.Handle,
length int, offset int64, fn func(rd io.Reader) error) error {
return be.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
@ -120,7 +121,7 @@ func TestBackendLoadWriteTo(t *testing.T) {
defer cleanup()
// setup backend which only works if it's WriteTo method is correctly propagated upwards
env.gopts.backendInnerTestHook = func(r restic.Backend) (restic.Backend, error) {
env.gopts.backendInnerTestHook = func(r backend.Backend) (backend.Backend, error) {
return &onlyLoadWithWriteToBackend{Backend: r}, nil
}
@ -140,7 +141,7 @@ func TestFindListOnce(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return newListOnceBackend(r), nil
}
@ -158,7 +159,7 @@ func TestFindListOnce(t *testing.T) {
snapshotIDs := restic.NewIDSet()
// specify the two oldest snapshots explicitly and use "latest" to reference the newest one
for sn := range FindFilteredSnapshots(context.TODO(), repo.Backend(), repo, &restic.SnapshotFilter{}, []string{
for sn := range FindFilteredSnapshots(context.TODO(), repo, repo, &restic.SnapshotFilter{}, []string{
secondSnapshot[0].String(),
secondSnapshot[1].String()[:8],
"latest",

7
cmd/restic/lock.go
View File

@ -6,6 +6,7 @@ import (
"sync"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -131,7 +132,7 @@ type refreshLockRequest struct {
result chan bool
}
func refreshLocks(ctx context.Context, backend restic.Backend, lockInfo *lockContext, refreshed chan<- struct{}, forceRefresh <-chan refreshLockRequest) {
func refreshLocks(ctx context.Context, backend backend.Backend, lockInfo *lockContext, refreshed chan<- struct{}, forceRefresh <-chan refreshLockRequest) {
debug.Log("start")
lock := lockInfo.lock
ticker := time.NewTicker(refreshInterval)
@ -257,8 +258,8 @@ func monitorLockRefresh(ctx context.Context, lockInfo *lockContext, refreshed <-
}
}
func tryRefreshStaleLock(ctx context.Context, backend restic.Backend, lock *restic.Lock, cancel context.CancelFunc) bool {
freeze := restic.AsBackend[restic.FreezeBackend](backend)
func tryRefreshStaleLock(ctx context.Context, be backend.Backend, lock *restic.Lock, cancel context.CancelFunc) bool {
freeze := backend.AsBackend[backend.FreezeBackend](be)
if freeze != nil {
debug.Log("freezing backend")
freeze.Freeze()

19
cmd/restic/lock_test.go
View File

@ -9,6 +9,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/debug"
@ -104,11 +105,11 @@ func TestLockConflict(t *testing.T) {
}
type writeOnceBackend struct {
restic.Backend
backend.Backend
written bool
}
func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (b *writeOnceBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if b.written {
return fmt.Errorf("fail after first write")
}
@ -117,7 +118,7 @@ func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic.
}
func TestLockFailedRefresh(t *testing.T) {
repo, cleanup, env := openLockTestRepo(t, func(r restic.Backend) (restic.Backend, error) {
repo, cleanup, env := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) {
return &writeOnceBackend{Backend: r}, nil
})
defer cleanup()
@ -143,11 +144,11 @@ func TestLockFailedRefresh(t *testing.T) {
}
type loggingBackend struct {
restic.Backend
backend.Backend
t *testing.T
}
func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (b *loggingBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
b.t.Logf("save %v @ %v", h, time.Now())
err := b.Backend.Save(ctx, h, rd)
b.t.Logf("save finished %v @ %v", h, time.Now())
@ -155,7 +156,7 @@ func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re
}
func TestLockSuccessfulRefresh(t *testing.T) {
repo, cleanup, env := openLockTestRepo(t, func(r restic.Backend) (restic.Backend, error) {
repo, cleanup, env := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) {
return &loggingBackend{
Backend: r,
t: t,
@ -193,12 +194,12 @@ func TestLockSuccessfulRefresh(t *testing.T) {
}
type slowBackend struct {
restic.Backend
backend.Backend
m sync.Mutex
sleep time.Duration
}
func (b *slowBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (b *slowBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
b.m.Lock()
sleep := b.sleep
b.m.Unlock()
@ -208,7 +209,7 @@ func (b *slowBackend) Save(ctx context.Context, h restic.Handle, rd restic.Rewin
func TestLockSuccessfulStaleRefresh(t *testing.T) {
var sb *slowBackend
repo, cleanup, env := openLockTestRepo(t, func(r restic.Backend) (restic.Backend, error) {
repo, cleanup, env := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) {
sb = &slowBackend{Backend: r}
return sb, nil
})

13
internal/archiver/archiver_test.go
View File

@ -15,6 +15,7 @@ import (
"time"
"github.com/google/go-cmp/cmp"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/errors"
@ -1842,26 +1843,26 @@ func TestArchiverErrorReporting(t *testing.T) {
}
type noCancelBackend struct {
restic.Backend
backend.Backend
}
func (c *noCancelBackend) Remove(_ context.Context, h restic.Handle) error {
func (c *noCancelBackend) Remove(_ context.Context, h backend.Handle) error {
return c.Backend.Remove(context.Background(), h)
}
func (c *noCancelBackend) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) error {
func (c *noCancelBackend) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) error {
return c.Backend.Save(context.Background(), h, rd)
}
func (c *noCancelBackend) Load(_ context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (c *noCancelBackend) Load(_ context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return c.Backend.Load(context.Background(), h, length, offset, fn)
}
func (c *noCancelBackend) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) {
func (c *noCancelBackend) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) {
return c.Backend.Stat(context.Background(), h)
}
func (c *noCancelBackend) List(_ context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (c *noCancelBackend) List(_ context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
return c.Backend.List(context.Background(), t, fn)
}

30
internal/backend/azure/azure.go
View File

@ -15,9 +15,9 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
@ -43,7 +43,7 @@ const saveLargeSize = 256 * 1024 * 1024
const defaultListMaxItems = 5000
// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}
var _ backend.Backend = &Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("azure", ParseConfig, location.NoPassword, Create, Open)
@ -197,7 +197,7 @@ func (be *Backend) Path() string {
}
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
debug.Log("InsertObject(%v, %v)", be.cfg.AccountName, objName)
@ -214,7 +214,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
return err
}
func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.RewindReader) error {
func (be *Backend) saveSmall(ctx context.Context, objName string, rd backend.RewindReader) error {
blockBlobClient := be.container.NewBlockBlobClient(objName)
// upload it as a new "block", use the base64 hash for the ID
@ -239,7 +239,7 @@ func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.Rewi
return errors.Wrap(err, "CommitBlockList")
}
func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.RewindReader) error {
func (be *Backend) saveLarge(ctx context.Context, objName string, rd backend.RewindReader) error {
blockBlobClient := be.container.NewBlockBlobClient(objName)
buf := make([]byte, 100*1024*1024)
@ -294,11 +294,11 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
objName := be.Filename(h)
blockBlobClient := be.container.NewBlobClient(objName)
@ -317,17 +317,17 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int,
}
// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
objName := be.Filename(h)
blobClient := be.container.NewBlobClient(objName)
props, err := blobClient.GetProperties(ctx, nil)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "blob.GetProperties")
return backend.FileInfo{}, errors.Wrap(err, "blob.GetProperties")
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Size: *props.ContentLength,
Name: h.Name,
}
@ -335,7 +335,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo,
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
blob := be.container.NewBlobClient(objName)
@ -350,7 +350,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, _ := be.Basedir(t)
// make sure prefix ends with a slash
@ -381,7 +381,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: path.Base(m),
Size: *item.Properties.ContentLength,
}
@ -407,7 +407,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *Backend) Delete(ctx context.Context) error {
return backend.DefaultDelete(ctx, be)
return util.DefaultDelete(ctx, be)
}
// Close does nothing

4
internal/backend/azure/azure_test.go
View File

@ -122,11 +122,11 @@ func TestUploadLargeFile(t *testing.T) {
data := rtest.Random(23, 300*1024*1024)
id := restic.Hash(data)
h := restic.Handle{Name: id.String(), Type: restic.PackFile}
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
t.Logf("hash of %d bytes: %v", len(data), id)
err = be.Save(ctx, h, restic.NewByteReader(data, be.Hasher()))
err = be.Save(ctx, h, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatal(err)
}

4
internal/backend/azure/config.go
View File

@ -5,9 +5,9 @@ import (
"path"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to an azure compatible
@ -57,7 +57,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {

32
internal/backend/b2/b2.go
View File

@ -12,9 +12,9 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/Backblaze/blazer/b2"
"github.com/Backblaze/blazer/base"
@ -34,8 +34,8 @@ type b2Backend struct {
// Billing happens in 1000 item granlarity, but we are more interested in reducing the number of network round trips
const defaultListMaxItems = 10 * 1000
// ensure statically that *b2Backend implements restic.Backend.
var _ restic.Backend = &b2Backend{}
// ensure statically that *b2Backend implements backend.Backend.
var _ backend.Backend = &b2Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("b2", ParseConfig, location.NoPassword, Create, Open)
@ -85,7 +85,7 @@ func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Clien
}
// Open opens a connection to the B2 service.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
debug.Log("cfg %#v", cfg)
ctx, cancel := context.WithCancel(ctx)
@ -118,7 +118,7 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend
// Create opens a connection to the B2 service. If the bucket does not exist yet,
// it is created.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
debug.Log("cfg %#v", cfg)
ctx, cancel := context.WithCancel(ctx)
@ -188,14 +188,14 @@ func (be *b2Backend) IsNotExist(err error) bool {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *b2Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *b2Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
name := be.Layout.Filename(h)
obj := be.bucket.Object(name)
@ -213,7 +213,7 @@ func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int
}
// Save stores data in the backend at the handle.
func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *b2Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
@ -237,18 +237,18 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind
}
// Stat returns information about a blob.
func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
func (be *b2Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
name := be.Filename(h)
obj := be.bucket.Object(name)
info, err := obj.Attrs(ctx)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Stat")
return backend.FileInfo{}, errors.Wrap(err, "Stat")
}
return restic.FileInfo{Size: info.Size, Name: h.Name}, nil
return backend.FileInfo{Size: info.Size, Name: h.Name}, nil
}
// Remove removes the blob with the given name and type.
func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
func (be *b2Backend) Remove(ctx context.Context, h backend.Handle) error {
// the retry backend will also repeat the remove method up to 10 times
for i := 0; i < 3; i++ {
obj := be.bucket.Object(be.Filename(h))
@ -284,7 +284,7 @@ func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
}
// List returns a channel that yields all names of blobs of type t.
func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *b2Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
@ -299,7 +299,7 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic
return err
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: path.Base(obj.Name()),
Size: attrs.Size,
}
@ -313,7 +313,7 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *b2Backend) Delete(ctx context.Context) error {
return backend.DefaultDelete(ctx, be)
return util.DefaultDelete(ctx, be)
}
// Close does nothing

4
internal/backend/b2/config.go
View File

@ -6,9 +6,9 @@ import (
"regexp"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to an b2 compatible
@ -82,7 +82,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {

10
internal/restic/backend.go → internal/backend/backend.go
View File

@ -1,4 +1,4 @@
package restic
package backend
import (
"context"
@ -27,7 +27,7 @@ type Backend interface {
// HasAtomicReplace returns whether Save() can atomically replace files
HasAtomicReplace() bool
// Remove removes a File described by h.
// Remove removes a File described by h.
Remove(ctx context.Context, h Handle) error
// Close the backend
@ -43,7 +43,7 @@ type Backend interface {
// The function fn may be called multiple times during the same Load invocation
// and therefore must be idempotent.
//
// Implementations are encouraged to use backend.DefaultLoad
// Implementations are encouraged to use util.DefaultLoad
Load(ctx context.Context, h Handle, length int, offset int64, fn func(rd io.Reader) error) error
// Stat returns information about the File identified by h.
@ -70,7 +70,7 @@ type Backend interface {
Delete(ctx context.Context) error
}
type BackendUnwrapper interface {
type Unwrapper interface {
// Unwrap returns the underlying backend or nil if there is none.
Unwrap() Backend
}
@ -81,7 +81,7 @@ func AsBackend[B Backend](b Backend) B {
return be
}
if be, ok := b.(BackendUnwrapper); ok {
if be, ok := b.(Unwrapper); ok {
b = be.Unwrap()
} else {
// not the backend we're looking for

38
internal/backend/backend_test.go Normal file
View File

@ -0,0 +1,38 @@
package backend_test
import (
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/test"
)
type testBackend struct {
backend.Backend
}
func (t *testBackend) Unwrap() backend.Backend {
return nil
}
type otherTestBackend struct {
backend.Backend
}
func (t *otherTestBackend) Unwrap() backend.Backend {
return t.Backend
}
func TestAsBackend(t *testing.T) {
other := otherTestBackend{}
test.Assert(t, backend.AsBackend[*testBackend](other) == nil, "otherTestBackend is not a testBackend backend")
testBe := &testBackend{}
test.Assert(t, backend.AsBackend[*testBackend](testBe) == testBe, "testBackend was not returned")
wrapper := &otherTestBackend{Backend: testBe}
test.Assert(t, backend.AsBackend[*testBackend](wrapper) == testBe, "failed to unwrap testBackend backend")
wrapper.Backend = other
test.Assert(t, backend.AsBackend[*testBackend](wrapper) == nil, "a wrapped otherTestBackend is not a testBackend")
}

20
internal/backend/dryrun/dry_backend.go
View File

@ -5,8 +5,8 @@ import (
"hash"
"io"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
// Backend passes reads through to an underlying layer and accepts writes, but
@ -15,20 +15,20 @@ import (
// the repo and does normal operations else.
// This is used for `backup --dry-run`.
type Backend struct {
b restic.Backend
b backend.Backend
}
// statically ensure that Backend implements restic.Backend.
var _ restic.Backend = &Backend{}
// statically ensure that Backend implements backend.Backend.
var _ backend.Backend = &Backend{}
func New(be restic.Backend) *Backend {
func New(be backend.Backend) *Backend {
b := &Backend{b: be}
debug.Log("created new dry backend")
return b
}
// Save adds new Data to the backend.
func (be *Backend) Save(_ context.Context, h restic.Handle, _ restic.RewindReader) error {
func (be *Backend) Save(_ context.Context, h backend.Handle, _ backend.RewindReader) error {
if err := h.Valid(); err != nil {
return err
}
@ -38,7 +38,7 @@ func (be *Backend) Save(_ context.Context, h restic.Handle, _ restic.RewindReade
}
// Remove deletes a file from the backend.
func (be *Backend) Remove(_ context.Context, _ restic.Handle) error {
func (be *Backend) Remove(_ context.Context, _ backend.Handle) error {
return nil
}
@ -72,14 +72,14 @@ func (be *Backend) IsNotExist(err error) bool {
return be.b.IsNotExist(err)
}
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
return be.b.List(ctx, t, fn)
}
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(io.Reader) error) error {
return be.b.Load(ctx, h, length, offset, fn)
}
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
return be.b.Stat(ctx, h)
}

16
internal/backend/dryrun/dry_backend_test.go
View File

@ -8,16 +8,16 @@ import (
"strings"
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/dryrun"
"github.com/restic/restic/internal/backend/mem"
)
// make sure that Backend implements backend.Backend
var _ restic.Backend = &dryrun.Backend{}
var _ backend.Backend = &dryrun.Backend{}
func newBackends() (*dryrun.Backend, restic.Backend) {
func newBackends() (*dryrun.Backend, backend.Backend) {
m := mem.New()
return dryrun.New(m), m
}
@ -30,7 +30,7 @@ func TestDry(t *testing.T) {
// won't pass. Instead, perform a series of operations over the backend, testing the state
// at each step.
steps := []struct {
be restic.Backend
be backend.Backend
op string
fname string
content string
@ -61,13 +61,13 @@ func TestDry(t *testing.T) {
for i, step := range steps {
var err error
handle := restic.Handle{Type: restic.PackFile, Name: step.fname}
handle := backend.Handle{Type: backend.PackFile, Name: step.fname}
switch step.op {
case "save":
err = step.be.Save(ctx, handle, restic.NewByteReader([]byte(step.content), step.be.Hasher()))
err = step.be.Save(ctx, handle, backend.NewByteReader([]byte(step.content), step.be.Hasher()))
case "list":
fileList := []string{}
err = step.be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
err = step.be.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
fileList = append(fileList, fi.Name)
return nil
})
@ -86,7 +86,7 @@ func TestDry(t *testing.T) {
case "remove":
err = step.be.Remove(ctx, handle)
case "stat":
var fi restic.FileInfo
var fi backend.FileInfo
fi, err = step.be.Stat(ctx, handle)
if err == nil {
fis := fmt.Sprintf("%s %d", fi.Name, fi.Size)

8
internal/restic/file.go → internal/backend/file.go
View File

@ -1,4 +1,4 @@
package restic
package backend
import (
"fmt"
@ -41,9 +41,9 @@ func (t FileType) String() string {
// Handle is used to store and access data in a backend.
type Handle struct {
Type FileType
ContainedBlobType BlobType
Name string
Type FileType
IsMetadata bool
Name string
}
func (h Handle) String() string {

2
internal/restic/file_test.go → internal/backend/file_test.go
View File

@ -1,4 +1,4 @@
package restic
package backend
import (
"testing"

4
internal/backend/gs/config.go
View File

@ -5,9 +5,9 @@ import (
"path"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to a Google Cloud Storage
@ -59,7 +59,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {

32
internal/backend/gs/gs.go
View File

@ -16,8 +16,8 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
@ -45,8 +45,8 @@ type Backend struct {
layout.Layout
}
// Ensure that *Backend implements restic.Backend.
var _ restic.Backend = &Backend{}
// Ensure that *Backend implements backend.Backend.
var _ backend.Backend = &Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open)
@ -122,7 +122,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
}
// Open opens the gs backend at the specified bucket.
func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Open(_ context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
return open(cfg, rt)
}
@ -131,7 +131,7 @@ func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend,
//
// The service account must have the "storage.buckets.create" permission to
// create a bucket the does not yet exist.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
be, err := open(cfg, rt)
if err != nil {
return nil, errors.Wrap(err, "open")
@ -203,7 +203,7 @@ func (be *Backend) Path() string {
}
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
// Set chunk size to zero to disable resumable uploads.
@ -253,14 +253,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
if length == 0 {
// negative length indicates read till end to GCS lib
length = -1
@ -277,20 +277,20 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int,
}
// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h)
attr, err := be.bucket.Object(objName).Attrs(ctx)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "service.Objects.Get")
return backend.FileInfo{}, errors.Wrap(err, "service.Objects.Get")
}
return restic.FileInfo{Size: attr.Size, Name: h.Name}, nil
return backend.FileInfo{Size: attr.Size, Name: h.Name}, nil
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
err := be.bucket.Object(objName).Delete(ctx)
@ -304,7 +304,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, _ := be.Basedir(t)
// make sure prefix ends with a slash
@ -330,7 +330,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: path.Base(m),
Size: int64(attrs.Size),
}
@ -350,7 +350,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *Backend) Delete(ctx context.Context) error {
return backend.DefaultDelete(ctx, be)
return util.DefaultDelete(ctx, be)
}
// Close does nothing.

11
internal/backend/layout/layout.go
View File

@ -7,6 +7,7 @@ import (
"path/filepath"
"regexp"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
@ -15,9 +16,9 @@ import (
// Layout computes paths for file name storage.
type Layout interface {
Filename(restic.Handle) string
Dirname(restic.Handle) string
Basedir(restic.FileType) (dir string, subdirs bool)
Filename(backend.Handle) string
Dirname(backend.Handle) string
Basedir(backend.FileType) (dir string, subdirs bool)
Paths() []string
Name() string
}
@ -102,13 +103,13 @@ func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, err
}
// key file in the "keys" dir (DefaultLayout)
foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile]))
foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[backend.KeyFile]))
if err != nil {
return nil, err
}
// key file in the "key" dir (S3LegacyLayout)
foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile]))
foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[backend.KeyFile]))
if err != nil {
return nil, err
}

28
internal/backend/layout/layout_default.go
View File

@ -3,7 +3,7 @@ package layout
import (
"encoding/hex"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
)
// DefaultLayout implements the default layout for local and sftp backends, as
@ -15,12 +15,12 @@ type DefaultLayout struct {
Join func(...string) string
}
var defaultLayoutPaths = map[restic.FileType]string{
restic.PackFile: "data",
restic.SnapshotFile: "snapshots",
restic.IndexFile: "index",
restic.LockFile: "locks",
restic.KeyFile: "keys",
var defaultLayoutPaths = map[backend.FileType]string{
backend.PackFile: "data",
backend.SnapshotFile: "snapshots",
backend.IndexFile: "index",
backend.LockFile: "locks",
backend.KeyFile: "keys",
}
func (l *DefaultLayout) String() string {
@ -33,10 +33,10 @@ func (l *DefaultLayout) Name() string {
}
// Dirname returns the directory path for a given file type and name.
func (l *DefaultLayout) Dirname(h restic.Handle) string {
func (l *DefaultLayout) Dirname(h backend.Handle) string {
p := defaultLayoutPaths[h.Type]
if h.Type == restic.PackFile && len(h.Name) > 2 {
if h.Type == backend.PackFile && len(h.Name) > 2 {
p = l.Join(p, h.Name[:2]) + "/"
}
@ -44,9 +44,9 @@ func (l *DefaultLayout) Dirname(h restic.Handle) string {
}
// Filename returns a path to a file, including its name.
func (l *DefaultLayout) Filename(h restic.Handle) string {
func (l *DefaultLayout) Filename(h backend.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
if h.Type == backend.ConfigFile {
return l.Join(l.Path, "config")
}
@ -62,15 +62,15 @@ func (l *DefaultLayout) Paths() (dirs []string) {
// also add subdirs
for i := 0; i < 256; i++ {
subdir := hex.EncodeToString([]byte{byte(i)})
dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[restic.PackFile], subdir))
dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[backend.PackFile], subdir))
}
return dirs
}
// Basedir returns the base dir name for type t.
func (l *DefaultLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) {
if t == restic.PackFile {
func (l *DefaultLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) {
if t == backend.PackFile {
subdirs = true
}

14
internal/backend/layout/layout_rest.go
View File

@ -1,6 +1,8 @@
package layout
import "github.com/restic/restic/internal/restic"
import (
"github.com/restic/restic/internal/backend"
)
// RESTLayout implements the default layout for the REST protocol.
type RESTLayout struct {
@ -21,8 +23,8 @@ func (l *RESTLayout) Name() string {
}
// Dirname returns the directory path for a given file type and name.
func (l *RESTLayout) Dirname(h restic.Handle) string {
if h.Type == restic.ConfigFile {
func (l *RESTLayout) Dirname(h backend.Handle) string {
if h.Type == backend.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
@ -30,10 +32,10 @@ func (l *RESTLayout) Dirname(h restic.Handle) string {
}
// Filename returns a path to a file, including its name.
func (l *RESTLayout) Filename(h restic.Handle) string {
func (l *RESTLayout) Filename(h backend.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
if h.Type == backend.ConfigFile {
name = "config"
}
@ -49,6 +51,6 @@ func (l *RESTLayout) Paths() (dirs []string) {
}
// Basedir returns the base dir name for files of type t.
func (l *RESTLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) {
func (l *RESTLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) {
return l.URL + l.Join(l.Path, restLayoutPaths[t]), false
}

26
internal/backend/layout/layout_s3legacy.go
View File

@ -1,6 +1,8 @@
package layout
import "github.com/restic/restic/internal/restic"
import (
"github.com/restic/restic/internal/backend"
)
// S3LegacyLayout implements the old layout used for s3 cloud storage backends, as
// described in the Design document.
@ -10,12 +12,12 @@ type S3LegacyLayout struct {
Join func(...string) string
}
var s3LayoutPaths = map[restic.FileType]string{
restic.PackFile: "data",
restic.SnapshotFile: "snapshot",
restic.IndexFile: "index",
restic.LockFile: "lock",
restic.KeyFile: "key",
var s3LayoutPaths = map[backend.FileType]string{
backend.PackFile: "data",
backend.SnapshotFile: "snapshot",
backend.IndexFile: "index",
backend.LockFile: "lock",
backend.KeyFile: "key",
}
func (l *S3LegacyLayout) String() string {
@ -44,8 +46,8 @@ func (l *S3LegacyLayout) join(url string, items ...string) string {
}
// Dirname returns the directory path for a given file type and name.
func (l *S3LegacyLayout) Dirname(h restic.Handle) string {
if h.Type == restic.ConfigFile {
func (l *S3LegacyLayout) Dirname(h backend.Handle) string {
if h.Type == backend.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
@ -53,10 +55,10 @@ func (l *S3LegacyLayout) Dirname(h restic.Handle) string {
}
// Filename returns a path to a file, including its name.
func (l *S3LegacyLayout) Filename(h restic.Handle) string {
func (l *S3LegacyLayout) Filename(h backend.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
if h.Type == backend.ConfigFile {
name = "config"
}
@ -72,6 +74,6 @@ func (l *S3LegacyLayout) Paths() (dirs []string) {
}
// Basedir returns the base dir name for type t.
func (l *S3LegacyLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) {
func (l *S3LegacyLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) {
return l.Join(l.Path, s3LayoutPaths[t]), false
}

80
internal/backend/layout/layout_test.go
View File

@ -9,7 +9,7 @@ import (
"sort"
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test"
)
@ -19,79 +19,79 @@ func TestDefaultLayout(t *testing.T) {
var tests = []struct {
path string
join func(...string) string
restic.Handle
backend.Handle
filename string
}{
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.PackFile, Name: "0123456"},
backend.Handle{Type: backend.PackFile, Name: "0123456"},
filepath.Join(tempdir, "data", "01", "0123456"),
},
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
filepath.Join(tempdir, "config"),
},
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
filepath.Join(tempdir, "snapshots", "123456"),
},
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.IndexFile, Name: "123456"},
backend.Handle{Type: backend.IndexFile, Name: "123456"},
filepath.Join(tempdir, "index", "123456"),
},
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.LockFile, Name: "123456"},
backend.Handle{Type: backend.LockFile, Name: "123456"},
filepath.Join(tempdir, "locks", "123456"),
},
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.KeyFile, Name: "123456"},
backend.Handle{Type: backend.KeyFile, Name: "123456"},
filepath.Join(tempdir, "keys", "123456"),
},
{
"",
path.Join,
restic.Handle{Type: restic.PackFile, Name: "0123456"},
backend.Handle{Type: backend.PackFile, Name: "0123456"},
"data/01/0123456",
},
{
"",
path.Join,
restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
"config",
},
{
"",
path.Join,
restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
"snapshots/123456",
},
{
"",
path.Join,
restic.Handle{Type: restic.IndexFile, Name: "123456"},
backend.Handle{Type: backend.IndexFile, Name: "123456"},
"index/123456",
},
{
"",
path.Join,
restic.Handle{Type: restic.LockFile, Name: "123456"},
backend.Handle{Type: backend.LockFile, Name: "123456"},
"locks/123456",
},
{
"",
path.Join,
restic.Handle{Type: restic.KeyFile, Name: "123456"},
backend.Handle{Type: backend.KeyFile, Name: "123456"},
"keys/123456",
},
}
@ -143,31 +143,31 @@ func TestRESTLayout(t *testing.T) {
path := rtest.TempDir(t)
var tests = []struct {
restic.Handle
backend.Handle
filename string
}{
{
restic.Handle{Type: restic.PackFile, Name: "0123456"},
backend.Handle{Type: backend.PackFile, Name: "0123456"},
filepath.Join(path, "data", "0123456"),
},
{
restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
filepath.Join(path, "config"),
},
{
restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
filepath.Join(path, "snapshots", "123456"),
},
{
restic.Handle{Type: restic.IndexFile, Name: "123456"},
backend.Handle{Type: backend.IndexFile, Name: "123456"},
filepath.Join(path, "index", "123456"),
},
{
restic.Handle{Type: restic.LockFile, Name: "123456"},
backend.Handle{Type: backend.LockFile, Name: "123456"},
filepath.Join(path, "locks", "123456"),
},
{
restic.Handle{Type: restic.KeyFile, Name: "123456"},
backend.Handle{Type: backend.KeyFile, Name: "123456"},
filepath.Join(path, "keys", "123456"),
},
}
@ -209,61 +209,61 @@ func TestRESTLayout(t *testing.T) {
func TestRESTLayoutURLs(t *testing.T) {
var tests = []struct {
l Layout
h restic.Handle
h backend.Handle
fn string
dir string
}{
{
&RESTLayout{URL: "https://hostname.foo", Path: "", Join: path.Join},
restic.Handle{Type: restic.PackFile, Name: "foobar"},
backend.Handle{Type: backend.PackFile, Name: "foobar"},
"https://hostname.foo/data/foobar",
"https://hostname.foo/data/",
},
{
&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
backend.Handle{Type: backend.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/locks/foobar",
"https://hostname.foo:1234/prefix/repo/locks/",
},
{
&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
{
&S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.PackFile, Name: "foobar"},
backend.Handle{Type: backend.PackFile, Name: "foobar"},
"https://hostname.foo/data/foobar",
"https://hostname.foo/data/",
},
{
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
backend.Handle{Type: backend.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/lock/foobar",
"https://hostname.foo:1234/prefix/repo/lock/",
},
{
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
{
&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
restic.Handle{Type: restic.PackFile, Name: "foobar"},
backend.Handle{Type: backend.PackFile, Name: "foobar"},
"data/foobar",
"data/",
},
{
&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
backend.Handle{Type: backend.LockFile, Name: "foobar"},
"lock/foobar",
"lock/",
},
{
&S3LegacyLayout{URL: "", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"/config",
"/",
},
@ -288,31 +288,31 @@ func TestS3LegacyLayout(t *testing.T) {
path := rtest.TempDir(t)
var tests = []struct {
restic.Handle
backend.Handle
filename string
}{
{
restic.Handle{Type: restic.PackFile, Name: "0123456"},
backend.Handle{Type: backend.PackFile, Name: "0123456"},
filepath.Join(path, "data", "0123456"),
},
{
restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
filepath.Join(path, "config"),
},
{
restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
filepath.Join(path, "snapshot", "123456"),
},
{
restic.Handle{Type: restic.IndexFile, Name: "123456"},
backend.Handle{Type: backend.IndexFile, Name: "123456"},
filepath.Join(path, "index", "123456"),
},
{
restic.Handle{Type: restic.LockFile, Name: "123456"},
backend.Handle{Type: backend.LockFile, Name: "123456"},
filepath.Join(path, "lock", "123456"),
},
{
restic.Handle{Type: restic.KeyFile, Name: "123456"},
backend.Handle{Type: backend.KeyFile, Name: "123456"},
filepath.Join(path, "key", "123456"),
},
}
@ -415,8 +415,8 @@ func TestParseLayout(t *testing.T) {
}
// test that the functions work (and don't panic)
_ = layout.Dirname(restic.Handle{Type: restic.PackFile})
_ = layout.Filename(restic.Handle{Type: restic.PackFile, Name: "1234"})
_ = layout.Dirname(backend.Handle{Type: backend.PackFile})
_ = layout.Filename(backend.Handle{Type: backend.PackFile, Name: "1234"})
_ = layout.Paths()
layoutName := fmt.Sprintf("%T", layout)

22
internal/backend/limiter/limiter_backend.go
View File

@ -4,12 +4,12 @@ import (
"context"
"io"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
)
func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) {
return func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) {
var be restic.Backend
func WrapBackendConstructor[B backend.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (backend.Backend, error) {
return func(ctx context.Context, cfg C, lim Limiter) (backend.Backend, error) {
var be backend.Backend
be, err := constructor(ctx, cfg)
if err != nil {
return nil, err
@ -24,7 +24,7 @@ func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx contex
// LimitBackend wraps a Backend and applies rate limiting to Load() and Save()
// calls on the backend.
func LimitBackend(be restic.Backend, l Limiter) restic.Backend {
func LimitBackend(be backend.Backend, l Limiter) backend.Backend {
return rateLimitedBackend{
Backend: be,
limiter: l,
@ -32,11 +32,11 @@ func LimitBackend(be restic.Backend, l Limiter) restic.Backend {
}
type rateLimitedBackend struct {
restic.Backend
backend.Backend
limiter Limiter
}
func (r rateLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (r rateLimitedBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
limited := limitedRewindReader{
RewindReader: rd,
limited: r.limiter.Upstream(rd),
@ -46,7 +46,7 @@ func (r rateLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic
}
type limitedRewindReader struct {
restic.RewindReader
backend.RewindReader
limited io.Reader
}
@ -55,13 +55,13 @@ func (l limitedRewindReader) Read(b []byte) (int, error) {
return l.limited.Read(b)
}
func (r rateLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
func (r rateLimitedBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
return r.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
return consumer(newDownstreamLimitedReader(rd, r.limiter))
})
}
func (r rateLimitedBackend) Unwrap() restic.Backend { return r.Backend }
func (r rateLimitedBackend) Unwrap() backend.Backend { return r.Backend }
type limitedReader struct {
io.Reader
@ -85,4 +85,4 @@ func (l *limitedReader) WriteTo(w io.Writer) (int64, error) {
return l.writerTo.WriteTo(l.limiter.DownstreamWriter(w))
}
var _ restic.Backend = (*rateLimitedBackend)(nil)
var _ backend.Backend = (*rateLimitedBackend)(nil)

12
internal/backend/limiter/limiter_backend_test.go
View File

@ -8,8 +8,8 @@ import (
"io"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -21,11 +21,11 @@ func randomBytes(t *testing.T, size int) []byte {
}
func TestLimitBackendSave(t *testing.T) {
testHandle := restic.Handle{Type: restic.PackFile, Name: "test"}
testHandle := backend.Handle{Type: backend.PackFile, Name: "test"}
data := randomBytes(t, 1234)
be := mock.NewBackend()
be.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
be.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
buf := new(bytes.Buffer)
_, err := io.Copy(buf, rd)
if err != nil {
@ -39,7 +39,7 @@ func TestLimitBackendSave(t *testing.T) {
limiter := NewStaticLimiter(Limits{42 * 1024, 42 * 1024})
limbe := LimitBackend(be, limiter)
rd := restic.NewByteReader(data, nil)
rd := backend.NewByteReader(data, nil)
err := limbe.Save(context.TODO(), testHandle, rd)
rtest.OK(t, err)
}
@ -64,7 +64,7 @@ func (r *tracedReadWriteToCloser) Close() error {
}
func TestLimitBackendLoad(t *testing.T) {
testHandle := restic.Handle{Type: restic.PackFile, Name: "test"}
testHandle := backend.Handle{Type: backend.PackFile, Name: "test"}
data := randomBytes(t, 1234)
for _, test := range []struct {
@ -72,7 +72,7 @@ func TestLimitBackendLoad(t *testing.T) {
}{{false, false}, {false, true}, {true, false}, {true, true}} {
be := mock.NewBackend()
src := newTracedReadWriteToCloser(bytes.NewReader(data))
be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
if length != 0 || offset != 0 {
return nil, fmt.Errorf("Not supported")
}

4
internal/backend/local/layout_test.go
View File

@ -5,7 +5,7 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test"
)
@ -49,7 +49,7 @@ func TestLayout(t *testing.T) {
}
packs := make(map[string]bool)
err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
packs[fi.Name] = false
return nil
})

38
internal/backend/local/local.go
View File

@ -12,10 +12,10 @@ import (
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
"github.com/cenkalti/backoff/v4"
)
@ -24,11 +24,11 @@ import (
type Local struct {
Config
layout.Layout
backend.Modes
util.Modes
}
// ensure statically that *Local implements restic.Backend.
var _ restic.Backend = &Local{}
// ensure statically that *Local implements backend.Backend.
var _ backend.Backend = &Local{}
func NewFactory() location.Factory {
return location.NewLimitedBackendFactory("local", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open))
@ -42,8 +42,8 @@ func open(ctx context.Context, cfg Config) (*Local, error) {
return nil, err
}
fi, err := fs.Stat(l.Filename(restic.Handle{Type: restic.ConfigFile}))
m := backend.DeriveModesFromFileInfo(fi, err)
fi, err := fs.Stat(l.Filename(backend.Handle{Type: backend.ConfigFile}))
m := util.DeriveModesFromFileInfo(fi, err)
debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir)
return &Local{
@ -70,7 +70,7 @@ func Create(ctx context.Context, cfg Config) (*Local, error) {
}
// test if config file already exists
_, err = fs.Lstat(be.Filename(restic.Handle{Type: restic.ConfigFile}))
_, err = fs.Lstat(be.Filename(backend.Handle{Type: backend.ConfigFile}))
if err == nil {
return nil, errors.New("config file already exists")
}
@ -111,7 +111,7 @@ func (b *Local) IsNotExist(err error) bool {
}
// Save stores data in the backend at the handle.
func (b *Local) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) (err error) {
func (b *Local) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) (err error) {
finalname := b.Filename(h)
dir := filepath.Dir(finalname)
@ -209,11 +209,11 @@ var tempFile = os.CreateTemp // Overridden by test.
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return backend.DefaultLoad(ctx, h, length, offset, b.openReader, fn)
func (b *Local) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, b.openReader, fn)
}
func (b *Local) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (b *Local) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
f, err := fs.Open(b.Filename(h))
if err != nil {
return nil, err
@ -235,17 +235,17 @@ func (b *Local) openReader(_ context.Context, h restic.Handle, length int, offse
}
// Stat returns information about a blob.
func (b *Local) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) {
func (b *Local) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) {
fi, err := fs.Stat(b.Filename(h))
if err != nil {
return restic.FileInfo{}, errors.WithStack(err)
return backend.FileInfo{}, errors.WithStack(err)
}
return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil
return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil
}
// Remove removes the blob with the given name and type.
func (b *Local) Remove(_ context.Context, h restic.Handle) error {
func (b *Local) Remove(_ context.Context, h backend.Handle) error {
fn := b.Filename(h)
// reset read-only flag
@ -259,7 +259,7 @@ func (b *Local) Remove(_ context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) (err error) {
func (b *Local) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) (err error) {
basedir, subdirs := b.Basedir(t)
if subdirs {
err = visitDirs(ctx, basedir, fn)
@ -279,7 +279,7 @@ func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.File
// two levels of directory structure (including dir itself as the first level).
// Also, visitDirs assumes it sees a directory full of directories, while
// visitFiles wants a directory full or regular files.
func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error) error {
func visitDirs(ctx context.Context, dir string, fn func(backend.FileInfo) error) error {
d, err := fs.Open(dir)
if err != nil {
return err
@ -306,7 +306,7 @@ func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error)
return ctx.Err()
}
func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error, ignoreNotADirectory bool) error {
func visitFiles(ctx context.Context, dir string, fn func(backend.FileInfo) error, ignoreNotADirectory bool) error {
d, err := fs.Open(dir)
if err != nil {
return err
@ -340,7 +340,7 @@ func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error,
default:
}
err := fn(restic.FileInfo{
err := fn(backend.FileInfo{
Name: fi.Name(),
Size: fi.Size(),
})

4
internal/backend/local/local_internal_test.go
View File

@ -8,7 +8,7 @@ import (
"syscall"
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test"
"github.com/cenkalti/backoff/v4"
@ -32,7 +32,7 @@ func TestNoSpacePermanent(t *testing.T) {
rtest.OK(t, be.Close())
}()
h := restic.Handle{Type: restic.ConfigFile}
h := backend.Handle{Type: backend.ConfigFile}
err = be.Save(context.Background(), h, nil)
_, ok := err.(*backoff.PermanentError)
rtest.Assert(t, ok,

4
internal/backend/location/display_location_test.go
View File

@ -3,15 +3,15 @@ package location_test
import (
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
func TestStripPassword(t *testing.T) {
registry := location.NewRegistry()
registry.Register(
location.NewHTTPBackendFactory[any, restic.Backend]("test", nil,
location.NewHTTPBackendFactory[any, backend.Backend]("test", nil,
func(s string) string {
return "cleaned"
}, nil, nil,

4
internal/backend/location/location_test.go
View File

@ -3,8 +3,8 @@ package location_test
import (
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
@ -13,7 +13,7 @@ type testConfig struct {
}
func testFactory() location.Factory {
return location.NewHTTPBackendFactory[testConfig, restic.Backend](
return location.NewHTTPBackendFactory[testConfig, backend.Backend](
"local",
func(s string) (*testConfig, error) {
return &testConfig{loc: s}, nil

16
internal/backend/location/registry.go
View File

@ -4,8 +4,8 @@ import (
"context"
"net/http"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/restic"
)
type Registry struct {
@ -33,11 +33,11 @@ type Factory interface {
Scheme() string
ParseConfig(s string) (interface{}, error)
StripPassword(s string) string
Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error)
Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error)
Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error)
Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error)
}
type genericBackendFactory[C any, T restic.Backend] struct {
type genericBackendFactory[C any, T backend.Backend] struct {
scheme string
parseConfigFn func(s string) (*C, error)
stripPasswordFn func(s string) string
@ -58,14 +58,14 @@ func (f *genericBackendFactory[C, T]) StripPassword(s string) string {
}
return s
}
func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) {
func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) {
return f.createFn(ctx, *cfg.(*C), rt, lim)
}
func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) {
func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) {
return f.openFn(ctx, *cfg.(*C), rt, lim)
}
func NewHTTPBackendFactory[C any, T restic.Backend](
func NewHTTPBackendFactory[C any, T backend.Backend](
scheme string,
parseConfigFn func(s string) (*C, error),
stripPasswordFn func(s string) string,
@ -85,7 +85,7 @@ func NewHTTPBackendFactory[C any, T restic.Backend](
}
}
func NewLimitedBackendFactory[C any, T restic.Backend](
func NewLimitedBackendFactory[C any, T backend.Backend](
scheme string,
parseConfigFn func(s string) (*C, error),
stripPasswordFn func(s string) string,

22
internal/backend/logger/log.go
View File

@ -4,18 +4,18 @@ import (
"context"
"io"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
type Backend struct {
restic.Backend
backend.Backend
}
// statically ensure that Backend implements restic.Backend.
var _ restic.Backend = &Backend{}
// statically ensure that Backend implements backend.Backend.
var _ backend.Backend = &Backend{}
func New(be restic.Backend) *Backend {
func New(be backend.Backend) *Backend {
return &Backend{Backend: be}
}
@ -26,7 +26,7 @@ func (be *Backend) IsNotExist(err error) bool {
}
// Save adds new Data to the backend.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
debug.Log("Save(%v, %v)", h, rd.Length())
err := be.Backend.Save(ctx, h, rd)
debug.Log(" save err %v", err)
@ -34,28 +34,28 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
}
// Remove deletes a file from the backend.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
debug.Log("Remove(%v)", h)
err := be.Backend.Remove(ctx, h)
debug.Log(" remove err %v", err)
return err
}
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(io.Reader) error) error {
debug.Log("Load(%v, length %v, offset %v)", h, length, offset)
err := be.Backend.Load(ctx, h, length, offset, fn)
debug.Log(" load err %v", err)
return err
}
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
debug.Log("Stat(%v)", h)
fi, err := be.Backend.Stat(ctx, h)
debug.Log(" stat err %v", err)
return fi, err
}
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
debug.Log("List(%v)", t)
err := be.Backend.List(ctx, t, fn)
debug.Log(" list err %v", err)
@ -76,4 +76,4 @@ func (be *Backend) Close() error {
return err
}
func (be *Backend) Unwrap() restic.Backend { return be.Backend }
func (be *Backend) Unwrap() backend.Backend { return be.Backend }

40
internal/backend/mem/mem_backend.go
View File

@ -12,15 +12,15 @@ import (
"github.com/cespare/xxhash/v2"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
type memMap map[restic.Handle][]byte
type memMap map[backend.Handle][]byte
// make sure that MemoryBackend implements backend.Backend
var _ restic.Backend = &MemoryBackend{}
var _ backend.Backend = &MemoryBackend{}
// NewFactory creates a persistent mem backend
func NewFactory() location.Factory {
@ -69,12 +69,12 @@ func (be *MemoryBackend) IsNotExist(err error) bool {
}
// Save adds new Data to the backend.
func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *MemoryBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
be.m.Lock()
defer be.m.Unlock()
h.ContainedBlobType = restic.InvalidBlob
if h.Type == restic.ConfigFile {
h.IsMetadata = false
if h.Type == backend.ConfigFile {
h.Name = ""
}
@ -112,16 +112,16 @@ func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *MemoryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
func (be *MemoryBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *MemoryBackend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
be.m.Lock()
defer be.m.Unlock()
h.ContainedBlobType = restic.InvalidBlob
if h.Type == restic.ConfigFile {
h.IsMetadata = false
if h.Type == backend.ConfigFile {
h.Name = ""
}
@ -143,29 +143,29 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length
}
// Stat returns information about a file in the backend.
func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (be *MemoryBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
be.m.Lock()
defer be.m.Unlock()
h.ContainedBlobType = restic.InvalidBlob
if h.Type == restic.ConfigFile {
h.IsMetadata = false
if h.Type == backend.ConfigFile {
h.Name = ""
}
e, ok := be.data[h]
if !ok {
return restic.FileInfo{}, errNotFound
return backend.FileInfo{}, errNotFound
}
return restic.FileInfo{Size: int64(len(e)), Name: h.Name}, ctx.Err()
return backend.FileInfo{Size: int64(len(e)), Name: h.Name}, ctx.Err()
}
// Remove deletes a file from the backend.
func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error {
func (be *MemoryBackend) Remove(ctx context.Context, h backend.Handle) error {
be.m.Lock()
defer be.m.Unlock()
h.ContainedBlobType = restic.InvalidBlob
h.IsMetadata = false
if _, ok := be.data[h]; !ok {
return errNotFound
}
@ -176,7 +176,7 @@ func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error {
}
// List returns a channel which yields entries from the backend.
func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *MemoryBackend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
entries := make(map[string]int64)
be.m.Lock()
@ -190,7 +190,7 @@ func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(re
be.m.Unlock()
for name, size := range entries {
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: name,
Size: size,
}

28
internal/backend/mock/backend.go
View File

@ -5,19 +5,19 @@ import (
"hash"
"io"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
// Backend implements a mock backend.
type Backend struct {
CloseFn func() error
IsNotExistFn func(err error) bool
SaveFn func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error
OpenReaderFn func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error)
StatFn func(ctx context.Context, h restic.Handle) (restic.FileInfo, error)
ListFn func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error
RemoveFn func(ctx context.Context, h restic.Handle) error
SaveFn func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error
OpenReaderFn func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error)
StatFn func(ctx context.Context, h backend.Handle) (backend.FileInfo, error)
ListFn func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error
RemoveFn func(ctx context.Context, h backend.Handle) error
DeleteFn func(ctx context.Context) error
ConnectionsFn func() uint
LocationFn func() string
@ -84,7 +84,7 @@ func (m *Backend) IsNotExist(err error) bool {
}
// Save data in the backend.
func (m *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (m *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if m.SaveFn == nil {
return errors.New("not implemented")
}
@ -94,7 +94,7 @@ func (m *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (m *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (m *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
rd, err := m.openReader(ctx, h, length, offset)
if err != nil {
return err
@ -107,7 +107,7 @@ func (m *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
return rd.Close()
}
func (m *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (m *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
if m.OpenReaderFn == nil {
return nil, errors.New("not implemented")
}
@ -116,16 +116,16 @@ func (m *Backend) openReader(ctx context.Context, h restic.Handle, length int, o
}
// Stat an object in the backend.
func (m *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (m *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
if m.StatFn == nil {
return restic.FileInfo{}, errors.New("not implemented")
return backend.FileInfo{}, errors.New("not implemented")
}
return m.StatFn(ctx, h)
}
// List items of type t.
func (m *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (m *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
if m.ListFn == nil {
return nil
}
@ -134,7 +134,7 @@ func (m *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi
}
// Remove data from the backend.
func (m *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (m *Backend) Remove(ctx context.Context, h backend.Handle) error {
if m.RemoveFn == nil {
return errors.New("not implemented")
}
@ -152,4 +152,4 @@ func (m *Backend) Delete(ctx context.Context) error {
}
// Make sure that Backend implements the backend interface.
var _ restic.Backend = &Backend{}
var _ backend.Backend = &Backend{}

5
internal/backend/rclone/backend.go
View File

@ -21,6 +21,7 @@ import (
"github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"golang.org/x/net/http2"
@ -81,7 +82,7 @@ func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, chan stru
cmd.Stdin = r
cmd.Stdout = w
bg, err := backend.StartForeground(cmd)
bg, err := util.StartForeground(cmd)
// close rclone side of pipes
errR := r.Close()
errW := w.Close()
@ -93,7 +94,7 @@ func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, chan stru
err = errW
}
if err != nil {
if backend.IsErrDot(err) {
if util.IsErrDot(err) {
return nil, nil, nil, nil, errors.Errorf("cannot implicitly run relative executable %v found in current directory, use -o rclone.program=./<program> to override", cmd.Path)
}
return nil, nil, nil, nil, err

6
internal/backend/rclone/internal_test.go
View File

@ -5,8 +5,8 @@ import (
"os/exec"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -32,9 +32,9 @@ func TestRcloneExit(t *testing.T) {
t.Log("killed rclone")
for i := 0; i < 10; i++ {
_, err = be.Stat(context.TODO(), restic.Handle{
_, err = be.Stat(context.TODO(), backend.Handle{
Name: "foo",
Type: restic.PackFile,
Type: backend.PackFile,
})
rtest.Assert(t, err != nil, "expected an error")
}

9
internal/backend/readerat.go
View File

@ -6,13 +6,12 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
type backendReaderAt struct {
ctx context.Context
be restic.Backend
h restic.Handle
be Backend
h Handle
}
func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
@ -22,12 +21,12 @@ func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
// ReaderAt returns an io.ReaderAt for a file in the backend. The returned reader
// should not escape the caller function to avoid unexpected interactions with the
// embedded context
func ReaderAt(ctx context.Context, be restic.Backend, h restic.Handle) io.ReaderAt {
func ReaderAt(ctx context.Context, be Backend, h Handle) io.ReaderAt {
return backendReaderAt{ctx: ctx, be: be, h: h}
}
// ReadAt reads from the backend handle h at the given position.
func ReadAt(ctx context.Context, be restic.Backend, h restic.Handle, offset int64, p []byte) (n int, err error) {
func ReadAt(ctx context.Context, be Backend, h Handle, offset int64, p []byte) (n int, err error) {
debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p))
err = be.Load(ctx, h, len(p), offset, func(rd io.Reader) (ierr error) {

4
internal/backend/rest/config.go
View File

@ -5,9 +5,9 @@ import (
"os"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to a REST server.
@ -73,7 +73,7 @@ func prepareURL(s string) string {
return s
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {

48
internal/backend/rest/rest.go
View File

@ -14,13 +14,13 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
// make sure the rest backend implements restic.Backend
var _ restic.Backend = &Backend{}
// make sure the rest backend implements backend.Backend
var _ backend.Backend = &Backend{}
// Backend uses the REST protocol to access data stored on a server.
type Backend struct {
@ -65,7 +65,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, er
return nil, err
}
_, err = be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
_, err = be.Stat(ctx, backend.Handle{Type: backend.ConfigFile})
if err == nil {
return nil, errors.New("config file already exists")
}
@ -118,7 +118,7 @@ func (b *Backend) HasAtomicReplace() bool {
}
// Save stores data in the backend at the handle.
func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
@ -157,7 +157,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
// notExistError is returned whenever the requested file does not exist on the
// server.
type notExistError struct {
restic.Handle
backend.Handle
}
func (e *notExistError) Error() string {
@ -172,7 +172,7 @@ func (b *Backend) IsNotExist(err error) bool {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
r, err := b.openReader(ctx, h, length, offset)
if err != nil {
return err
@ -201,7 +201,7 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
return err
}
func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
req, err := http.NewRequestWithContext(ctx, "GET", b.Filename(h), nil)
if err != nil {
return nil, errors.WithStack(err)
@ -238,37 +238,37 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o
}
// Stat returns information about a blob.
func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodHead, b.Filename(h), nil)
if err != nil {
return restic.FileInfo{}, errors.WithStack(err)
return backend.FileInfo{}, errors.WithStack(err)
}
req.Header.Set("Accept", ContentTypeV2)
resp, err := b.client.Do(req)
if err != nil {
return restic.FileInfo{}, errors.WithStack(err)
return backend.FileInfo{}, errors.WithStack(err)
}
_, _ = io.Copy(io.Discard, resp.Body)
if err = resp.Body.Close(); err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Close")
return backend.FileInfo{}, errors.Wrap(err, "Close")
}
if resp.StatusCode == http.StatusNotFound {
_ = resp.Body.Close()
return restic.FileInfo{}, &notExistError{h}
return backend.FileInfo{}, &notExistError{h}
}
if resp.StatusCode != http.StatusOK {
return restic.FileInfo{}, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status)
return backend.FileInfo{}, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status)
}
if resp.ContentLength < 0 {
return restic.FileInfo{}, errors.New("negative content length")
return backend.FileInfo{}, errors.New("negative content length")
}
bi := restic.FileInfo{
bi := backend.FileInfo{
Size: resp.ContentLength,
Name: h.Name,
}
@ -277,7 +277,7 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e
}
// Remove removes the blob with the given name and type.
func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (b *Backend) Remove(ctx context.Context, h backend.Handle) error {
req, err := http.NewRequestWithContext(ctx, "DELETE", b.Filename(h), nil)
if err != nil {
return errors.WithStack(err)
@ -309,8 +309,8 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
url := b.Dirname(restic.Handle{Type: t})
func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
url := b.Dirname(backend.Handle{Type: t})
if !strings.HasSuffix(url, "/") {
url += "/"
}
@ -346,7 +346,7 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi
// listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET
// /data/`) only returns the names of the files, so we need to issue an HTTP
// HEAD request for each file.
func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
func (b *Backend) listv1(ctx context.Context, t backend.FileType, resp *http.Response, fn func(backend.FileInfo) error) error {
debug.Log("parsing API v1 response")
dec := json.NewDecoder(resp.Body)
var list []string
@ -355,7 +355,7 @@ func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Resp
}
for _, m := range list {
fi, err := b.Stat(ctx, restic.Handle{Name: m, Type: t})
fi, err := b.Stat(ctx, backend.Handle{Name: m, Type: t})
if err != nil {
return err
}
@ -380,7 +380,7 @@ func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Resp
// listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET
// /data/`) returns the names and sizes of all files.
func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(restic.FileInfo) error) error {
func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(backend.FileInfo) error) error {
debug.Log("parsing API v2 response")
dec := json.NewDecoder(resp.Body)
@ -397,7 +397,7 @@ func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(resti
return ctx.Err()
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: item.Name,
Size: item.Size,
}
@ -424,5 +424,5 @@ func (b *Backend) Close() error {
// Delete removes all data in the backend.
func (b *Backend) Delete(ctx context.Context) error {
return backend.DefaultDelete(ctx, b)
return util.DefaultDelete(ctx, b)
}

14
internal/backend/rest/rest_int_test.go
View File

@ -10,8 +10,8 @@ import (
"strconv"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/restic"
)
func TestListAPI(t *testing.T) {
@ -22,7 +22,7 @@ func TestListAPI(t *testing.T) {
Data string // response data
Requests int
Result []restic.FileInfo
Result []backend.FileInfo
}{
{
Name: "content-type-unknown",
@ -32,7 +32,7 @@ func TestListAPI(t *testing.T) {
"3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352",
"8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b"
]`,
Result: []restic.FileInfo{
Result: []backend.FileInfo{
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386},
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214},
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393},
@ -47,7 +47,7 @@ func TestListAPI(t *testing.T) {
"3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352",
"8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b"
]`,
Result: []restic.FileInfo{
Result: []backend.FileInfo{
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386},
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214},
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393},
@ -62,7 +62,7 @@ func TestListAPI(t *testing.T) {
{"name": "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "size": 1002},
{"name": "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", "size": 1003}
]`,
Result: []restic.FileInfo{
Result: []backend.FileInfo{
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 1001},
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 1002},
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 1003},
@ -122,8 +122,8 @@ func TestListAPI(t *testing.T) {
t.Fatal(err)
}
var list []restic.FileInfo
err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
var list []backend.FileInfo
err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
list = append(list, fi)
return nil
})

24
internal/backend/retry/backend_retry.go
View File

@ -7,27 +7,27 @@ import (
"time"
"github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
// Backend retries operations on the backend in case of an error with a
// backoff.
type Backend struct {
restic.Backend
backend.Backend
MaxTries int
Report func(string, error, time.Duration)
Success func(string, int)
}
// statically ensure that RetryBackend implements restic.Backend.
var _ restic.Backend = &Backend{}
// statically ensure that RetryBackend implements backend.Backend.
var _ backend.Backend = &Backend{}
// New wraps be with a backend that retries operations after a
// backoff. report is called with a description and the error, if one occurred.
// success is called with the number of retries before a successful operation
// (it is not called if it succeeded on the first try)
func New(be restic.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend {
func New(be backend.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend {
return &Backend{
Backend: be,
MaxTries: maxTries,
@ -92,7 +92,7 @@ func (be *Backend) retry(ctx context.Context, msg string, f func() error) error
}
// Save stores the data in the backend under the given handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
return be.retry(ctx, fmt.Sprintf("Save(%v)", h), func() error {
err := rd.Rewind()
if err != nil {
@ -125,7 +125,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
// given offset. If length is larger than zero, only a portion of the file
// is returned. rd must be closed after use. If an error is returned, the
// ReadCloser must be nil.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) {
return be.retry(ctx, fmt.Sprintf("Load(%v, %v, %v)", h, length, offset),
func() error {
err := be.Backend.Load(ctx, h, length, offset, consumer)
@ -137,7 +137,7 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
}
// Stat returns information about the File identified by h.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (fi backend.FileInfo, err error) {
err = be.retry(ctx, fmt.Sprintf("Stat(%v)", h),
func() error {
var innerError error
@ -153,7 +153,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInf
}
// Remove removes a File with type t and name.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) (err error) {
return be.retry(ctx, fmt.Sprintf("Remove(%v)", h), func() error {
return be.Backend.Remove(ctx, h)
})
@ -163,7 +163,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) {
// error is returned by the underlying backend, the request is retried. When fn
// returns an error, the operation is aborted and the error is returned to the
// caller.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
// create a new context that we can cancel when fn returns an error, so
// that listing is aborted
listCtx, cancel := context.WithCancel(ctx)
@ -173,7 +173,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
var innerErr error // remember when fn returned an error, so we can return that to the caller
err := be.retry(listCtx, fmt.Sprintf("List(%v)", t), func() error {
return be.Backend.List(ctx, t, func(fi restic.FileInfo) error {
return be.Backend.List(ctx, t, func(fi backend.FileInfo) error {
if _, ok := listed[fi.Name]; ok {
return nil
}
@ -196,6 +196,6 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
return err
}
func (be *Backend) Unwrap() restic.Backend {
func (be *Backend) Unwrap() backend.Backend {
return be.Backend
}

57
internal/backend/retry/backend_retry_test.go
View File

@ -8,6 +8,7 @@ import (
"time"
"github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -18,7 +19,7 @@ func TestBackendSaveRetry(t *testing.T) {
buf := bytes.NewBuffer(nil)
errcount := 0
be := &mock.Backend{
SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if errcount == 0 {
errcount++
_, err := io.CopyN(io.Discard, rd, 120)
@ -38,7 +39,7 @@ func TestBackendSaveRetry(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
data := test.Random(23, 5*1024*1024+11241)
err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher()))
err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatal(err)
}
@ -56,14 +57,14 @@ func TestBackendSaveRetryAtomic(t *testing.T) {
errcount := 0
calledRemove := false
be := &mock.Backend{
SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if errcount == 0 {
errcount++
return errors.New("injected error")
}
return nil
},
RemoveFn: func(ctx context.Context, h restic.Handle) error {
RemoveFn: func(ctx context.Context, h backend.Handle) error {
calledRemove = true
return nil
},
@ -74,7 +75,7 @@ func TestBackendSaveRetryAtomic(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
data := test.Random(23, 5*1024*1024+11241)
err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher()))
err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatal(err)
}
@ -91,15 +92,15 @@ func TestBackendListRetry(t *testing.T) {
retry := 0
be := &mock.Backend{
ListFn: func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
ListFn: func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
// fail during first retry, succeed during second
retry++
if retry == 1 {
_ = fn(restic.FileInfo{Name: ID1})
_ = fn(backend.FileInfo{Name: ID1})
return errors.New("test list error")
}
_ = fn(restic.FileInfo{Name: ID1})
_ = fn(restic.FileInfo{Name: ID2})
_ = fn(backend.FileInfo{Name: ID1})
_ = fn(backend.FileInfo{Name: ID2})
return nil
},
}
@ -108,7 +109,7 @@ func TestBackendListRetry(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
var listed []string
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
listed = append(listed, fi.Name)
return nil
})
@ -121,10 +122,10 @@ func TestBackendListRetryErrorFn(t *testing.T) {
var names = []string{"id1", "id2", "foo", "bar"}
be := &mock.Backend{
ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error {
ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error {
t.Logf("List called for %v", tpe)
for _, name := range names {
err := fn(restic.FileInfo{Name: name})
err := fn(backend.FileInfo{Name: name})
if err != nil {
return err
}
@ -141,7 +142,7 @@ func TestBackendListRetryErrorFn(t *testing.T) {
var listed []string
run := 0
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
t.Logf("fn called for %v", fi.Name)
run++
// return an error for the third item in the list
@ -172,7 +173,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
retries := 0
be := &mock.Backend{
ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error {
ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error {
t.Logf("List called for %v, retries %v", tpe, retries)
retries++
for i, name := range names {
@ -180,7 +181,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
return ErrBackendTest
}
err := fn(restic.FileInfo{Name: name})
err := fn(backend.FileInfo{Name: name})
if err != nil {
return err
}
@ -195,7 +196,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
retryBackend := New(be, maxRetries, nil, nil)
var listed []string
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
t.Logf("fn called for %v", fi.Name)
listed = append(listed, fi.Name)
return nil
@ -252,7 +253,7 @@ func TestBackendLoadRetry(t *testing.T) {
attempt := 0
be := mock.NewBackend()
be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
// returns failing reader on first invocation, good reader on subsequent invocations
attempt++
if attempt > 1 {
@ -265,7 +266,7 @@ func TestBackendLoadRetry(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
var buf []byte
err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) {
err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
buf, err = io.ReadAll(rd)
return err
})
@ -280,7 +281,7 @@ func TestBackendLoadNotExists(t *testing.T) {
attempt := 0
be := mock.NewBackend()
be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
attempt++
if attempt > 1 {
t.Fail()
@ -295,7 +296,7 @@ func TestBackendLoadNotExists(t *testing.T) {
TestFastRetries(t)
retryBackend := New(be, 10, nil, nil)
err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) {
err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
return nil
})
test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err)
@ -308,13 +309,13 @@ func TestBackendStatNotExists(t *testing.T) {
attempt := 0
be := mock.NewBackend()
be.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
be.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
attempt++
if attempt > 1 {
t.Fail()
return restic.FileInfo{}, errors.New("must not retry")
return backend.FileInfo{}, errors.New("must not retry")
}
return restic.FileInfo{}, notFound
return backend.FileInfo{}, notFound
}
be.IsNotExistFn = func(err error) bool {
return errors.Is(err, notFound)
@ -323,7 +324,7 @@ func TestBackendStatNotExists(t *testing.T) {
TestFastRetries(t)
retryBackend := New(be, 10, nil, nil)
_, err := retryBackend.Stat(context.TODO(), restic.Handle{})
_, err := retryBackend.Stat(context.TODO(), backend.Handle{})
test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err)
test.Equals(t, 1, attempt)
}
@ -337,7 +338,7 @@ func TestBackendCanceledContext(t *testing.T) {
// check that we received the expected context canceled error instead
TestFastRetries(t)
retryBackend := New(mock.NewBackend(), 2, nil, nil)
h := restic.Handle{Type: restic.PackFile, Name: restic.NewRandomID().String()}
h := backend.Handle{Type: backend.PackFile, Name: restic.NewRandomID().String()}
// create an already canceled context
ctx, cancel := context.WithCancel(context.Background())
@ -346,15 +347,15 @@ func TestBackendCanceledContext(t *testing.T) {
_, err := retryBackend.Stat(ctx, h)
assertIsCanceled(t, err)
err = retryBackend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
err = retryBackend.Save(ctx, h, backend.NewByteReader([]byte{}, nil))
assertIsCanceled(t, err)
err = retryBackend.Remove(ctx, h)
assertIsCanceled(t, err)
err = retryBackend.Load(ctx, restic.Handle{}, 0, 0, func(rd io.Reader) (err error) {
err = retryBackend.Load(ctx, backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
return nil
})
assertIsCanceled(t, err)
err = retryBackend.List(ctx, restic.PackFile, func(restic.FileInfo) error {
err = retryBackend.List(ctx, backend.PackFile, func(backend.FileInfo) error {
return nil
})
assertIsCanceled(t, err)

2
internal/restic/rewind_reader.go → internal/backend/rewind_reader.go
View File

@ -1,4 +1,4 @@
package restic
package backend
import (
"bytes"

2
internal/restic/rewind_reader_test.go → internal/backend/rewind_reader_test.go
View File

@ -1,4 +1,4 @@
package restic
package backend
import (
"bytes"

4
internal/backend/s3/config.go
View File

@ -6,9 +6,9 @@ import (
"path"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to an s3 compatible
@ -94,7 +94,7 @@ func createConfig(endpoint, bucket, prefix string, useHTTP bool) (*Config, error
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {

34
internal/backend/s3/s3.go
View File

@ -14,9 +14,9 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
@ -30,7 +30,7 @@ type Backend struct {
}
// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}
var _ backend.Backend = &Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open)
@ -127,13 +127,13 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro
// Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
return open(ctx, cfg, rt)
}
// Create opens the S3 backend at bucket and region and creates the bucket if
// it does not exist yet.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
be, err := open(ctx, cfg, rt)
if err != nil {
return nil, errors.Wrap(err, "open")
@ -272,7 +272,7 @@ func (be *Backend) Path() string {
}
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
opts := minio.PutObjectOptions{StorageClass: be.cfg.StorageClass}
@ -294,14 +294,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
objName := be.Filename(h)
opts := minio.GetObjectOptions{}
@ -326,7 +326,7 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int,
}
// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h)
var obj *minio.Object
@ -334,7 +334,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
obj, err = be.client.GetObject(ctx, be.cfg.Bucket, objName, opts)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "client.GetObject")
return backend.FileInfo{}, errors.Wrap(err, "client.GetObject")
}
// make sure that the object is closed properly.
@ -347,14 +347,14 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
fi, err := obj.Stat()
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Stat")
return backend.FileInfo{}, errors.Wrap(err, "Stat")
}
return restic.FileInfo{Size: fi.Size, Name: h.Name}, nil
return backend.FileInfo{Size: fi.Size, Name: h.Name}, nil
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{})
@ -368,7 +368,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, recursive := be.Basedir(t)
// make sure prefix ends with a slash
@ -400,7 +400,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: path.Base(m),
Size: obj.Size,
}
@ -424,14 +424,14 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *Backend) Delete(ctx context.Context) error {
return backend.DefaultDelete(ctx, be)
return util.DefaultDelete(ctx, be)
}
// Close does nothing
func (be *Backend) Close() error { return nil }
// Rename moves a file based on the new layout l.
func (be *Backend) Rename(ctx context.Context, h restic.Handle, l layout.Layout) error {
func (be *Backend) Rename(ctx context.Context, h backend.Handle, l layout.Layout) error {
debug.Log("Rename %v to %v", h, l)
oldname := be.Filename(h)
newname := l.Filename(h)

4
internal/backend/s3/s3_test.go
View File

@ -14,11 +14,11 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -117,7 +117,7 @@ func newMinioTestSuite(t testing.TB) (*test.Suite[s3.Config], func()) {
return &cfg, nil
},
Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be restic.Backend, err error) {
Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be backend.Backend, err error) {
for i := 0; i < 10; i++ {
be, err = s3.Create(ctx, cfg, rt)
if err != nil {

28
internal/backend/sema/backend.go
View File

@ -6,22 +6,22 @@ import (
"sync"
"github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
// make sure that connectionLimitedBackend implements restic.Backend
var _ restic.Backend = &connectionLimitedBackend{}
// make sure that connectionLimitedBackend implements backend.Backend
var _ backend.Backend = &connectionLimitedBackend{}
// connectionLimitedBackend limits the number of concurrent operations.
type connectionLimitedBackend struct {
restic.Backend
backend.Backend
sem semaphore
freezeLock sync.Mutex
}
// NewBackend creates a backend that limits the concurrent operations on the underlying backend
func NewBackend(be restic.Backend) restic.Backend {
func NewBackend(be backend.Backend) backend.Backend {
sem, err := newSemaphore(be.Connections())
if err != nil {
panic(err)
@ -35,9 +35,9 @@ func NewBackend(be restic.Backend) restic.Backend {
// typeDependentLimit acquire a token unless the FileType is a lock file. The returned function
// must be called to release the token.
func (be *connectionLimitedBackend) typeDependentLimit(t restic.FileType) func() {
func (be *connectionLimitedBackend) typeDependentLimit(t backend.FileType) func() {
// allow concurrent lock file operations to ensure that the lock refresh is always possible
if t == restic.LockFile {
if t == backend.LockFile {
return func() {}
}
be.sem.GetToken()
@ -59,7 +59,7 @@ func (be *connectionLimitedBackend) Unfreeze() {
}
// Save adds new Data to the backend.
func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *connectionLimitedBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if err := h.Valid(); err != nil {
return backoff.Permanent(err)
}
@ -75,7 +75,7 @@ func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, r
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *connectionLimitedBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
if err := h.Valid(); err != nil {
return backoff.Permanent(err)
}
@ -96,22 +96,22 @@ func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, l
}
// Stat returns information about a file in the backend.
func (be *connectionLimitedBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (be *connectionLimitedBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
if err := h.Valid(); err != nil {
return restic.FileInfo{}, backoff.Permanent(err)
return backend.FileInfo{}, backoff.Permanent(err)
}
defer be.typeDependentLimit(h.Type)()
if ctx.Err() != nil {
return restic.FileInfo{}, ctx.Err()
return backend.FileInfo{}, ctx.Err()
}
return be.Backend.Stat(ctx, h)
}
// Remove deletes a file from the backend.
func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle) error {
func (be *connectionLimitedBackend) Remove(ctx context.Context, h backend.Handle) error {
if err := h.Valid(); err != nil {
return backoff.Permanent(err)
}
@ -125,6 +125,6 @@ func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle)
return be.Backend.Remove(ctx, h)
}
func (be *connectionLimitedBackend) Unwrap() restic.Backend {
func (be *connectionLimitedBackend) Unwrap() backend.Backend {
return be.Backend
}

64
internal/backend/sema/backend_test.go
View File

@ -8,37 +8,37 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
"golang.org/x/sync/errgroup"
)
func TestParameterValidationSave(t *testing.T) {
m := mock.NewBackend()
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
return nil
}
be := sema.NewBackend(m)
err := be.Save(context.TODO(), restic.Handle{}, nil)
err := be.Save(context.TODO(), backend.Handle{}, nil)
test.Assert(t, err != nil, "Save() with invalid handle did not return an error")
}
func TestParameterValidationLoad(t *testing.T) {
m := mock.NewBackend()
m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
m.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return io.NopCloser(nil), nil
}
be := sema.NewBackend(m)
nilCb := func(rd io.Reader) error { return nil }
err := be.Load(context.TODO(), restic.Handle{}, 10, 0, nilCb)
err := be.Load(context.TODO(), backend.Handle{}, 10, 0, nilCb)
test.Assert(t, err != nil, "Load() with invalid handle did not return an error")
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
err = be.Load(context.TODO(), h, 10, -1, nilCb)
test.Assert(t, err != nil, "Save() with negative offset did not return an error")
err = be.Load(context.TODO(), h, -1, 0, nilCb)
@ -47,23 +47,23 @@ func TestParameterValidationLoad(t *testing.T) {
func TestParameterValidationStat(t *testing.T) {
m := mock.NewBackend()
m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
return restic.FileInfo{}, nil
m.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
return backend.FileInfo{}, nil
}
be := sema.NewBackend(m)
_, err := be.Stat(context.TODO(), restic.Handle{})
_, err := be.Stat(context.TODO(), backend.Handle{})
test.Assert(t, err != nil, "Stat() with invalid handle did not return an error")
}
func TestParameterValidationRemove(t *testing.T) {
m := mock.NewBackend()
m.RemoveFn = func(ctx context.Context, h restic.Handle) error {
m.RemoveFn = func(ctx context.Context, h backend.Handle) error {
return nil
}
be := sema.NewBackend(m)
err := be.Remove(context.TODO(), restic.Handle{})
err := be.Remove(context.TODO(), backend.Handle{})
test.Assert(t, err != nil, "Remove() with invalid handle did not return an error")
}
@ -71,7 +71,7 @@ func TestUnwrap(t *testing.T) {
m := mock.NewBackend()
be := sema.NewBackend(m)
unwrapper := be.(restic.BackendUnwrapper)
unwrapper := be.(backend.Unwrapper)
test.Assert(t, unwrapper.Unwrap() == m, "Unwrap() returned wrong backend")
}
@ -100,7 +100,7 @@ func countingBlocker() (func(), func(int) int) {
return wait, unblock
}
func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be restic.Backend) func() error, unblock func(int) int, isUnlimited bool) {
func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be backend.Backend) func() error, unblock func(int) int, isUnlimited bool) {
expectBlocked := int(2)
workerCount := expectBlocked + 1
@ -125,13 +125,13 @@ func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(b
func TestConcurrencyLimitSave(t *testing.T) {
wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) {
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
wait()
return nil
}
}, func(be restic.Backend) func() error {
}, func(be backend.Backend) func() error {
return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
return be.Save(context.TODO(), h, nil)
}
}, unblock, false)
@ -140,13 +140,13 @@ func TestConcurrencyLimitSave(t *testing.T) {
func TestConcurrencyLimitLoad(t *testing.T) {
wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) {
m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
m.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
wait()
return io.NopCloser(nil), nil
}
}, func(be restic.Backend) func() error {
}, func(be backend.Backend) func() error {
return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
nilCb := func(rd io.Reader) error { return nil }
return be.Load(context.TODO(), h, 10, 0, nilCb)
}
@ -156,13 +156,13 @@ func TestConcurrencyLimitLoad(t *testing.T) {
func TestConcurrencyLimitStat(t *testing.T) {
wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) {
m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
m.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
wait()
return restic.FileInfo{}, nil
return backend.FileInfo{}, nil
}
}, func(be restic.Backend) func() error {
}, func(be backend.Backend) func() error {
return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
_, err := be.Stat(context.TODO(), h)
return err
}
@ -172,13 +172,13 @@ func TestConcurrencyLimitStat(t *testing.T) {
func TestConcurrencyLimitDelete(t *testing.T) {
wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) {
m.RemoveFn = func(ctx context.Context, h restic.Handle) error {
m.RemoveFn = func(ctx context.Context, h backend.Handle) error {
wait()
return nil
}
}, func(be restic.Backend) func() error {
}, func(be backend.Backend) func() error {
return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
return be.Remove(context.TODO(), h)
}
}, unblock, false)
@ -187,13 +187,13 @@ func TestConcurrencyLimitDelete(t *testing.T) {
func TestConcurrencyUnlimitedLockSave(t *testing.T) {
wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) {
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
wait()
return nil
}
}, func(be restic.Backend) func() error {
}, func(be backend.Backend) func() error {
return func() error {
h := restic.Handle{Type: restic.LockFile, Name: "foobar"}
h := backend.Handle{Type: backend.LockFile, Name: "foobar"}
return be.Save(context.TODO(), h, nil)
}
}, unblock, true)
@ -202,13 +202,13 @@ func TestConcurrencyUnlimitedLockSave(t *testing.T) {
func TestFreeze(t *testing.T) {
var counter int64
m := mock.NewBackend()
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
atomic.AddInt64(&counter, 1)
return nil
}
m.ConnectionsFn = func() uint { return 2 }
be := sema.NewBackend(m)
fb := be.(restic.FreezeBackend)
fb := be.(backend.FreezeBackend)
// Freeze backend
fb.Freeze()
@ -218,7 +218,7 @@ func TestFreeze(t *testing.T) {
wg.Add(1)
go func() {
defer wg.Done()
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
test.OK(t, be.Save(context.TODO(), h, nil))
}()

4
internal/backend/sftp/layout_test.go
View File

@ -6,8 +6,8 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/sftp"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -56,7 +56,7 @@ func TestLayout(t *testing.T) {
}
packs := make(map[string]bool)
err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
packs[fi.Name] = false
return nil
})

40
internal/backend/sftp/sftp.go
View File

@ -17,9 +17,9 @@ import (
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/cenkalti/backoff/v4"
"github.com/pkg/sftp"
@ -38,10 +38,10 @@ type SFTP struct {
layout.Layout
Config
backend.Modes
util.Modes
}
var _ restic.Backend = &SFTP{}
var _ backend.Backend = &SFTP{}
func NewFactory() location.Factory {
return location.NewLimitedBackendFactory("sftp", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open))
@ -83,9 +83,9 @@ func startClient(cfg Config) (*SFTP, error) {
return nil, errors.Wrap(err, "cmd.StdoutPipe")
}
bg, err := backend.StartForeground(cmd)
bg, err := util.StartForeground(cmd)
if err != nil {
if backend.IsErrDot(err) {
if util.IsErrDot(err) {
return nil, errors.Errorf("cannot implicitly run relative executable %v found in current directory, use -o sftp.command=./<command> to override", cmd.Path)
}
return nil, err
@ -152,8 +152,8 @@ func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) {
debug.Log("layout: %v\n", sftp.Layout)
fi, err := sftp.c.Stat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile}))
m := backend.DeriveModesFromFileInfo(fi, err)
fi, err := sftp.c.Stat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile}))
m := util.DeriveModesFromFileInfo(fi, err)
debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir)
sftp.Config = cfg
@ -259,10 +259,10 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) {
return nil, err
}
sftp.Modes = backend.DefaultModes
sftp.Modes = util.DefaultModes
// test if config file already exists
_, err = sftp.c.Lstat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile}))
_, err = sftp.c.Lstat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile}))
if err == nil {
return nil, errors.New("config file already exists")
}
@ -313,7 +313,7 @@ func tempSuffix() string {
}
// Save stores data in the backend at the handle.
func (r *SFTP) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) error {
func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) error {
if err := r.clientError(); err != nil {
return err
}
@ -413,11 +413,11 @@ func (r *SFTP) checkNoSpace(dir string, size int64, origErr error) error {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (r *SFTP) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return backend.DefaultLoad(ctx, h, length, offset, r.openReader, fn)
func (r *SFTP) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, r.openReader, fn)
}
func (r *SFTP) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (r *SFTP) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
f, err := r.c.Open(r.Filename(h))
if err != nil {
return nil, err
@ -441,21 +441,21 @@ func (r *SFTP) openReader(_ context.Context, h restic.Handle, length int, offset
}
// Stat returns information about a blob.
func (r *SFTP) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) {
func (r *SFTP) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) {
if err := r.clientError(); err != nil {
return restic.FileInfo{}, err
return backend.FileInfo{}, err
}
fi, err := r.c.Lstat(r.Filename(h))
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Lstat")
return backend.FileInfo{}, errors.Wrap(err, "Lstat")
}
return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil
return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil
}
// Remove removes the content stored at name.
func (r *SFTP) Remove(_ context.Context, h restic.Handle) error {
func (r *SFTP) Remove(_ context.Context, h backend.Handle) error {
if err := r.clientError(); err != nil {
return err
}
@ -465,7 +465,7 @@ func (r *SFTP) Remove(_ context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (r *SFTP) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
basedir, subdirs := r.Basedir(t)
walker := r.c.Walk(basedir)
for {
@ -498,7 +498,7 @@ func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileI
debug.Log("send %v\n", path.Base(walker.Path()))
rfi := restic.FileInfo{
rfi := backend.FileInfo{
Name: path.Base(walker.Path()),
Size: fi.Size(),
}

4
internal/backend/swift/config.go
View File

@ -4,9 +4,9 @@ import (
"os"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains basic configuration needed to specify swift location for a swift server
@ -74,7 +74,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {

30
internal/backend/swift/swift.go
View File

@ -16,9 +16,9 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/ncw/swift/v2"
)
@ -32,8 +32,8 @@ type beSwift struct {
layout.Layout
}
// ensure statically that *beSwift implements restic.Backend.
var _ restic.Backend = &beSwift{}
// ensure statically that *beSwift implements backend.Backend.
var _ backend.Backend = &beSwift{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("swift", ParseConfig, location.NoPassword, Open, Open)
@ -41,7 +41,7 @@ func NewFactory() location.Factory {
// Open opens the swift backend at a container in region. The container is
// created if it does not exist yet.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
debug.Log("config %#v", cfg)
be := &beSwift{
@ -134,11 +134,11 @@ func (be *beSwift) HasAtomicReplace() bool {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
func (be *beSwift) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *beSwift) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
objName := be.Filename(h)
@ -160,7 +160,7 @@ func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int,
}
// Save stores data in the backend at the handle.
func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *beSwift) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
encoding := "binary/octet-stream"
@ -174,19 +174,19 @@ func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
}
// Stat returns information about a blob.
func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
func (be *beSwift) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h)
obj, _, err := be.conn.Object(ctx, be.container, objName)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "conn.Object")
return backend.FileInfo{}, errors.Wrap(err, "conn.Object")
}
return restic.FileInfo{Size: obj.Bytes, Name: h.Name}, nil
return backend.FileInfo{Size: obj.Bytes, Name: h.Name}, nil
}
// Remove removes the blob with the given name and type.
func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error {
func (be *beSwift) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
err := be.conn.ObjectDelete(ctx, be.container, objName)
@ -195,7 +195,7 @@ func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *beSwift) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, _ := be.Basedir(t)
prefix += "/"
@ -212,7 +212,7 @@ func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: m,
Size: obj.Bytes,
}
@ -245,7 +245,7 @@ func (be *beSwift) IsNotExist(err error) bool {
// Delete removes all restic objects in the container.
// It will not remove the container itself.
func (be *beSwift) Delete(ctx context.Context) error {
return backend.DefaultDelete(ctx, be)
return util.DefaultDelete(ctx, be)
}
// Close does nothing

4
internal/backend/swift/swift_test.go
View File

@ -6,9 +6,9 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/swift"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -20,7 +20,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] {
// wait for removals for at least 5m
WaitForDelayedRemoval: 5 * time.Minute,
ErrorHandler: func(t testing.TB, be restic.Backend, err error) error {
ErrorHandler: func(t testing.TB, be backend.Backend, err error) error {
if err == nil {
return nil
}

13
internal/backend/test/benchmarks.go
View File

@ -6,22 +6,23 @@ import (
"io"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
func saveRandomFile(t testing.TB, be restic.Backend, length int) ([]byte, restic.Handle) {
func saveRandomFile(t testing.TB, be backend.Backend, length int) ([]byte, backend.Handle) {
data := test.Random(23, length)
id := restic.Hash(data)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := be.Save(context.TODO(), handle, restic.NewByteReader(data, be.Hasher()))
handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := be.Save(context.TODO(), handle, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatalf("Save() error: %+v", err)
}
return data, handle
}
func remove(t testing.TB, be restic.Backend, h restic.Handle) {
func remove(t testing.TB, be backend.Backend, h backend.Handle) {
if err := be.Remove(context.TODO(), h); err != nil {
t.Fatalf("Remove() returned error: %v", err)
}
@ -146,9 +147,9 @@ func (s *Suite[C]) BenchmarkSave(t *testing.B) {
length := 1<<24 + 2123
data := test.Random(23, length)
id := restic.Hash(data)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
rd := restic.NewByteReader(data, be.Hasher())
rd := backend.NewByteReader(data, be.Hasher())
t.SetBytes(int64(length))
t.ResetTimer()

2
internal/backend/test/doc.go
View File

@ -17,7 +17,7 @@
//
// func newTestSuite(t testing.TB) *test.Suite {
// return &test.Suite{
// Create: func(cfg interface{}) (restic.Backend, error) {
// Create: func(cfg interface{}) (backend.Backend, error) {
// [...]
// },
// [...]

13
internal/backend/test/suite.go
View File

@ -11,7 +11,6 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
@ -35,7 +34,7 @@ type Suite[C any] struct {
WaitForDelayedRemoval time.Duration
// ErrorHandler allows ignoring certain errors.
ErrorHandler func(testing.TB, restic.Backend, error) error
ErrorHandler func(testing.TB, backend.Backend, error) error
}
// RunTests executes all defined tests as subtests of t.
@ -156,7 +155,7 @@ func (s *Suite[C]) RunBenchmarks(b *testing.B) {
s.cleanup(b)
}
func (s *Suite[C]) createOrError() (restic.Backend, error) {
func (s *Suite[C]) createOrError() (backend.Backend, error) {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
return nil, fmt.Errorf("cannot create transport for tests: %v", err)
@ -167,7 +166,7 @@ func (s *Suite[C]) createOrError() (restic.Backend, error) {
return nil, err
}
_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
_, err = be.Stat(context.TODO(), backend.Handle{Type: backend.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
@ -179,7 +178,7 @@ func (s *Suite[C]) createOrError() (restic.Backend, error) {
return be, nil
}
func (s *Suite[C]) create(t testing.TB) restic.Backend {
func (s *Suite[C]) create(t testing.TB) backend.Backend {
be, err := s.createOrError()
if err != nil {
t.Fatal(err)
@ -187,7 +186,7 @@ func (s *Suite[C]) create(t testing.TB) restic.Backend {
return be
}
func (s *Suite[C]) open(t testing.TB) restic.Backend {
func (s *Suite[C]) open(t testing.TB) backend.Backend {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
@ -208,7 +207,7 @@ func (s *Suite[C]) cleanup(t testing.TB) {
s.close(t, be)
}
func (s *Suite[C]) close(t testing.TB, be restic.Backend) {
func (s *Suite[C]) close(t testing.TB, be backend.Backend) {
err := be.Close()
if err != nil {
t.Fatal(err)

100
internal/backend/test/tests.go
View File

@ -27,7 +27,7 @@ func seedRand(t testing.TB) {
t.Logf("rand initialized with seed %d", seed)
}
func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, error) {
func beTest(ctx context.Context, be backend.Backend, h backend.Handle) (bool, error) {
_, err := be.Stat(ctx, h)
if err != nil && be.IsNotExist(err) {
return false, nil
@ -49,7 +49,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
defer s.close(t, b)
// remove a config if present
cfgHandle := restic.Handle{Type: restic.ConfigFile}
cfgHandle := backend.Handle{Type: backend.ConfigFile}
cfgPresent, err := beTest(context.TODO(), b, cfgHandle)
if err != nil {
t.Fatalf("unable to test for config: %+v", err)
@ -60,7 +60,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
}
// save a config
store(t, b, restic.ConfigFile, []byte("test config"))
store(t, b, backend.ConfigFile, []byte("test config"))
// now create the backend again, this must fail
_, err = s.createOrError()
@ -69,7 +69,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
}
// remove config
err = b.Remove(context.TODO(), restic.Handle{Type: restic.ConfigFile, Name: ""})
err = b.Remove(context.TODO(), backend.Handle{Type: backend.ConfigFile, Name: ""})
if err != nil {
t.Fatalf("unexpected error removing config: %+v", err)
}
@ -94,13 +94,13 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
var testString = "Config"
// create config and read it back
_, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.ConfigFile})
_, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.ConfigFile})
if err == nil {
t.Fatalf("did not get expected error for non-existing config")
}
test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize error from LoadAll(): %v", err)
err = b.Save(context.TODO(), restic.Handle{Type: restic.ConfigFile}, restic.NewByteReader([]byte(testString), b.Hasher()))
err = b.Save(context.TODO(), backend.Handle{Type: backend.ConfigFile}, backend.NewByteReader([]byte(testString), b.Hasher()))
if err != nil {
t.Fatalf("Save() error: %+v", err)
}
@ -108,7 +108,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
// try accessing the config with different names, should all return the
// same config
for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
h := restic.Handle{Type: restic.ConfigFile, Name: name}
h := backend.Handle{Type: backend.ConfigFile, Name: name}
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
if err != nil {
t.Fatalf("unable to read config with name %q: %+v", name, err)
@ -120,7 +120,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
}
// remove the config
remove(t, b, restic.Handle{Type: restic.ConfigFile})
remove(t, b, backend.Handle{Type: backend.ConfigFile})
}
// TestLoad tests the backend's Load function.
@ -130,7 +130,7 @@ func (s *Suite[C]) TestLoad(t *testing.T) {
b := s.open(t)
defer s.close(t, b)
err := testLoad(b, restic.Handle{Type: restic.PackFile, Name: "foobar"})
err := testLoad(b, backend.Handle{Type: backend.PackFile, Name: "foobar"})
if err == nil {
t.Fatalf("Load() did not return an error for non-existing blob")
}
@ -141,8 +141,8 @@ func (s *Suite[C]) TestLoad(t *testing.T) {
data := test.Random(23, length)
id := restic.Hash(data)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
err = b.Save(context.TODO(), handle, restic.NewByteReader(data, b.Hasher()))
handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
err = b.Save(context.TODO(), handle, backend.NewByteReader(data, b.Hasher()))
if err != nil {
t.Fatalf("Save() error: %+v", err)
}
@ -243,7 +243,7 @@ func (s *Suite[C]) TestList(t *testing.T) {
// Check that the backend is empty to start with
var found []string
err := b.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
found = append(found, fi.Name)
return nil
})
@ -259,8 +259,8 @@ func (s *Suite[C]) TestList(t *testing.T) {
for i := 0; i < numTestFiles; i++ {
data := test.Random(rand.Int(), rand.Intn(100)+55)
id := restic.Hash(data)
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher()))
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
if err != nil {
t.Fatal(err)
}
@ -284,7 +284,7 @@ func (s *Suite[C]) TestList(t *testing.T) {
s.SetListMaxItems(test.maxItems)
}
err := b.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
id, err := restic.ParseID(fi.Name)
if err != nil {
t.Fatal(err)
@ -320,9 +320,9 @@ func (s *Suite[C]) TestList(t *testing.T) {
}
t.Logf("remove %d files", numTestFiles)
handles := make([]restic.Handle, 0, len(list1))
handles := make([]backend.Handle, 0, len(list1))
for id := range list1 {
handles = append(handles, restic.Handle{Type: restic.PackFile, Name: id.String()})
handles = append(handles, backend.Handle{Type: backend.PackFile, Name: id.String()})
}
err = s.delayedRemove(t, b, handles...)
@ -340,13 +340,13 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
b := s.open(t)
defer s.close(t, b)
testFiles := make([]restic.Handle, 0, numTestFiles)
testFiles := make([]backend.Handle, 0, numTestFiles)
for i := 0; i < numTestFiles; i++ {
data := []byte(fmt.Sprintf("random test blob %v", i))
id := restic.Hash(data)
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher()))
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
if err != nil {
t.Fatal(err)
}
@ -358,7 +358,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
cancel()
// pass in a cancelled context
err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
t.Errorf("got FileInfo %v for cancelled context", fi)
return nil
})
@ -373,7 +373,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
defer cancel()
i := 0
err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
i++
// cancel the context on the first file
if i == 1 {
@ -396,7 +396,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
defer cancel()
i := 0
err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
// cancel the context at the last file
i++
if i == numTestFiles {
@ -423,7 +423,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
i := 0
// pass in a context with a timeout
err := b.List(ctxTimeout, restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(ctxTimeout, backend.PackFile, func(fi backend.FileInfo) error {
i++
// wait until the context is cancelled
@ -494,11 +494,11 @@ func (s *Suite[C]) TestSave(t *testing.T) {
data := test.Random(23, length)
id = sha256.Sum256(data)
h := restic.Handle{
Type: restic.PackFile,
h := backend.Handle{
Type: backend.PackFile,
Name: id.String(),
}
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher()))
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
test.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
@ -546,7 +546,7 @@ func (s *Suite[C]) TestSave(t *testing.T) {
t.Fatal(err)
}
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
// wrap the tempfile in an errorCloser, so we can detect if the backend
// closes the reader
@ -585,7 +585,7 @@ func (s *Suite[C]) TestSave(t *testing.T) {
}
type incompleteByteReader struct {
restic.ByteReader
backend.ByteReader
}
func (r *incompleteByteReader) Length() int64 {
@ -609,8 +609,8 @@ func (s *Suite[C]) TestSaveError(t *testing.T) {
copy(id[:], data)
// test that incomplete uploads fail
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &incompleteByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())})
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &incompleteByteReader{ByteReader: *backend.NewByteReader(data, b.Hasher())})
// try to delete possible leftovers
_ = s.delayedRemove(t, b, h)
if err == nil {
@ -619,7 +619,7 @@ func (s *Suite[C]) TestSaveError(t *testing.T) {
}
type wrongByteReader struct {
restic.ByteReader
backend.ByteReader
}
func (b *wrongByteReader) Hash() []byte {
@ -648,8 +648,8 @@ func (s *Suite[C]) TestSaveWrongHash(t *testing.T) {
copy(id[:], data)
// test that upload with hash mismatch fails
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())})
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *backend.NewByteReader(data, b.Hasher())})
exists, err2 := beTest(context.TODO(), b, h)
if err2 != nil {
t.Fatal(err2)
@ -674,23 +674,23 @@ var testStrings = []struct {
{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
}
func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) restic.Handle {
func store(t testing.TB, b backend.Backend, tpe backend.FileType, data []byte) backend.Handle {
id := restic.Hash(data)
h := restic.Handle{Name: id.String(), Type: tpe}
err := b.Save(context.TODO(), h, restic.NewByteReader([]byte(data), b.Hasher()))
h := backend.Handle{Name: id.String(), Type: tpe}
err := b.Save(context.TODO(), h, backend.NewByteReader([]byte(data), b.Hasher()))
test.OK(t, err)
return h
}
// testLoad loads a blob (but discards its contents).
func testLoad(b restic.Backend, h restic.Handle) error {
func testLoad(b backend.Backend, h backend.Handle) error {
return b.Load(context.TODO(), h, 0, 0, func(rd io.Reader) (ierr error) {
_, ierr = io.Copy(io.Discard, rd)
return ierr
})
}
func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error {
func (s *Suite[C]) delayedRemove(t testing.TB, be backend.Backend, handles ...backend.Handle) error {
// Some backend (swift, I'm looking at you) may implement delayed
// removal of data. Let's wait a bit if this happens.
@ -734,11 +734,11 @@ func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...res
return nil
}
func delayedList(t testing.TB, b restic.Backend, tpe restic.FileType, max int, maxwait time.Duration) restic.IDs {
func delayedList(t testing.TB, b backend.Backend, tpe backend.FileType, max int, maxwait time.Duration) restic.IDs {
list := restic.NewIDSet()
start := time.Now()
for i := 0; i < max; i++ {
err := b.List(context.TODO(), tpe, func(fi restic.FileInfo) error {
err := b.List(context.TODO(), tpe, func(fi backend.FileInfo) error {
id := restic.TestParseID(fi.Name)
list.Insert(id)
return nil
@ -763,9 +763,9 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
test.Assert(t, !b.IsNotExist(nil), "IsNotExist() recognized nil error")
for _, tpe := range []restic.FileType{
restic.PackFile, restic.KeyFile, restic.LockFile,
restic.SnapshotFile, restic.IndexFile,
for _, tpe := range []backend.FileType{
backend.PackFile, backend.KeyFile, backend.LockFile,
backend.SnapshotFile, backend.IndexFile,
} {
// detect non-existing files
for _, ts := range testStrings {
@ -773,7 +773,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
test.OK(t, err)
// test if blob is already in repository
h := restic.Handle{Type: tpe, Name: id.String()}
h := backend.Handle{Type: tpe, Name: id.String()}
ret, err := beTest(context.TODO(), b, h)
test.OK(t, err)
test.Assert(t, !ret, "blob was found to exist before creating")
@ -799,7 +799,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
store(t, b, tpe, []byte(ts.data))
// test Load()
h := restic.Handle{Type: tpe, Name: ts.id}
h := backend.Handle{Type: tpe, Name: ts.id}
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
test.OK(t, err)
test.Equals(t, ts.data, string(buf))
@ -823,7 +823,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
// test adding the first file again
ts := testStrings[0]
h := restic.Handle{Type: tpe, Name: ts.id}
h := backend.Handle{Type: tpe, Name: ts.id}
// remove and recreate
err := s.delayedRemove(t, b, h)
@ -835,7 +835,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
test.Assert(t, !ok, "removed blob still present")
// create blob
err = b.Save(context.TODO(), h, restic.NewByteReader([]byte(ts.data), b.Hasher()))
err = b.Save(context.TODO(), h, backend.NewByteReader([]byte(ts.data), b.Hasher()))
test.OK(t, err)
// list items
@ -859,12 +859,12 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list)
}
var handles []restic.Handle
var handles []backend.Handle
for _, ts := range testStrings {
id, err := restic.ParseID(ts.id)
test.OK(t, err)
h := restic.Handle{Type: tpe, Name: id.String()}
h := backend.Handle{Type: tpe, Name: id.String()}
found, err := beTest(context.TODO(), b, h)
test.OK(t, err)

50
internal/backend/util/defaults.go Normal file
View File

@ -0,0 +1,50 @@
package util
import (
"context"
"io"
"github.com/restic/restic/internal/backend"
)
// DefaultLoad implements Backend.Load using lower-level openReader func
func DefaultLoad(ctx context.Context, h backend.Handle, length int, offset int64,
openReader func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error),
fn func(rd io.Reader) error) error {
rd, err := openReader(ctx, h, length, offset)
if err != nil {
return err
}
err = fn(rd)
if err != nil {
_ = rd.Close() // ignore secondary errors closing the reader
return err
}
return rd.Close()
}
// DefaultDelete removes all restic keys in the bucket. It will not remove the bucket itself.
func DefaultDelete(ctx context.Context, be backend.Backend) error {
alltypes := []backend.FileType{
backend.PackFile,
backend.KeyFile,
backend.LockFile,
backend.SnapshotFile,
backend.IndexFile}
for _, t := range alltypes {
err := be.List(ctx, t, func(fi backend.FileInfo) error {
return be.Remove(ctx, backend.Handle{Type: t, Name: fi.Name})
})
if err != nil {
return nil
}
}
err := be.Remove(ctx, backend.Handle{Type: backend.ConfigFile})
if err != nil && be.IsNotExist(err) {
err = nil
}
return err
}

64
internal/backend/util/defaults_test.go Normal file
View File

@ -0,0 +1,64 @@
package util_test
import (
"context"
"io"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/errors"
rtest "github.com/restic/restic/internal/test"
)
type mockReader struct {
closed bool
}
func (rd *mockReader) Read(_ []byte) (n int, err error) {
return 0, nil
}
func (rd *mockReader) Close() error {
rd.closed = true
return nil
}
func TestDefaultLoad(t *testing.T) {
h := backend.Handle{Name: "id", Type: backend.PackFile}
rd := &mockReader{}
// happy case, assert correct parameters are passed around and content stream is closed
err := util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) {
rtest.Equals(t, h, ih)
rtest.Equals(t, int(10), length)
rtest.Equals(t, int64(11), offset)
return rd, nil
}, func(ird io.Reader) error {
rtest.Equals(t, rd, ird)
return nil
})
rtest.OK(t, err)
rtest.Equals(t, true, rd.closed)
// unhappy case, assert producer errors are handled correctly
err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return nil, errors.Errorf("producer error")
}, func(ird io.Reader) error {
t.Fatalf("unexpected consumer invocation")
return nil
})
rtest.Equals(t, "producer error", err.Error())
// unhappy case, assert consumer errors are handled correctly
rd = &mockReader{}
err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return rd, nil
}, func(ird io.Reader) error {
return errors.Errorf("consumer error")
})
rtest.Equals(t, true, rd.closed)
rtest.Equals(t, "consumer error", err.Error())
}

2
internal/backend/errdot_119.go → internal/backend/util/errdot_119.go
View File

@ -8,7 +8,7 @@
// Once the minimum Go version restic supports is 1.19, remove this file and
// replace any calls to it with the corresponding code as per below.
package backend
package util
import (
"errors"

2
internal/backend/errdot_old.go → internal/backend/util/errdot_old.go
View File

@ -6,7 +6,7 @@
// Once the minimum Go version restic supports is 1.19, remove this file
// and perform the actions listed in errdot_119.go.
package backend
package util
func IsErrDot(err error) bool {
return false

2
internal/backend/foreground.go → internal/backend/util/foreground.go
View File

@ -1,4 +1,4 @@
package backend
package util
import (
"os"

2
internal/backend/foreground_sysv.go → internal/backend/util/foreground_sysv.go
View File

@ -1,7 +1,7 @@
//go:build aix || solaris
// +build aix solaris
package backend
package util
import (
"os/exec"

6
internal/backend/foreground_test.go → internal/backend/util/foreground_test.go
View File

@ -1,7 +1,7 @@
//go:build !windows
// +build !windows
package backend_test
package util_test
import (
"bufio"
@ -10,7 +10,7 @@ import (
"strings"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/util"
rtest "github.com/restic/restic/internal/test"
)
@ -22,7 +22,7 @@ func TestForeground(t *testing.T) {
stdout, err := cmd.StdoutPipe()
rtest.OK(t, err)
bg, err := backend.StartForeground(cmd)
bg, err := util.StartForeground(cmd)
rtest.OK(t, err)
defer func() {
rtest.OK(t, cmd.Wait())

2
internal/backend/foreground_unix.go → internal/backend/util/foreground_unix.go
View File

@ -1,7 +1,7 @@
//go:build !aix && !solaris && !windows
// +build !aix,!solaris,!windows
package backend
package util
import (
"os"

2
internal/backend/foreground_windows.go → internal/backend/util/foreground_windows.go
View File

@ -1,4 +1,4 @@
package backend
package util
import (
"os/exec"

2
internal/backend/paths.go → internal/backend/util/paths.go
View File

@ -1,4 +1,4 @@
package backend
package util
import "os"

110
internal/backend/utils.go
View File

@ -3,18 +3,36 @@ package backend
import (
"bytes"
"context"
"encoding/hex"
"fmt"
"io"
"github.com/minio/sha256-simd"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
func verifyContentMatchesName(s string, data []byte) (bool, error) {
if len(s) != hex.EncodedLen(sha256.Size) {
return false, fmt.Errorf("invalid length for ID: %q", s)
}
b, err := hex.DecodeString(s)
if err != nil {
return false, fmt.Errorf("invalid ID: %s", err)
}
var id [sha256.Size]byte
copy(id[:], b)
hashed := sha256.Sum256(data)
return id == hashed, nil
}
// LoadAll reads all data stored in the backend for the handle into the given
// buffer, which is truncated. If the buffer is not large enough or nil, a new
// one is allocated.
func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle) ([]byte, error) {
func LoadAll(ctx context.Context, buf []byte, be Backend, h Handle) ([]byte, error) {
retriedInvalidData := false
err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
// make sure this is idempotent, in case an error occurs this function may be called multiple times!
@ -28,9 +46,8 @@ func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle
// retry loading damaged data only once. If a file fails to download correctly
// the second time, then it is likely corrupted at the backend. Return the data
// to the caller in that case to let it decide what to do with the data.
if !retriedInvalidData && h.Type != restic.ConfigFile {
id, err := restic.ParseID(h.Name)
if err == nil && !restic.Hash(buf).Equal(id) {
if !retriedInvalidData && h.Type != ConfigFile {
if matches, err := verifyContentMatchesName(h.Name, buf); err == nil && !matches {
debug.Log("retry loading broken blob %v", h)
retriedInvalidData = true
return errors.Errorf("loadAll(%v): invalid data returned", h)
@ -57,86 +74,3 @@ type LimitedReadCloser struct {
func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
return &LimitedReadCloser{Closer: r, LimitedReader: io.LimitedReader{R: r, N: n}}
}
// DefaultLoad implements Backend.Load using lower-level openReader func
func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64,
openReader func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error),
fn func(rd io.Reader) error) error {
rd, err := openReader(ctx, h, length, offset)
if err != nil {
return err
}
err = fn(rd)
if err != nil {
_ = rd.Close() // ignore secondary errors closing the reader
return err
}
return rd.Close()
}
// DefaultDelete removes all restic keys in the bucket. It will not remove the bucket itself.
func DefaultDelete(ctx context.Context, be restic.Backend) error {
alltypes := []restic.FileType{
restic.PackFile,
restic.KeyFile,
restic.LockFile,
restic.SnapshotFile,
restic.IndexFile}
for _, t := range alltypes {
err := be.List(ctx, t, func(fi restic.FileInfo) error {
return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name})
})
if err != nil {
return nil
}
}
err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
if err != nil && be.IsNotExist(err) {
err = nil
}
return err
}
type memorizedLister struct {
fileInfos []restic.FileInfo
tpe restic.FileType
}
func (m *memorizedLister) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
if t != m.tpe {
return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t)
}
for _, fi := range m.fileInfos {
if ctx.Err() != nil {
break
}
err := fn(fi)
if err != nil {
return err
}
}
return ctx.Err()
}
func MemorizeList(ctx context.Context, be restic.Lister, t restic.FileType) (restic.Lister, error) {
if _, ok := be.(*memorizedLister); ok {
return be, nil
}
var fileInfos []restic.FileInfo
err := be.List(ctx, t, func(fi restic.FileInfo) error {
fileInfos = append(fileInfos, fi)
return nil
})
if err != nil {
return nil, err
}
return &memorizedLister{
fileInfos: fileInfos,
tpe: t,
}, nil
}

121
internal/backend/utils_test.go
View File

@ -3,7 +3,6 @@ package backend_test
import (
"bytes"
"context"
"fmt"
"io"
"math/rand"
"testing"
@ -11,7 +10,6 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -27,11 +25,11 @@ func TestLoadAll(t *testing.T) {
data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB)
id := restic.Hash(data)
h := restic.Handle{Name: id.String(), Type: restic.PackFile}
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher()))
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
rtest.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), buf, b, restic.Handle{Type: restic.PackFile, Name: id.String()})
buf, err := backend.LoadAll(context.TODO(), buf, b, backend.Handle{Type: backend.PackFile, Name: id.String()})
rtest.OK(t, err)
if len(buf) != len(data) {
@ -46,10 +44,10 @@ func TestLoadAll(t *testing.T) {
}
}
func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle {
func save(t testing.TB, be backend.Backend, buf []byte) backend.Handle {
id := restic.Hash(buf)
h := restic.Handle{Name: id.String(), Type: restic.PackFile}
err := be.Save(context.TODO(), h, restic.NewByteReader(buf, be.Hasher()))
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
err := be.Save(context.TODO(), h, backend.NewByteReader(buf, be.Hasher()))
if err != nil {
t.Fatal(err)
}
@ -57,10 +55,10 @@ func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle {
}
type quickRetryBackend struct {
restic.Backend
backend.Backend
}
func (be *quickRetryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *quickRetryBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
err := be.Backend.Load(ctx, h, length, offset, fn)
if err != nil {
// retry
@ -77,19 +75,19 @@ func TestLoadAllBroken(t *testing.T) {
// damage buffer
data[0] ^= 0xff
b.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
b.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(data)), nil
}
// must fail on first try
_, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.PackFile, Name: id.String()})
_, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.PackFile, Name: id.String()})
if err == nil {
t.Fatalf("missing expected error")
}
// must return the broken data after a retry
be := &quickRetryBackend{Backend: b}
buf, err := backend.LoadAll(context.TODO(), nil, be, restic.Handle{Type: restic.PackFile, Name: id.String()})
buf, err := backend.LoadAll(context.TODO(), nil, be, backend.Handle{Type: backend.PackFile, Name: id.String()})
rtest.OK(t, err)
if !bytes.Equal(buf, data) {
@ -105,7 +103,7 @@ func TestLoadAllAppend(t *testing.T) {
h2 := save(t, b, randomData)
var tests = []struct {
handle restic.Handle
handle backend.Handle
buf []byte
want []byte
}{
@ -149,98 +147,3 @@ func TestLoadAllAppend(t *testing.T) {
})
}
}
type mockReader struct {
closed bool
}
func (rd *mockReader) Read(_ []byte) (n int, err error) {
return 0, nil
}
func (rd *mockReader) Close() error {
rd.closed = true
return nil
}
func TestDefaultLoad(t *testing.T) {
h := restic.Handle{Name: "id", Type: restic.PackFile}
rd := &mockReader{}
// happy case, assert correct parameters are passed around and content stream is closed
err := backend.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) {
rtest.Equals(t, h, ih)
rtest.Equals(t, int(10), length)
rtest.Equals(t, int64(11), offset)
return rd, nil
}, func(ird io.Reader) error {
rtest.Equals(t, rd, ird)
return nil
})
rtest.OK(t, err)
rtest.Equals(t, true, rd.closed)
// unhappy case, assert producer errors are handled correctly
err = backend.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) {
return nil, errors.Errorf("producer error")
}, func(ird io.Reader) error {
t.Fatalf("unexpected consumer invocation")
return nil
})
rtest.Equals(t, "producer error", err.Error())
// unhappy case, assert consumer errors are handled correctly
rd = &mockReader{}
err = backend.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) {
return rd, nil
}, func(ird io.Reader) error {
return errors.Errorf("consumer error")
})
rtest.Equals(t, true, rd.closed)
rtest.Equals(t, "consumer error", err.Error())
}
func TestMemoizeList(t *testing.T) {
// setup backend to serve as data source for memoized list
be := mock.NewBackend()
files := []restic.FileInfo{
{Size: 42, Name: restic.NewRandomID().String()},
{Size: 45, Name: restic.NewRandomID().String()},
}
be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
for _, fi := range files {
if err := fn(fi); err != nil {
return err
}
}
return nil
}
mem, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile)
rtest.OK(t, err)
err = mem.List(context.TODO(), restic.IndexFile, func(fi restic.FileInfo) error {
t.Fatal("file type mismatch")
return nil // the memoized lister must return an error by itself
})
rtest.Assert(t, err != nil, "missing error on file typ mismatch")
var memFiles []restic.FileInfo
err = mem.List(context.TODO(), restic.SnapshotFile, func(fi restic.FileInfo) error {
memFiles = append(memFiles, fi)
return nil
})
rtest.OK(t, err)
rtest.Equals(t, files, memFiles)
}
func TestMemoizeListError(t *testing.T) {
// setup backend to serve as data source for memoized list
be := mock.NewBackend()
be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
return fmt.Errorf("list error")
}
_, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile)
rtest.Assert(t, err != nil, "missing error on list error")
}

36
internal/cache/backend.go vendored
View File

@ -5,35 +5,35 @@ import (
"io"
"sync"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
// Backend wraps a restic.Backend and adds a cache.
type Backend struct {
restic.Backend
backend.Backend
*Cache
// inProgress contains the handle for all files that are currently
// downloaded. The channel in the value is closed as soon as the download
// is finished.
inProgressMutex sync.Mutex
inProgress map[restic.Handle]chan struct{}
inProgress map[backend.Handle]chan struct{}
}
// ensure Backend implements restic.Backend
var _ restic.Backend = &Backend{}
// ensure Backend implements backend.Backend
var _ backend.Backend = &Backend{}
func newBackend(be restic.Backend, c *Cache) *Backend {
func newBackend(be backend.Backend, c *Cache) *Backend {
return &Backend{
Backend: be,
Cache: c,
inProgress: make(map[restic.Handle]chan struct{}),
inProgress: make(map[backend.Handle]chan struct{}),
}
}
// Remove deletes a file from the backend and the cache if it has been cached.
func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (b *Backend) Remove(ctx context.Context, h backend.Handle) error {
debug.Log("cache Remove(%v)", h)
err := b.Backend.Remove(ctx, h)
if err != nil {
@ -43,18 +43,18 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
return b.Cache.remove(h)
}
func autoCacheTypes(h restic.Handle) bool {
func autoCacheTypes(h backend.Handle) bool {
switch h.Type {
case restic.IndexFile, restic.SnapshotFile:
case backend.IndexFile, backend.SnapshotFile:
return true
case restic.PackFile:
return h.ContainedBlobType == restic.TreeBlob
case backend.PackFile:
return h.IsMetadata
}
return false
}
// Save stores a new file in the backend and the cache.
func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if !autoCacheTypes(h) {
return b.Backend.Save(ctx, h, rd)
}
@ -89,7 +89,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
return nil
}
func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error {
finish := make(chan struct{})
b.inProgressMutex.Lock()
@ -133,7 +133,7 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
}
// loadFromCache will try to load the file from the cache.
func (b *Backend) loadFromCache(h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
rd, err := b.Cache.load(h, length, offset)
if err != nil {
return false, err
@ -148,7 +148,7 @@ func (b *Backend) loadFromCache(h restic.Handle, length int, offset int64, consu
}
// Load loads a file from the cache or the backend.
func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
b.inProgressMutex.Lock()
waitForFinish, inProgress := b.inProgress[h]
b.inProgressMutex.Unlock()
@ -194,7 +194,7 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
// Stat tests whether the backend has a file. If it does not exist but still
// exists in the cache, it is removed from the cache.
func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
debug.Log("cache Stat(%v)", h)
fi, err := b.Backend.Stat(ctx, h)
@ -215,6 +215,6 @@ func (b *Backend) IsNotExist(err error) bool {
return b.Backend.IsNotExist(err)
}
func (b *Backend) Unwrap() restic.Backend {
func (b *Backend) Unwrap() backend.Backend {
return b.Backend
}

20
internal/cache/backend_test.go vendored
View File

@ -16,7 +16,7 @@ import (
"github.com/restic/restic/internal/test"
)
func loadAndCompare(t testing.TB, be restic.Backend, h restic.Handle, data []byte) {
func loadAndCompare(t testing.TB, be backend.Backend, h backend.Handle, data []byte) {
buf, err := backend.LoadAll(context.TODO(), nil, be, h)
if err != nil {
t.Fatal(err)
@ -31,25 +31,25 @@ func loadAndCompare(t testing.TB, be restic.Backend, h restic.Handle, data []byt
}
}
func save(t testing.TB, be restic.Backend, h restic.Handle, data []byte) {
err := be.Save(context.TODO(), h, restic.NewByteReader(data, be.Hasher()))
func save(t testing.TB, be backend.Backend, h backend.Handle, data []byte) {
err := be.Save(context.TODO(), h, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatal(err)
}
}
func remove(t testing.TB, be restic.Backend, h restic.Handle) {
func remove(t testing.TB, be backend.Backend, h backend.Handle) {
err := be.Remove(context.TODO(), h)
if err != nil {
t.Fatal(err)
}
}
func randomData(n int) (restic.Handle, []byte) {
func randomData(n int) (backend.Handle, []byte) {
data := test.Random(rand.Int(), n)
id := restic.Hash(data)
h := restic.Handle{
Type: restic.IndexFile,
h := backend.Handle{
Type: backend.IndexFile,
Name: id.String(),
}
return h, data
@ -114,11 +114,11 @@ func TestBackend(t *testing.T) {
}
type loadErrorBackend struct {
restic.Backend
backend.Backend
loadError error
}
func (be loadErrorBackend) Load(_ context.Context, _ restic.Handle, _ int, _ int64, _ func(rd io.Reader) error) error {
func (be loadErrorBackend) Load(_ context.Context, _ backend.Handle, _ int, _ int64, _ func(rd io.Reader) error) error {
time.Sleep(10 * time.Millisecond)
return be.loadError
}
@ -137,7 +137,7 @@ func TestErrorBackend(t *testing.T) {
loadError: testErr,
}
loadTest := func(wg *sync.WaitGroup, be restic.Backend) {
loadTest := func(wg *sync.WaitGroup, be backend.Backend) {
defer wg.Done()
buf, err := backend.LoadAll(context.TODO(), nil, be, h)

Some files were not shown because too many files have changed in this diff Show More