convert MemorizeList to be repository based

Ideally, code that uses a repository shouldn't directly interact with
the underlying backend. Thus, move MemorizeList one layer up.
This commit is contained in:
Michael Eischer 2023-10-01 13:05:56 +02:00
parent 1b8a67fe76
commit c7b770eb1f
42 changed files with 209 additions and 223 deletions

View File

@ -453,7 +453,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
f.Tags = []restic.TagList{opts.Tags.Flatten()} f.Tags = []restic.TagList{opts.Tags.Flatten()}
} }
sn, _, err := f.FindLatest(ctx, repo.Backend(), repo, snName) sn, _, err := f.FindLatest(ctx, repo, repo, snName)
// Snapshot not found is ok if no explicit parent was set // Snapshot not found is ok if no explicit parent was set
if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) { if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
err = nil err = nil

View File

@ -106,7 +106,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
Println(string(buf)) Println(string(buf))
return nil return nil
case "snapshot": case "snapshot":
sn, _, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1]) sn, _, err := restic.FindSnapshot(ctx, repo, repo, args[1])
if err != nil { if err != nil {
return errors.Fatalf("could not find snapshot: %v\n", err) return errors.Fatalf("could not find snapshot: %v\n", err)
} }
@ -193,7 +193,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
return errors.Fatal("blob not found") return errors.Fatal("blob not found")
case "tree": case "tree":
sn, subfolder, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1]) sn, subfolder, err := restic.FindSnapshot(ctx, repo, repo, args[1])
if err != nil { if err != nil {
return errors.Fatalf("could not find snapshot: %v\n", err) return errors.Fatalf("could not find snapshot: %v\n", err)
} }

View File

@ -4,7 +4,6 @@ import (
"context" "context"
"fmt" "fmt"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
@ -88,12 +87,12 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
return err return err
} }
srcSnapshotLister, err := backend.MemorizeList(ctx, srcRepo.Backend(), restic.SnapshotFile) srcSnapshotLister, err := restic.MemorizeList(ctx, srcRepo, restic.SnapshotFile)
if err != nil { if err != nil {
return err return err
} }
dstSnapshotLister, err := backend.MemorizeList(ctx, dstRepo.Backend(), restic.SnapshotFile) dstSnapshotLister, err := restic.MemorizeList(ctx, dstRepo, restic.SnapshotFile)
if err != nil { if err != nil {
return err return err
} }

View File

@ -78,7 +78,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
} }
func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error { func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
return restic.ForAllSnapshots(ctx, repo.Backend(), repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error { return restic.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
if err != nil { if err != nil {
return err return err
} }
@ -107,7 +107,7 @@ type Blob struct {
func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error { func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
var m sync.Mutex var m sync.Mutex
return restic.ParallelList(ctx, repo.Backend(), restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error { return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
blobs, _, err := repo.ListPack(ctx, id, size) blobs, _, err := repo.ListPack(ctx, id, size)
if err != nil { if err != nil {
Warnf("error for pack %v: %v\n", id.Str(), err) Warnf("error for pack %v: %v\n", id.Str(), err)
@ -134,7 +134,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
} }
func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error { func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
Printf("index_id: %v\n", id) Printf("index_id: %v\n", id)
if err != nil { if err != nil {
return err return err
@ -447,7 +447,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
for _, name := range args { for _, name := range args {
id, err := restic.ParseID(name) id, err := restic.ParseID(name)
if err != nil { if err != nil {
id, err = restic.Find(ctx, repo.Backend(), restic.PackFile, name) id, err = restic.Find(ctx, repo, restic.PackFile, name)
if err != nil { if err != nil {
Warnf("error: %v\n", err) Warnf("error: %v\n", err)
continue continue

View File

@ -7,7 +7,6 @@ import (
"reflect" "reflect"
"sort" "sort"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -58,7 +57,7 @@ func init() {
f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata") f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
} }
func loadSnapshot(ctx context.Context, be backend.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) { func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) {
sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc) sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc)
if err != nil { if err != nil {
return nil, "", errors.Fatal(err.Error()) return nil, "", errors.Fatal(err.Error())
@ -346,7 +345,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
} }
// cache snapshots listing // cache snapshots listing
be, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) be, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil { if err != nil {
return err return err
} }

View File

@ -147,7 +147,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
Hosts: opts.Hosts, Hosts: opts.Hosts,
Paths: opts.Paths, Paths: opts.Paths,
Tags: opts.Tags, Tags: opts.Tags,
}).FindLatest(ctx, repo.Backend(), repo, snapshotIDString) }).FindLatest(ctx, repo, repo, snapshotIDString)
if err != nil { if err != nil {
return errors.Fatalf("failed to find snapshot: %v", err) return errors.Fatalf("failed to find snapshot: %v", err)
} }

View File

@ -9,7 +9,6 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/filter"
@ -584,7 +583,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
} }
} }
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil { if err != nil {
return err return err
} }

View File

@ -183,7 +183,7 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
var snapshots restic.Snapshots var snapshots restic.Snapshots
removeSnIDs := restic.NewIDSet() removeSnIDs := restic.NewIDSet()
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
snapshots = append(snapshots, sn) snapshots = append(snapshots, sn)
} }

View File

@ -60,7 +60,7 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions
var m sync.Mutex var m sync.Mutex
var keys []keyInfo var keys []keyInfo
err := restic.ParallelList(ctx, s.Backend(), restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error { err := restic.ParallelList(ctx, s, restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
k, err := repository.LoadKey(ctx, s, id) k, err := repository.LoadKey(ctx, s, id)
if err != nil { if err != nil {
Warnf("LoadKey() failed: %v\n", err) Warnf("LoadKey() failed: %v\n", err)
@ -238,7 +238,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error {
return err return err
} }
id, err := restic.Find(ctx, repo.Backend(), restic.KeyFile, args[1]) id, err := restic.Find(ctx, repo, restic.KeyFile, args[1])
if err != nil { if err != nil {
return err return err
} }

View File

@ -63,7 +63,7 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args
case "locks": case "locks":
t = restic.LockFile t = restic.LockFile
case "blobs": case "blobs":
return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil { if err != nil {
return err return err
} }

View File

@ -9,7 +9,6 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -170,7 +169,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
return err return err
} }
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil { if err != nil {
return err return err
} }

View File

@ -810,7 +810,7 @@ func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Rep
func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) { func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) {
var snapshotTrees restic.IDs var snapshotTrees restic.IDs
Verbosef("loading all snapshots...\n") Verbosef("loading all snapshots...\n")
err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, ignoreSnapshots, err = restic.ForAllSnapshots(ctx, repo, repo, ignoreSnapshots,
func(id restic.ID, sn *restic.Snapshot, err error) error { func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil { if err != nil {
debug.Log("failed to load snapshot %v (error %v)", id, err) debug.Log("failed to load snapshot %v (error %v)", id, err)

View File

@ -5,7 +5,6 @@ import (
"os" "os"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@ -52,7 +51,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
return err return err
} }
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil { if err != nil {
return err return err
} }

View File

@ -88,7 +88,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti
} else { } else {
Verbosef("loading indexes...\n") Verbosef("loading indexes...\n")
mi := index.NewMasterIndex() mi := index.NewMasterIndex()
err := index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { err := index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil { if err != nil {
Warnf("removing invalid index %v: %v\n", id, err) Warnf("removing invalid index %v: %v\n", id, err)
obsoleteIndexes = append(obsoleteIndexes, id) obsoleteIndexes = append(obsoleteIndexes, id)

View File

@ -3,7 +3,6 @@ package main
import ( import (
"context" "context"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/walker" "github.com/restic/restic/internal/walker"
@ -84,7 +83,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
repo.SetDryRun() repo.SetDryRun()
} }
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil { if err != nil {
return err return err
} }

View File

@ -168,7 +168,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
Hosts: opts.Hosts, Hosts: opts.Hosts,
Paths: opts.Paths, Paths: opts.Paths,
Tags: opts.Tags, Tags: opts.Tags,
}).FindLatest(ctx, repo.Backend(), repo, snapshotIDString) }).FindLatest(ctx, repo, repo, snapshotIDString)
if err != nil { if err != nil {
return errors.Fatalf("failed to find snapshot: %v", err) return errors.Fatalf("failed to find snapshot: %v", err)
} }

View File

@ -207,7 +207,7 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
repo.SetDryRun() repo.SetDryRun()
} }
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil { if err != nil {
return err return err
} }

View File

@ -73,7 +73,7 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions
} }
var snapshots restic.Snapshots var snapshots restic.Snapshots
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
snapshots = append(snapshots, sn) snapshots = append(snapshots, sn)
} }
snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy) snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy)

View File

@ -8,7 +8,6 @@ import (
"strings" "strings"
"github.com/restic/chunker" "github.com/restic/chunker"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -94,7 +93,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args
} }
} }
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil { if err != nil {
return err return err
} }

View File

@ -120,7 +120,7 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st
} }
changeCnt := 0 changeCnt := 0
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten()) changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten())
if err != nil { if err != nil {
Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err) Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err)

View File

@ -3,7 +3,6 @@ package main
import ( import (
"context" "context"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/spf13/pflag" "github.com/spf13/pflag"
) )
@ -29,11 +28,11 @@ func initSingleSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter)
} }
// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. // FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
func FindFilteredSnapshots(ctx context.Context, be backend.Lister, loader restic.LoaderUnpacked, f *restic.SnapshotFilter, snapshotIDs []string) <-chan *restic.Snapshot { func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, f *restic.SnapshotFilter, snapshotIDs []string) <-chan *restic.Snapshot {
out := make(chan *restic.Snapshot) out := make(chan *restic.Snapshot)
go func() { go func() {
defer close(out) defer close(out)
be, err := backend.MemorizeList(ctx, be, restic.SnapshotFile) be, err := restic.MemorizeList(ctx, be, restic.SnapshotFile)
if err != nil { if err != nil {
Warnf("could not load snapshots: %v\n", err) Warnf("could not load snapshots: %v\n", err)
return return

View File

@ -159,7 +159,7 @@ func TestFindListOnce(t *testing.T) {
snapshotIDs := restic.NewIDSet() snapshotIDs := restic.NewIDSet()
// specify the two oldest snapshots explicitly and use "latest" to reference the newest one // specify the two oldest snapshots explicitly and use "latest" to reference the newest one
for sn := range FindFilteredSnapshots(context.TODO(), repo.Backend(), repo, &restic.SnapshotFilter{}, []string{ for sn := range FindFilteredSnapshots(context.TODO(), repo, repo, &restic.SnapshotFilter{}, []string{
secondSnapshot[0].String(), secondSnapshot[0].String(),
secondSnapshot[1].String()[:8], secondSnapshot[1].String()[:8],
"latest", "latest",

View File

@ -27,7 +27,7 @@ type Backend interface {
// HasAtomicReplace returns whether Save() can atomically replace files // HasAtomicReplace returns whether Save() can atomically replace files
HasAtomicReplace() bool HasAtomicReplace() bool
// Remove removes a File described by h. // Remove removes a File described by h.
Remove(ctx context.Context, h Handle) error Remove(ctx context.Context, h Handle) error
// Close the backend // Close the backend
@ -110,8 +110,3 @@ type FileInfo struct {
type ApplyEnvironmenter interface { type ApplyEnvironmenter interface {
ApplyEnvironment(prefix string) ApplyEnvironment(prefix string)
} }
// Lister allows listing files in a backend.
type Lister interface {
List(context.Context, FileType, func(FileInfo) error) error
}

View File

@ -74,44 +74,3 @@ type LimitedReadCloser struct {
func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser { func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
return &LimitedReadCloser{Closer: r, LimitedReader: io.LimitedReader{R: r, N: n}} return &LimitedReadCloser{Closer: r, LimitedReader: io.LimitedReader{R: r, N: n}}
} }
type memorizedLister struct {
fileInfos []FileInfo
tpe FileType
}
func (m *memorizedLister) List(ctx context.Context, t FileType, fn func(FileInfo) error) error {
if t != m.tpe {
return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t)
}
for _, fi := range m.fileInfos {
if ctx.Err() != nil {
break
}
err := fn(fi)
if err != nil {
return err
}
}
return ctx.Err()
}
func MemorizeList(ctx context.Context, be Lister, t FileType) (Lister, error) {
if _, ok := be.(*memorizedLister); ok {
return be, nil
}
var fileInfos []FileInfo
err := be.List(ctx, t, func(fi FileInfo) error {
fileInfos = append(fileInfos, fi)
return nil
})
if err != nil {
return nil, err
}
return &memorizedLister{
fileInfos: fileInfos,
tpe: t,
}, nil
}

View File

@ -3,7 +3,6 @@ package backend_test
import ( import (
"bytes" "bytes"
"context" "context"
"fmt"
"io" "io"
"math/rand" "math/rand"
"testing" "testing"
@ -148,47 +147,3 @@ func TestLoadAllAppend(t *testing.T) {
}) })
} }
} }
func TestMemoizeList(t *testing.T) {
// setup backend to serve as data source for memoized list
be := mock.NewBackend()
files := []backend.FileInfo{
{Size: 42, Name: restic.NewRandomID().String()},
{Size: 45, Name: restic.NewRandomID().String()},
}
be.ListFn = func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
for _, fi := range files {
if err := fn(fi); err != nil {
return err
}
}
return nil
}
mem, err := backend.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.OK(t, err)
err = mem.List(context.TODO(), backend.IndexFile, func(fi backend.FileInfo) error {
t.Fatal("file type mismatch")
return nil // the memoized lister must return an error by itself
})
rtest.Assert(t, err != nil, "missing error on file typ mismatch")
var memFiles []backend.FileInfo
err = mem.List(context.TODO(), backend.SnapshotFile, func(fi backend.FileInfo) error {
memFiles = append(memFiles, fi)
return nil
})
rtest.OK(t, err)
rtest.Equals(t, files, memFiles)
}
func TestMemoizeListError(t *testing.T) {
// setup backend to serve as data source for memoized list
be := mock.NewBackend()
be.ListFn = func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
return fmt.Errorf("list error")
}
_, err := backend.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.Assert(t, err != nil, "missing error on list error")
}

View File

@ -39,7 +39,7 @@ type Checker struct {
trackUnused bool trackUnused bool
masterIndex *index.MasterIndex masterIndex *index.MasterIndex
snapshots backend.Lister snapshots restic.Lister
repo restic.Repository repo restic.Repository
} }
@ -102,7 +102,7 @@ func (e *ErrPackData) Error() string {
func (c *Checker) LoadSnapshots(ctx context.Context) error { func (c *Checker) LoadSnapshots(ctx context.Context) error {
var err error var err error
c.snapshots, err = backend.MemorizeList(ctx, c.repo.Backend(), restic.SnapshotFile) c.snapshots, err = restic.MemorizeList(ctx, c.repo, restic.SnapshotFile)
return err return err
} }
@ -126,7 +126,7 @@ func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID
func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []error, errs []error) { func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []error, errs []error) {
debug.Log("Start") debug.Log("Start")
indexList, err := backend.MemorizeList(ctx, c.repo.Backend(), restic.IndexFile) indexList, err := restic.MemorizeList(ctx, c.repo, restic.IndexFile)
if err != nil { if err != nil {
// abort if an error occurs while listing the indexes // abort if an error occurs while listing the indexes
return hints, append(errs, err) return hints, append(errs, err)
@ -134,13 +134,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
if p != nil { if p != nil {
var numIndexFiles uint64 var numIndexFiles uint64
err := indexList.List(ctx, restic.IndexFile, func(fi backend.FileInfo) error { err := indexList.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
_, err := restic.ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
return nil
}
numIndexFiles++ numIndexFiles++
return nil return nil
}) })
@ -367,7 +361,7 @@ func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan restic.TreeI
} }
} }
func loadSnapshotTreeIDs(ctx context.Context, lister backend.Lister, repo restic.Repository) (ids restic.IDs, errs []error) { func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.Repository) (ids restic.IDs, errs []error) {
err := restic.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { err := restic.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil { if err != nil {
errs = append(errs, err) errs = append(errs, err)

View File

@ -295,7 +295,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
} }
var snapshots restic.Snapshots var snapshots restic.Snapshots
err := d.root.cfg.Filter.FindAll(ctx, d.root.repo.Backend(), d.root.repo, nil, func(id string, sn *restic.Snapshot, err error) error { err := d.root.cfg.Filter.FindAll(ctx, d.root.repo, d.root.repo, nil, func(id string, sn *restic.Snapshot, err error) error {
if sn != nil { if sn != nil {
snapshots = append(snapshots, sn) snapshots = append(snapshots, sn)
} }

View File

@ -5,14 +5,13 @@ import (
"runtime" "runtime"
"sync" "sync"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
) )
// ForAllIndexes loads all index files in parallel and calls the given callback. // ForAllIndexes loads all index files in parallel and calls the given callback.
// It is guaranteed that the function is not run concurrently. If the callback // It is guaranteed that the function is not run concurrently. If the callback
// returns an error, this function is cancelled and also returns that error. // returns an error, this function is cancelled and also returns that error.
func ForAllIndexes(ctx context.Context, lister backend.Lister, repo restic.Repository, func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Repository,
fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error { fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error {
// decoding an index can take quite some time such that this can be both CPU- or IO-bound // decoding an index can take quite some time such that this can be both CPU- or IO-bound

View File

@ -29,7 +29,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
// check that all expected indexes are loaded without errors // check that all expected indexes are loaded without errors
indexIDs := restic.NewIDSet() indexIDs := restic.NewIDSet()
var indexErr error var indexErr error
rtest.OK(t, index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { rtest.OK(t, index.ForAllIndexes(context.TODO(), repo, repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
if err != nil { if err != nil {
indexErr = err indexErr = err
} }
@ -42,7 +42,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
// must failed with the returned error // must failed with the returned error
iterErr := errors.New("error to pass upwards") iterErr := errors.New("error to pass upwards")
err := index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { err := index.ForAllIndexes(context.TODO(), repo, repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
return iterErr return iterErr
}) })

View File

@ -116,7 +116,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int,
checked := 0 checked := 0
if len(keyHint) > 0 { if len(keyHint) > 0 {
id, err := restic.Find(ctx, s.Backend(), restic.KeyFile, keyHint) id, err := restic.Find(ctx, s, restic.KeyFile, keyHint)
if err == nil { if err == nil {
key, err := OpenKey(ctx, s, id, password) key, err := OpenKey(ctx, s, id, password)

View File

@ -584,20 +584,14 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error {
func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
debug.Log("Loading index") debug.Log("Loading index")
indexList, err := backend.MemorizeList(ctx, r.Backend(), restic.IndexFile) indexList, err := restic.MemorizeList(ctx, r, restic.IndexFile)
if err != nil { if err != nil {
return err return err
} }
if p != nil { if p != nil {
var numIndexFiles uint64 var numIndexFiles uint64
err := indexList.List(ctx, restic.IndexFile, func(fi backend.FileInfo) error { err := indexList.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
_, err := restic.ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
return nil
}
numIndexFiles++ numIndexFiles++
return nil return nil
}) })

View File

@ -3,9 +3,6 @@ package restic
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
) )
// A MultipleIDMatchesError is returned by Find() when multiple IDs with a // A MultipleIDMatchesError is returned by Find() when multiple IDs with a
@ -27,21 +24,15 @@ func (e *NoIDByPrefixError) Error() string {
// Find loads the list of all files of type t and searches for names which // Find loads the list of all files of type t and searches for names which
// start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned. // start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned.
// If more than one is found, nil and ErrMultipleIDMatches is returned. // If more than one is found, nil and ErrMultipleIDMatches is returned.
func Find(ctx context.Context, be backend.Lister, t FileType, prefix string) (ID, error) { func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error) {
match := ID{} match := ID{}
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
err := be.List(ctx, t, func(fi backend.FileInfo) error { err := be.List(ctx, t, func(id ID, size int64) error {
// ignore filename which are not an id name := id.String()
id, err := ParseID(fi.Name) if len(name) >= len(prefix) && prefix == name[:len(prefix)] {
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
return nil
}
if len(fi.Name) >= len(prefix) && prefix == fi.Name[:len(prefix)] {
if match.IsNull() { if match.IsNull() {
match = id match = id
} else { } else {

View File

@ -1,39 +1,31 @@
package restic package restic_test
import ( import (
"context" "context"
"strings" "strings"
"testing" "testing"
"github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic"
) )
type mockBackend struct { var samples = restic.IDs{
list func(context.Context, FileType, func(backend.FileInfo) error) error restic.TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"),
} restic.TestParseID("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"),
restic.TestParseID("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"),
func (m mockBackend) List(ctx context.Context, t FileType, fn func(backend.FileInfo) error) error { restic.TestParseID("20ff988befa5fc40350f00d531a767606efefe242c837aaccb80673f286be53d"),
return m.list(ctx, t, fn) restic.TestParseID("326cb59dfe802304f96ee9b5b9af93bdee73a30f53981e5ec579aedb6f1d0f07"),
} restic.TestParseID("86b60b9594d1d429c4aa98fa9562082cabf53b98c7dc083abe5dae31074dd15a"),
restic.TestParseID("96c8dbe225079e624b5ce509f5bd817d1453cd0a85d30d536d01b64a8669aeae"),
var samples = IDs{ restic.TestParseID("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"),
TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"),
TestParseID("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"),
TestParseID("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"),
TestParseID("20ff988befa5fc40350f00d531a767606efefe242c837aaccb80673f286be53d"),
TestParseID("326cb59dfe802304f96ee9b5b9af93bdee73a30f53981e5ec579aedb6f1d0f07"),
TestParseID("86b60b9594d1d429c4aa98fa9562082cabf53b98c7dc083abe5dae31074dd15a"),
TestParseID("96c8dbe225079e624b5ce509f5bd817d1453cd0a85d30d536d01b64a8669aeae"),
TestParseID("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"),
} }
func TestFind(t *testing.T) { func TestFind(t *testing.T) {
list := samples list := samples
m := mockBackend{} m := &ListHelper{}
m.list = func(ctx context.Context, t FileType, fn func(backend.FileInfo) error) error { m.ListFn = func(ctx context.Context, t restic.FileType, fn func(id restic.ID, size int64) error) error {
for _, id := range list { for _, id := range list {
err := fn(backend.FileInfo{Name: id.String()}) err := fn(id, 0)
if err != nil { if err != nil {
return err return err
} }
@ -41,17 +33,17 @@ func TestFind(t *testing.T) {
return nil return nil
} }
f, err := Find(context.TODO(), m, SnapshotFile, "20bdc1402a6fc9b633aa") f, err := restic.Find(context.TODO(), m, restic.SnapshotFile, "20bdc1402a6fc9b633aa")
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
expectedMatch := TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff") expectedMatch := restic.TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff")
if f != expectedMatch { if f != expectedMatch {
t.Errorf("Wrong match returned want %s, got %s", expectedMatch, f) t.Errorf("Wrong match returned want %s, got %s", expectedMatch, f)
} }
f, err = Find(context.TODO(), m, SnapshotFile, "NotAPrefix") f, err = restic.Find(context.TODO(), m, restic.SnapshotFile, "NotAPrefix")
if _, ok := err.(*NoIDByPrefixError); !ok || !strings.Contains(err.Error(), "NotAPrefix") { if _, ok := err.(*restic.NoIDByPrefixError); !ok || !strings.Contains(err.Error(), "NotAPrefix") {
t.Error("Expected no snapshots to be found.") t.Error("Expected no snapshots to be found.")
} }
if !f.IsNull() { if !f.IsNull() {
@ -60,8 +52,8 @@ func TestFind(t *testing.T) {
// Try to match with a prefix longer than any ID. // Try to match with a prefix longer than any ID.
extraLengthID := samples[0].String() + "f" extraLengthID := samples[0].String() + "f"
f, err = Find(context.TODO(), m, SnapshotFile, extraLengthID) f, err = restic.Find(context.TODO(), m, restic.SnapshotFile, extraLengthID)
if _, ok := err.(*NoIDByPrefixError); !ok || !strings.Contains(err.Error(), extraLengthID) { if _, ok := err.(*restic.NoIDByPrefixError); !ok || !strings.Contains(err.Error(), extraLengthID) {
t.Errorf("Wrong error %v for no snapshots matched", err) t.Errorf("Wrong error %v for no snapshots matched", err)
} }
if !f.IsNull() { if !f.IsNull() {
@ -69,8 +61,8 @@ func TestFind(t *testing.T) {
} }
// Use a prefix that will match the prefix of multiple Ids in `samples`. // Use a prefix that will match the prefix of multiple Ids in `samples`.
f, err = Find(context.TODO(), m, SnapshotFile, "20bdc140") f, err = restic.Find(context.TODO(), m, restic.SnapshotFile, "20bdc140")
if _, ok := err.(*MultipleIDMatchesError); !ok { if _, ok := err.(*restic.MultipleIDMatchesError); !ok {
t.Errorf("Wrong error %v for multiple snapshots", err) t.Errorf("Wrong error %v for multiple snapshots", err)
} }
if !f.IsNull() { if !f.IsNull() {

52
internal/restic/lister.go Normal file
View File

@ -0,0 +1,52 @@
package restic
import (
"context"
"fmt"
)
type fileInfo struct {
id ID
size int64
}
type memorizedLister struct {
fileInfos []fileInfo
tpe FileType
}
func (m *memorizedLister) List(ctx context.Context, t FileType, fn func(ID, int64) error) error {
if t != m.tpe {
return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t)
}
for _, fi := range m.fileInfos {
if ctx.Err() != nil {
break
}
err := fn(fi.id, fi.size)
if err != nil {
return err
}
}
return ctx.Err()
}
func MemorizeList(ctx context.Context, be Lister, t FileType) (Lister, error) {
if _, ok := be.(*memorizedLister); ok {
return be, nil
}
var fileInfos []fileInfo
err := be.List(ctx, t, func(id ID, size int64) error {
fileInfos = append(fileInfos, fileInfo{id, size})
return nil
})
if err != nil {
return nil, err
}
return &memorizedLister{
fileInfos: fileInfos,
tpe: t,
}, nil
}

View File

@ -0,0 +1,68 @@
package restic_test
import (
"context"
"fmt"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
type ListHelper struct {
ListFn func(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error
}
func (l *ListHelper) List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error {
return l.ListFn(ctx, t, fn)
}
func TestMemoizeList(t *testing.T) {
// setup backend to serve as data source for memoized list
be := &ListHelper{}
type FileInfo struct {
ID restic.ID
Size int64
}
files := []FileInfo{
{ID: restic.NewRandomID(), Size: 42},
{ID: restic.NewRandomID(), Size: 45},
}
be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error {
for _, fi := range files {
if err := fn(fi.ID, fi.Size); err != nil {
return err
}
}
return nil
}
mem, err := restic.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.OK(t, err)
err = mem.List(context.TODO(), backend.IndexFile, func(id restic.ID, size int64) error {
t.Fatal("file type mismatch")
return nil // the memoized lister must return an error by itself
})
rtest.Assert(t, err != nil, "missing error on file typ mismatch")
var memFiles []FileInfo
err = mem.List(context.TODO(), backend.SnapshotFile, func(id restic.ID, size int64) error {
memFiles = append(memFiles, FileInfo{ID: id, Size: size})
return nil
})
rtest.OK(t, err)
rtest.Equals(t, files, memFiles)
}
func TestMemoizeListError(t *testing.T) {
// setup backend to serve as data source for memoized list
be := &ListHelper{}
be.ListFn = func(ctx context.Context, t backend.FileType, fn func(restic.ID, int64) error) error {
return fmt.Errorf("list error")
}
_, err := restic.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.Assert(t, err != nil, "missing error on list error")
}

View File

@ -403,7 +403,7 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) {
// RemoveAllLocks removes all locks forcefully. // RemoveAllLocks removes all locks forcefully.
func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) { func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) {
var processed uint32 var processed uint32
err := ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error { err := ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error {
err := repo.Backend().Remove(ctx, backend.Handle{Type: LockFile, Name: id.String()}) err := repo.Backend().Remove(ctx, backend.Handle{Type: LockFile, Name: id.String()})
if err == nil { if err == nil {
atomic.AddUint32(&processed, 1) atomic.AddUint32(&processed, 1)
@ -421,7 +421,7 @@ func ForAllLocks(ctx context.Context, repo Repository, excludeID *ID, fn func(ID
var m sync.Mutex var m sync.Mutex
// For locks decoding is nearly for free, thus just assume were only limited by IO // For locks decoding is nearly for free, thus just assume were only limited by IO
return ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error { return ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error {
if excludeID != nil && id.Equal(*excludeID) { if excludeID != nil && id.Equal(*excludeID) {
return nil return nil
} }

View File

@ -3,13 +3,11 @@ package restic
import ( import (
"context" "context"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
func ParallelList(ctx context.Context, r backend.Lister, t FileType, parallelism uint, fn func(context.Context, ID, int64) error) error { func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, fn func(context.Context, ID, int64) error) error {
type FileIDInfo struct { type FileIDInfo struct {
ID ID
Size int64 Size int64
@ -23,17 +21,11 @@ func ParallelList(ctx context.Context, r backend.Lister, t FileType, parallelism
// send list of index files through ch, which is closed afterwards // send list of index files through ch, which is closed afterwards
wg.Go(func() error { wg.Go(func() error {
defer close(ch) defer close(ch)
return r.List(ctx, t, func(fi backend.FileInfo) error { return r.List(ctx, t, func(id ID, size int64) error {
id, err := ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
return nil
}
select { select {
case <-ctx.Done(): case <-ctx.Done():
return nil return nil
case ch <- FileIDInfo{id, fi.Size}: case ch <- FileIDInfo{id, size}:
} }
return nil return nil
}) })

View File

@ -100,3 +100,8 @@ type MasterIndex interface {
Save(ctx context.Context, repo SaverUnpacked, packBlacklist IDSet, extraObsolete IDs, p *progress.Counter) (obsolete IDSet, err error) Save(ctx context.Context, repo SaverUnpacked, packBlacklist IDSet, extraObsolete IDs, p *progress.Counter) (obsolete IDSet, err error)
} }
// Lister allows listing files in a backend.
type Lister interface {
List(ctx context.Context, t FileType, fn func(ID, int64) error) error
}

View File

@ -8,7 +8,6 @@ import (
"sync" "sync"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
) )
@ -80,7 +79,7 @@ func SaveSnapshot(ctx context.Context, repo SaverUnpacked, sn *Snapshot) (ID, er
// If the called function returns an error, this function is cancelled and // If the called function returns an error, this function is cancelled and
// also returns this error. // also returns this error.
// If a snapshot ID is in excludeIDs, it will be ignored. // If a snapshot ID is in excludeIDs, it will be ignored.
func ForAllSnapshots(ctx context.Context, be backend.Lister, loader LoaderUnpacked, excludeIDs IDSet, fn func(ID, *Snapshot, error) error) error { func ForAllSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, excludeIDs IDSet, fn func(ID, *Snapshot, error) error) error {
var m sync.Mutex var m sync.Mutex
// For most snapshots decoding is nearly for free, thus just assume were only limited by IO // For most snapshots decoding is nearly for free, thus just assume were only limited by IO

View File

@ -7,7 +7,6 @@ import (
"strings" "strings"
"time" "time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
) )
@ -35,7 +34,7 @@ func (f *SnapshotFilter) matches(sn *Snapshot) bool {
// findLatest finds the latest snapshot with optional target/directory, // findLatest finds the latest snapshot with optional target/directory,
// tags, hostname, and timestamp filters. // tags, hostname, and timestamp filters.
func (f *SnapshotFilter) findLatest(ctx context.Context, be backend.Lister, loader LoaderUnpacked) (*Snapshot, error) { func (f *SnapshotFilter) findLatest(ctx context.Context, be Lister, loader LoaderUnpacked) (*Snapshot, error) {
var err error var err error
absTargets := make([]string, 0, len(f.Paths)) absTargets := make([]string, 0, len(f.Paths))
@ -91,7 +90,7 @@ func splitSnapshotID(s string) (id, subfolder string) {
// FindSnapshot takes a string and tries to find a snapshot whose ID matches // FindSnapshot takes a string and tries to find a snapshot whose ID matches
// the string as closely as possible. // the string as closely as possible.
func FindSnapshot(ctx context.Context, be backend.Lister, loader LoaderUnpacked, s string) (*Snapshot, string, error) { func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s string) (*Snapshot, string, error) {
s, subfolder := splitSnapshotID(s) s, subfolder := splitSnapshotID(s)
// no need to list snapshots if `s` is already a full id // no need to list snapshots if `s` is already a full id
@ -109,7 +108,7 @@ func FindSnapshot(ctx context.Context, be backend.Lister, loader LoaderUnpacked,
// FindLatest returns either the latest of a filtered list of all snapshots // FindLatest returns either the latest of a filtered list of all snapshots
// or a snapshot specified by `snapshotID`. // or a snapshot specified by `snapshotID`.
func (f *SnapshotFilter) FindLatest(ctx context.Context, be backend.Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, string, error) { func (f *SnapshotFilter) FindLatest(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, string, error) {
id, subfolder := splitSnapshotID(snapshotID) id, subfolder := splitSnapshotID(snapshotID)
if id == "latest" { if id == "latest" {
sn, err := f.findLatest(ctx, be, loader) sn, err := f.findLatest(ctx, be, loader)
@ -127,7 +126,7 @@ type SnapshotFindCb func(string, *Snapshot, error) error
var ErrInvalidSnapshotSyntax = errors.New("<snapshot>:<subfolder> syntax not allowed") var ErrInvalidSnapshotSyntax = errors.New("<snapshot>:<subfolder> syntax not allowed")
// FindAll yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. // FindAll yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
func (f *SnapshotFilter) FindAll(ctx context.Context, be backend.Lister, loader LoaderUnpacked, snapshotIDs []string, fn SnapshotFindCb) error { func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotIDs []string, fn SnapshotFindCb) error {
if len(snapshotIDs) != 0 { if len(snapshotIDs) != 0 {
var err error var err error
usedFilter := false usedFilter := false

View File

@ -16,7 +16,7 @@ func TestFindLatestSnapshot(t *testing.T) {
latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1) latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1)
f := restic.SnapshotFilter{Hosts: []string{"foo"}} f := restic.SnapshotFilter{Hosts: []string{"foo"}}
sn, _, err := f.FindLatest(context.TODO(), repo.Backend(), repo, "latest") sn, _, err := f.FindLatest(context.TODO(), repo, repo, "latest")
if err != nil { if err != nil {
t.Fatalf("FindLatest returned error: %v", err) t.Fatalf("FindLatest returned error: %v", err)
} }
@ -35,7 +35,7 @@ func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) {
sn, _, err := (&restic.SnapshotFilter{ sn, _, err := (&restic.SnapshotFilter{
Hosts: []string{"foo"}, Hosts: []string{"foo"},
TimestampLimit: parseTimeUTC("2018-08-08 08:08:08"), TimestampLimit: parseTimeUTC("2018-08-08 08:08:08"),
}).FindLatest(context.TODO(), repo.Backend(), repo, "latest") }).FindLatest(context.TODO(), repo, repo, "latest")
if err != nil { if err != nil {
t.Fatalf("FindLatest returned error: %v", err) t.Fatalf("FindLatest returned error: %v", err)
} }
@ -62,7 +62,7 @@ func TestFindLatestWithSubpath(t *testing.T) {
{desiredSnapshot.ID().String() + ":subfolder", "subfolder"}, {desiredSnapshot.ID().String() + ":subfolder", "subfolder"},
} { } {
t.Run("", func(t *testing.T) { t.Run("", func(t *testing.T) {
sn, subfolder, err := (&restic.SnapshotFilter{}).FindLatest(context.TODO(), repo.Backend(), repo, exp.query) sn, subfolder, err := (&restic.SnapshotFilter{}).FindLatest(context.TODO(), repo, repo, exp.query)
if err != nil { if err != nil {
t.Fatalf("FindLatest returned error: %v", err) t.Fatalf("FindLatest returned error: %v", err)
} }
@ -78,7 +78,7 @@ func TestFindAllSubpathError(t *testing.T) {
desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1)
count := 0 count := 0
test.OK(t, (&restic.SnapshotFilter{}).FindAll(context.TODO(), repo.Backend(), repo, test.OK(t, (&restic.SnapshotFilter{}).FindAll(context.TODO(), repo, repo,
[]string{"latest:subfolder", desiredSnapshot.ID().Str() + ":subfolder"}, []string{"latest:subfolder", desiredSnapshot.ID().Str() + ":subfolder"},
func(id string, sn *restic.Snapshot, err error) error { func(id string, sn *restic.Snapshot, err error) error {
if err == restic.ErrInvalidSnapshotSyntax { if err == restic.ErrInvalidSnapshotSyntax {

View File

@ -20,7 +20,7 @@ const (
// LoadAllSnapshots returns a list of all snapshots in the repo. // LoadAllSnapshots returns a list of all snapshots in the repo.
// If a snapshot ID is in excludeIDs, it will not be included in the result. // If a snapshot ID is in excludeIDs, it will not be included in the result.
func loadAllSnapshots(ctx context.Context, repo restic.Repository, excludeIDs restic.IDSet) (snapshots restic.Snapshots, err error) { func loadAllSnapshots(ctx context.Context, repo restic.Repository, excludeIDs restic.IDSet) (snapshots restic.Snapshots, err error) {
err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, excludeIDs, func(id restic.ID, sn *restic.Snapshot, err error) error { err = restic.ForAllSnapshots(ctx, repo, repo, excludeIDs, func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil { if err != nil {
return err return err
} }