From f7ae0cb78f4bbe469daf384351a0026ede5c013a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 1 Sep 2016 16:04:29 +0200 Subject: [PATCH] Fix cmds/restic for new structure --- src/cmds/restic/cmd_backup.go | 10 +-- src/cmds/restic/cmd_cache.go | 52 --------------- src/cmds/restic/cmd_cat.go | 21 +++--- src/cmds/restic/cmd_dump.go | 81 ++++++++++++------------ src/cmds/restic/cmd_find.go | 9 ++- src/cmds/restic/cmd_forget.go | 5 +- src/cmds/restic/cmd_init.go | 2 +- src/cmds/restic/cmd_key.go | 11 ++-- src/cmds/restic/cmd_list.go | 40 ++++++------ src/cmds/restic/cmd_ls.go | 9 ++- src/cmds/restic/cmd_prune.go | 28 ++++---- src/cmds/restic/cmd_restore.go | 3 +- src/cmds/restic/cmd_snapshots.go | 5 +- src/cmds/restic/global.go | 5 +- src/cmds/restic/integration_fuse_test.go | 4 +- src/cmds/restic/integration_test.go | 18 +++--- src/restic/backend/generic.go | 74 ---------------------- src/restic/backend/generic_test.go | 62 ------------------ src/restic/backend_find_test.go | 70 ++++++++++++++++++++ src/restic/snapshot.go | 13 ++++ 20 files changed, 202 insertions(+), 320 deletions(-) delete mode 100644 src/cmds/restic/cmd_cache.go delete mode 100644 src/restic/backend/generic.go delete mode 100644 src/restic/backend/generic_test.go create mode 100644 src/restic/backend_find_test.go diff --git a/src/cmds/restic/cmd_backup.go b/src/cmds/restic/cmd_backup.go index 7a7640249..38f6fcf70 100644 --- a/src/cmds/restic/cmd_backup.go +++ b/src/cmds/restic/cmd_backup.go @@ -6,7 +6,7 @@ import ( "os" "path/filepath" "restic" - "restic/backend" + "restic/archiver" "restic/debug" "restic/filter" "restic/fs" @@ -259,7 +259,7 @@ func (cmd CmdBackup) readFromStdin(args []string) error { return err } - _, id, err := restic.ArchiveReader(repo, cmd.newArchiveStdinProgress(), os.Stdin, cmd.StdinFilename) + _, id, err := archiver.ArchiveReader(repo, cmd.newArchiveStdinProgress(), os.Stdin, cmd.StdinFilename) if err != nil { return err } @@ -306,7 +306,7 @@ func (cmd CmdBackup) Execute(args []string) error { return err } - var parentSnapshotID *backend.ID + var parentSnapshotID *restic.ID // Force using a parent if !cmd.Force && cmd.Parent != "" { @@ -365,12 +365,12 @@ func (cmd CmdBackup) Execute(args []string) error { return !matched } - stat, err := restic.Scan(target, selectFilter, cmd.newScanProgress()) + stat, err := archiver.Scan(target, selectFilter, cmd.newScanProgress()) if err != nil { return err } - arch := restic.NewArchiver(repo) + arch := archiver.New(repo) arch.Excludes = cmd.Excludes arch.SelectFilter = selectFilter diff --git a/src/cmds/restic/cmd_cache.go b/src/cmds/restic/cmd_cache.go deleted file mode 100644 index aa4d5765f..000000000 --- a/src/cmds/restic/cmd_cache.go +++ /dev/null @@ -1,52 +0,0 @@ -package main - -import ( - "fmt" - - "restic" -) - -type CmdCache struct { - global *GlobalOptions -} - -func init() { - _, err := parser.AddCommand("cache", - "manage cache", - "The cache command creates and manages the local cache", - &CmdCache{global: &globalOpts}) - if err != nil { - panic(err) - } -} - -func (cmd CmdCache) Usage() string { - return "[update|clear]" -} - -func (cmd CmdCache) Execute(args []string) error { - repo, err := cmd.global.OpenRepository() - if err != nil { - return err - } - - lock, err := lockRepo(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - - cache, err := restic.NewCache(repo, cmd.global.CacheDir) - if err != nil { - return err - } - - fmt.Printf("clear cache for old snapshots\n") - err = cache.Clear(repo) - if err != nil { - return err - } - fmt.Printf("done\n") - - return nil -} diff --git a/src/cmds/restic/cmd_cat.go b/src/cmds/restic/cmd_cat.go index 4e52848da..75edc258b 100644 --- a/src/cmds/restic/cmd_cat.go +++ b/src/cmds/restic/cmd_cat.go @@ -8,7 +8,6 @@ import ( "restic" "restic/backend" "restic/debug" - "restic/pack" "restic/repository" ) @@ -48,9 +47,9 @@ func (cmd CmdCat) Execute(args []string) error { tpe := args[0] - var id backend.ID + var id restic.ID if tpe != "masterkey" && tpe != "config" { - id, err = backend.ParseID(args[1]) + id, err = restic.ParseID(args[1]) if err != nil { if tpe != "snapshot" { return restic.Fatalf("unable to parse ID: %v\n", err) @@ -67,7 +66,7 @@ func (cmd CmdCat) Execute(args []string) error { // handle all types that don't need an index switch tpe { case "config": - buf, err := json.MarshalIndent(repo.Config, "", " ") + buf, err := json.MarshalIndent(repo.Config(), "", " ") if err != nil { return err } @@ -75,7 +74,7 @@ func (cmd CmdCat) Execute(args []string) error { fmt.Println(string(buf)) return nil case "index": - buf, err := repo.LoadAndDecrypt(backend.Index, id) + buf, err := repo.LoadAndDecrypt(restic.IndexFile, id) if err != nil { return err } @@ -85,7 +84,7 @@ func (cmd CmdCat) Execute(args []string) error { case "snapshot": sn := &restic.Snapshot{} - err = repo.LoadJSONUnpacked(backend.Snapshot, id, sn) + err = repo.LoadJSONUnpacked(restic.SnapshotFile, id, sn) if err != nil { return err } @@ -99,7 +98,7 @@ func (cmd CmdCat) Execute(args []string) error { return nil case "key": - h := backend.Handle{Type: backend.Key, Name: id.String()} + h := restic.Handle{FileType: restic.KeyFile, Name: id.String()} buf, err := backend.LoadAll(repo.Backend(), h, nil) if err != nil { return err @@ -150,13 +149,13 @@ func (cmd CmdCat) Execute(args []string) error { switch tpe { case "pack": - h := backend.Handle{Type: backend.Data, Name: id.String()} + h := restic.Handle{FileType: restic.DataFile, Name: id.String()} buf, err := backend.LoadAll(repo.Backend(), h, nil) if err != nil { return err } - hash := backend.Hash(buf) + hash := restic.Hash(buf) if !hash.Equal(id) { fmt.Fprintf(cmd.global.stderr, "Warning: hash of data does not match ID, want\n %v\ngot:\n %v\n", id.String(), hash.String()) } @@ -165,7 +164,7 @@ func (cmd CmdCat) Execute(args []string) error { return err case "blob": - for _, t := range []pack.BlobType{pack.Data, pack.Tree} { + for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} { list, err := repo.Index().Lookup(id, t) if err != nil { continue @@ -187,7 +186,7 @@ func (cmd CmdCat) Execute(args []string) error { case "tree": debug.Log("cat", "cat tree %v", id.Str()) tree := restic.NewTree() - err = repo.LoadJSONPack(pack.Tree, id, tree) + err = repo.LoadJSONPack(restic.TreeBlob, id, tree) if err != nil { debug.Log("cat", "unable to load tree %v: %v", id.Str(), err) return err diff --git a/src/cmds/restic/cmd_dump.go b/src/cmds/restic/cmd_dump.go index 32b789094..9bab5151a 100644 --- a/src/cmds/restic/cmd_dump.go +++ b/src/cmds/restic/cmd_dump.go @@ -9,7 +9,6 @@ import ( "os" "restic" - "restic/backend" "restic/pack" "restic/repository" @@ -50,7 +49,7 @@ func debugPrintSnapshots(repo *repository.Repository, wr io.Writer) error { done := make(chan struct{}) defer close(done) - for id := range repo.List(backend.Snapshot, done) { + for id := range repo.List(restic.SnapshotFile, done) { snapshot, err := restic.LoadSnapshot(repo, id) if err != nil { fmt.Fprintf(os.Stderr, "LoadSnapshot(%v): %v", id.Str(), err) @@ -68,36 +67,36 @@ func debugPrintSnapshots(repo *repository.Repository, wr io.Writer) error { return nil } -func printTrees(repo *repository.Repository, wr io.Writer) error { - done := make(chan struct{}) - defer close(done) +// func printTrees(repo *repository.Repository, wr io.Writer) error { +// done := make(chan struct{}) +// defer close(done) - trees := []backend.ID{} +// trees := []restic.ID{} - for _, idx := range repo.Index().All() { - for blob := range idx.Each(nil) { - if blob.Type != pack.Tree { - continue - } +// for _, idx := range repo.Index().All() { +// for blob := range idx.Each(nil) { +// if blob.Type != pack.Tree { +// continue +// } - trees = append(trees, blob.ID) - } - } +// trees = append(trees, blob.ID) +// } +// } - for _, id := range trees { - tree, err := restic.LoadTree(repo, id) - if err != nil { - fmt.Fprintf(os.Stderr, "LoadTree(%v): %v", id.Str(), err) - continue - } +// for _, id := range trees { +// tree, err := restic.LoadTree(repo, id) +// if err != nil { +// fmt.Fprintf(os.Stderr, "LoadTree(%v): %v", id.Str(), err) +// continue +// } - fmt.Fprintf(wr, "tree_id: %v\n", id) +// fmt.Fprintf(wr, "tree_id: %v\n", id) - prettyPrintJSON(wr, tree) - } +// prettyPrintJSON(wr, tree) +// } - return nil -} +// return nil +// } const dumpPackWorkers = 10 @@ -110,10 +109,10 @@ type Pack struct { // Blob is the struct used in printPacks. type Blob struct { - Type pack.BlobType `json:"type"` - Length uint `json:"length"` - ID backend.ID `json:"id"` - Offset uint `json:"offset"` + Type restic.BlobType `json:"type"` + Length uint `json:"length"` + ID restic.ID `json:"id"` + Offset uint `json:"offset"` } func printPacks(repo *repository.Repository, wr io.Writer) error { @@ -123,14 +122,14 @@ func printPacks(repo *repository.Repository, wr io.Writer) error { f := func(job worker.Job, done <-chan struct{}) (interface{}, error) { name := job.Data.(string) - h := backend.Handle{Type: backend.Data, Name: name} + h := restic.Handle{FileType: restic.DataFile, Name: name} blobInfo, err := repo.Backend().Stat(h) if err != nil { return nil, err } - blobs, err := pack.List(repo.Key(), backend.ReaderAt(repo.Backend(), h), blobInfo.Size) + blobs, err := pack.List(repo.Key(), restic.ReaderAt(repo.Backend(), h), blobInfo.Size) if err != nil { return nil, err } @@ -143,7 +142,7 @@ func printPacks(repo *repository.Repository, wr io.Writer) error { wp := worker.New(dumpPackWorkers, f, jobCh, resCh) go func() { - for name := range repo.Backend().List(backend.Data, done) { + for name := range repo.Backend().List(restic.DataFile, done) { jobCh <- worker.Job{Data: name} } close(jobCh) @@ -157,7 +156,7 @@ func printPacks(repo *repository.Repository, wr io.Writer) error { continue } - entries := job.Result.([]pack.Blob) + entries := job.Result.([]restic.Blob) p := Pack{ Name: name, Blobs: make([]Blob, len(entries)), @@ -183,7 +182,7 @@ func (cmd CmdDump) DumpIndexes() error { done := make(chan struct{}) defer close(done) - for id := range cmd.repo.List(backend.Index, done) { + for id := range cmd.repo.List(restic.IndexFile, done) { fmt.Printf("index_id: %v\n", id) idx, err := repository.LoadIndex(cmd.repo, id) @@ -229,8 +228,8 @@ func (cmd CmdDump) Execute(args []string) error { return cmd.DumpIndexes() case "snapshots": return debugPrintSnapshots(repo, os.Stdout) - case "trees": - return printTrees(repo, os.Stdout) + // case "trees": + // return printTrees(repo, os.Stdout) case "packs": return printPacks(repo, os.Stdout) case "all": @@ -240,12 +239,12 @@ func (cmd CmdDump) Execute(args []string) error { return err } - fmt.Printf("\ntrees:\n") + // fmt.Printf("\ntrees:\n") - err = printTrees(repo, os.Stdout) - if err != nil { - return err - } + // err = printTrees(repo, os.Stdout) + // if err != nil { + // return err + // } fmt.Printf("\nindexes:\n") err = cmd.DumpIndexes() diff --git a/src/cmds/restic/cmd_find.go b/src/cmds/restic/cmd_find.go index 1c66cd757..783d65b35 100644 --- a/src/cmds/restic/cmd_find.go +++ b/src/cmds/restic/cmd_find.go @@ -5,7 +5,6 @@ import ( "time" "restic" - "restic/backend" "restic/debug" "restic/repository" ) @@ -59,7 +58,7 @@ func parseTime(str string) (time.Time, error) { return time.Time{}, restic.Fatalf("unable to parse time: %q", str) } -func (c CmdFind) findInTree(repo *repository.Repository, id backend.ID, path string) ([]findResult, error) { +func (c CmdFind) findInTree(repo *repository.Repository, id restic.ID, path string) ([]findResult, error) { debug.Log("restic.find", "checking tree %v\n", id) tree, err := restic.LoadTree(repo, id) if err != nil { @@ -92,7 +91,7 @@ func (c CmdFind) findInTree(repo *repository.Repository, id backend.ID, path str debug.Log("restic.find", " pattern does not match\n") } - if node.Type == "dir" { + if node.FileType == "dir" { subdirResults, err := c.findInTree(repo, *node.Subtree, filepath.Join(path, node.Name)) if err != nil { return nil, err @@ -105,7 +104,7 @@ func (c CmdFind) findInTree(repo *repository.Repository, id backend.ID, path str return results, nil } -func (c CmdFind) findInSnapshot(repo *repository.Repository, id backend.ID) error { +func (c CmdFind) findInSnapshot(repo *repository.Repository, id restic.ID) error { debug.Log("restic.find", "searching in snapshot %s\n for entries within [%s %s]", id.Str(), c.oldest, c.newest) sn, err := restic.LoadSnapshot(repo, id) @@ -184,7 +183,7 @@ func (c CmdFind) Execute(args []string) error { done := make(chan struct{}) defer close(done) - for snapshotID := range repo.List(backend.Snapshot, done) { + for snapshotID := range repo.List(restic.SnapshotFile, done) { err := c.findInSnapshot(repo, snapshotID) if err != nil { diff --git a/src/cmds/restic/cmd_forget.go b/src/cmds/restic/cmd_forget.go index 16da4b556..3f4ec12c6 100644 --- a/src/cmds/restic/cmd_forget.go +++ b/src/cmds/restic/cmd_forget.go @@ -4,7 +4,6 @@ import ( "fmt" "io" "restic" - "restic/backend" "strings" ) @@ -93,7 +92,7 @@ func (cmd CmdForget) Execute(args []string) error { } if !cmd.DryRun { - err = repo.Backend().Remove(backend.Snapshot, id.String()) + err = repo.Backend().Remove(restic.SnapshotFile, id.String()) if err != nil { return err } @@ -156,7 +155,7 @@ func (cmd CmdForget) Execute(args []string) error { if !cmd.DryRun { for _, sn := range remove { - err = repo.Backend().Remove(backend.Snapshot, sn.ID().String()) + err = repo.Backend().Remove(restic.SnapshotFile, sn.ID().String()) if err != nil { return err } diff --git a/src/cmds/restic/cmd_init.go b/src/cmds/restic/cmd_init.go index 49b0907ad..3e68ebdea 100644 --- a/src/cmds/restic/cmd_init.go +++ b/src/cmds/restic/cmd_init.go @@ -32,7 +32,7 @@ func (cmd CmdInit) Execute(args []string) error { cmd.global.Exitf(1, "creating key in backend at %s failed: %v\n", cmd.global.Repo, err) } - cmd.global.Verbosef("created restic backend %v at %s\n", s.Config.ID[:10], cmd.global.Repo) + cmd.global.Verbosef("created restic backend %v at %s\n", s.Config().ID[:10], cmd.global.Repo) cmd.global.Verbosef("\n") cmd.global.Verbosef("Please note that knowledge of your password is required to access\n") cmd.global.Verbosef("the repository. Losing your password means that your data is\n") diff --git a/src/cmds/restic/cmd_key.go b/src/cmds/restic/cmd_key.go index 67d5afa64..629f5ddf0 100644 --- a/src/cmds/restic/cmd_key.go +++ b/src/cmds/restic/cmd_key.go @@ -4,7 +4,6 @@ import ( "fmt" "restic" - "restic/backend" "restic/repository" ) @@ -28,7 +27,7 @@ func (cmd CmdKey) listKeys(s *repository.Repository) error { tab.Header = fmt.Sprintf(" %-10s %-10s %-10s %s", "ID", "User", "Host", "Created") tab.RowFormat = "%s%-10s %-10s %-10s %s" - plen, err := s.PrefixLength(backend.Key) + plen, err := s.PrefixLength(restic.KeyFile) if err != nil { return err } @@ -36,7 +35,7 @@ func (cmd CmdKey) listKeys(s *repository.Repository) error { done := make(chan struct{}) defer close(done) - for id := range s.List(backend.Key, done) { + for id := range s.List(restic.KeyFile, done) { k, err := repository.LoadKey(s, id.String()) if err != nil { cmd.global.Warnf("LoadKey() failed: %v\n", err) @@ -82,7 +81,7 @@ func (cmd CmdKey) deleteKey(repo *repository.Repository, name string) error { return restic.Fatal("refusing to remove key currently used to access repository") } - err := repo.Backend().Remove(backend.Key, name) + err := repo.Backend().Remove(restic.KeyFile, name) if err != nil { return err } @@ -97,7 +96,7 @@ func (cmd CmdKey) changePassword(repo *repository.Repository) error { return restic.Fatalf("creating new key failed: %v\n", err) } - err = repo.Backend().Remove(backend.Key, repo.KeyName()) + err = repo.Backend().Remove(restic.KeyFile, repo.KeyName()) if err != nil { return err } @@ -145,7 +144,7 @@ func (cmd CmdKey) Execute(args []string) error { return err } - id, err := backend.Find(repo.Backend(), backend.Key, args[1]) + id, err := restic.Find(repo.Backend(), restic.KeyFile, args[1]) if err != nil { return err } diff --git a/src/cmds/restic/cmd_list.go b/src/cmds/restic/cmd_list.go index 717d65ade..4418dbb08 100644 --- a/src/cmds/restic/cmd_list.go +++ b/src/cmds/restic/cmd_list.go @@ -1,9 +1,6 @@ package main -import ( - "restic" - "restic/backend" -) +import "restic" type CmdList struct { global *GlobalOptions @@ -41,31 +38,32 @@ func (cmd CmdList) Execute(args []string) error { } } - var t backend.Type + var t restic.FileType switch args[0] { - case "blobs": - err = repo.LoadIndex() - if err != nil { - return err - } + // case "blobs": + // restic.Lister + // err = repo.LoadIndex() + // if err != nil { + // return err + // } - for _, idx := range repo.Index().All() { - for blob := range idx.Each(nil) { - cmd.global.Printf("%s\n", blob.ID) - } - } + // for _, idx := range repo.Index().All() { + // for blob := range idx.Each(nil) { + // cmd.global.Printf("%s\n", blob.ID) + // } + // } - return nil + // return nil case "packs": - t = backend.Data + t = restic.DataFile case "index": - t = backend.Index + t = restic.IndexFile case "snapshots": - t = backend.Snapshot + t = restic.SnapshotFile case "keys": - t = backend.Key + t = restic.KeyFile case "locks": - t = backend.Lock + t = restic.LockFile default: return restic.Fatal("invalid type") } diff --git a/src/cmds/restic/cmd_ls.go b/src/cmds/restic/cmd_ls.go index c55670a93..8157c74f2 100644 --- a/src/cmds/restic/cmd_ls.go +++ b/src/cmds/restic/cmd_ls.go @@ -6,7 +6,6 @@ import ( "path/filepath" "restic" - "restic/backend" "restic/repository" ) @@ -31,7 +30,7 @@ func (cmd CmdLs) printNode(prefix string, n *restic.Node) string { return filepath.Join(prefix, n.Name) } - switch n.Type { + switch n.FileType { case "file": return fmt.Sprintf("%s %5d %5d %6d %s %s", n.Mode, n.UID, n.GID, n.Size, n.ModTime, filepath.Join(prefix, n.Name)) @@ -42,11 +41,11 @@ func (cmd CmdLs) printNode(prefix string, n *restic.Node) string { return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s", n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime, filepath.Join(prefix, n.Name), n.LinkTarget) default: - return fmt.Sprintf("", n.Type, n.Name) + return fmt.Sprintf("", n.FileType, n.Name) } } -func (cmd CmdLs) printTree(prefix string, repo *repository.Repository, id backend.ID) error { +func (cmd CmdLs) printTree(prefix string, repo *repository.Repository, id restic.ID) error { tree, err := restic.LoadTree(repo, id) if err != nil { return err @@ -55,7 +54,7 @@ func (cmd CmdLs) printTree(prefix string, repo *repository.Repository, id backen for _, entry := range tree.Nodes { cmd.global.Printf(cmd.printNode(prefix, entry) + "\n") - if entry.Type == "dir" && entry.Subtree != nil { + if entry.FileType == "dir" && entry.Subtree != nil { err = cmd.printTree(filepath.Join(prefix, entry.Name), repo, *entry.Subtree) if err != nil { return err diff --git a/src/cmds/restic/cmd_prune.go b/src/cmds/restic/cmd_prune.go index eee330131..c21cab16a 100644 --- a/src/cmds/restic/cmd_prune.go +++ b/src/cmds/restic/cmd_prune.go @@ -4,10 +4,8 @@ import ( "fmt" "os" "restic" - "restic/backend" "restic/debug" "restic/index" - "restic/pack" "restic/repository" "time" @@ -94,7 +92,7 @@ func (cmd CmdPrune) Execute(args []string) error { } cmd.global.Verbosef("counting files in repo\n") - for _ = range repo.List(backend.Data, done) { + for _ = range repo.List(restic.DataFile, done) { stats.packs++ } @@ -112,7 +110,7 @@ func (cmd CmdPrune) Execute(args []string) error { cmd.global.Verbosef("repository contains %v packs (%v blobs) with %v bytes\n", len(idx.Packs), len(idx.Blobs), formatBytes(uint64(stats.bytes))) - blobCount := make(map[pack.Handle]int) + blobCount := make(map[restic.BlobHandle]int) duplicateBlobs := 0 duplicateBytes := 0 @@ -120,7 +118,7 @@ func (cmd CmdPrune) Execute(args []string) error { for _, p := range idx.Packs { for _, entry := range p.Entries { stats.blobs++ - h := pack.Handle{ID: entry.ID, Type: entry.Type} + h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} blobCount[h]++ if blobCount[h] > 1 { @@ -144,8 +142,8 @@ func (cmd CmdPrune) Execute(args []string) error { cmd.global.Verbosef("find data that is still in use for %d snapshots\n", stats.snapshots) - usedBlobs := pack.NewBlobSet() - seenBlobs := pack.NewBlobSet() + usedBlobs := restic.NewBlobSet() + seenBlobs := restic.NewBlobSet() bar = newProgressMax(cmd.global.ShowProgress(), uint64(len(snapshots)), "snapshots") bar.Start() @@ -165,7 +163,7 @@ func (cmd CmdPrune) Execute(args []string) error { cmd.global.Verbosef("found %d of %d data blobs still in use\n", len(usedBlobs), stats.blobs) // find packs that need a rewrite - rewritePacks := backend.NewIDSet() + rewritePacks := restic.NewIDSet() for h, blob := range idx.Blobs { if !usedBlobs.Has(h) { rewritePacks.Merge(blob.Packs) @@ -178,11 +176,11 @@ func (cmd CmdPrune) Execute(args []string) error { } // find packs that are unneeded - removePacks := backend.NewIDSet() + removePacks := restic.NewIDSet() nextPack: for packID, p := range idx.Packs { for _, blob := range p.Entries { - h := pack.Handle{ID: blob.ID, Type: blob.Type} + h := restic.BlobHandle{ID: blob.ID, Type: blob.Type} if usedBlobs.Has(h) { continue nextPack } @@ -205,7 +203,7 @@ nextPack: } for packID := range removePacks { - err = repo.Backend().Remove(backend.Data, packID.String()) + err = repo.Backend().Remove(restic.DataFile, packID.String()) if err != nil { cmd.global.Warnf("unable to remove file %v from the repository\n", packID.Str()) } @@ -214,7 +212,7 @@ nextPack: cmd.global.Verbosef("creating new index\n") stats.packs = 0 - for _ = range repo.List(backend.Data, done) { + for _ = range repo.List(restic.DataFile, done) { stats.packs++ } bar = newProgressMax(cmd.global.ShowProgress(), uint64(stats.packs), "packs") @@ -223,9 +221,9 @@ nextPack: return err } - var supersedes backend.IDs - for idxID := range repo.List(backend.Index, done) { - err := repo.Backend().Remove(backend.Index, idxID.String()) + var supersedes restic.IDs + for idxID := range repo.List(restic.IndexFile, done) { + err := repo.Backend().Remove(restic.IndexFile, idxID.String()) if err != nil { fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", idxID.Str(), err) } diff --git a/src/cmds/restic/cmd_restore.go b/src/cmds/restic/cmd_restore.go index 9ff57565d..1a55cdb4e 100644 --- a/src/cmds/restic/cmd_restore.go +++ b/src/cmds/restic/cmd_restore.go @@ -2,7 +2,6 @@ package main import ( "restic" - "restic/backend" "restic/debug" "restic/filter" ) @@ -66,7 +65,7 @@ func (cmd CmdRestore) Execute(args []string) error { return err } - var id backend.ID + var id restic.ID if snapshotIDString == "latest" { id, err = restic.FindLatestSnapshot(repo, cmd.Paths, cmd.Host) diff --git a/src/cmds/restic/cmd_snapshots.go b/src/cmds/restic/cmd_snapshots.go index ccf889d23..23f9eb709 100644 --- a/src/cmds/restic/cmd_snapshots.go +++ b/src/cmds/restic/cmd_snapshots.go @@ -9,7 +9,6 @@ import ( "strings" "restic" - "restic/backend" ) type Table struct { @@ -92,7 +91,7 @@ func (cmd CmdSnapshots) Execute(args []string) error { defer close(done) list := []*restic.Snapshot{} - for id := range repo.List(backend.Snapshot, done) { + for id := range repo.List(restic.SnapshotFile, done) { sn, err := restic.LoadSnapshot(repo, id) if err != nil { fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err) @@ -115,7 +114,7 @@ func (cmd CmdSnapshots) Execute(args []string) error { } - plen, err := repo.PrefixLength(backend.Snapshot) + plen, err := repo.PrefixLength(restic.SnapshotFile) if err != nil { return err } diff --git a/src/cmds/restic/global.go b/src/cmds/restic/global.go index 2aedb026e..b7eff3e41 100644 --- a/src/cmds/restic/global.go +++ b/src/cmds/restic/global.go @@ -9,7 +9,6 @@ import ( "strings" "syscall" - "restic/backend" "restic/backend/local" "restic/backend/rest" "restic/backend/s3" @@ -270,7 +269,7 @@ func (o GlobalOptions) OpenRepository() (*repository.Repository, error) { } // Open the backend specified by a location config. -func open(s string) (backend.Backend, error) { +func open(s string) (restic.Backend, error) { debug.Log("open", "parsing location %v", s) loc, err := location.Parse(s) if err != nil { @@ -305,7 +304,7 @@ func open(s string) (backend.Backend, error) { } // Create the backend specified by URI. -func create(s string) (backend.Backend, error) { +func create(s string) (restic.Backend, error) { debug.Log("open", "parsing location %v", s) loc, err := location.Parse(s) if err != nil { diff --git a/src/cmds/restic/integration_fuse_test.go b/src/cmds/restic/integration_fuse_test.go index 25f4b7e84..30c261703 100644 --- a/src/cmds/restic/integration_fuse_test.go +++ b/src/cmds/restic/integration_fuse_test.go @@ -71,7 +71,7 @@ func TestMount(t *testing.T) { t.Skip("Skipping fuse tests") } - checkSnapshots := func(repo *repository.Repository, mountpoint string, snapshotIDs []backend.ID) { + checkSnapshots := func(repo *repository.Repository, mountpoint string, snapshotIDs []restic.ID) { snapshotsDir, err := os.Open(filepath.Join(mountpoint, "snapshots")) OK(t, err) namesInSnapshots, err := snapshotsDir.Readdirnames(-1) @@ -123,7 +123,7 @@ func TestMount(t *testing.T) { Assert(t, len(names) == 1 && names[0] == "snapshots", `The fuse virtual directory "snapshots" doesn't exist`) OK(t, mountpointDir.Close()) - checkSnapshots(repo, mountpoint, []backend.ID{}) + checkSnapshots(repo, mountpoint, []restic.ID{}) datafile := filepath.Join("testdata", "backup-data.tar.gz") fd, err := os.Open(datafile) diff --git a/src/cmds/restic/integration_test.go b/src/cmds/restic/integration_test.go index fff765bef..8bd9b81c5 100644 --- a/src/cmds/restic/integration_test.go +++ b/src/cmds/restic/integration_test.go @@ -25,8 +25,8 @@ import ( . "restic/test" ) -func parseIDsFromReader(t testing.TB, rd io.Reader) backend.IDs { - IDs := backend.IDs{} +func parseIDsFromReader(t testing.TB, rd io.Reader) restic.IDs { + IDs := restic.IDs{} sc := bufio.NewScanner(rd) for sc.Scan() { @@ -51,11 +51,11 @@ func cmdInit(t testing.TB, global GlobalOptions) { t.Logf("repository initialized at %v", global.Repo) } -func cmdBackup(t testing.TB, global GlobalOptions, target []string, parentID *backend.ID) { +func cmdBackup(t testing.TB, global GlobalOptions, target []string, parentID *restic.ID) { cmdBackupExcludes(t, global, target, parentID, nil) } -func cmdBackupExcludes(t testing.TB, global GlobalOptions, target []string, parentID *backend.ID, excludes []string) { +func cmdBackupExcludes(t testing.TB, global GlobalOptions, target []string, parentID *restic.ID, excludes []string) { cmd := &CmdBackup{global: &global, Excludes: excludes} if parentID != nil { cmd.Parent = parentID.String() @@ -66,19 +66,19 @@ func cmdBackupExcludes(t testing.TB, global GlobalOptions, target []string, pare OK(t, cmd.Execute(target)) } -func cmdList(t testing.TB, global GlobalOptions, tpe string) backend.IDs { +func cmdList(t testing.TB, global GlobalOptions, tpe string) restic.IDs { cmd := &CmdList{global: &global} return executeAndParseIDs(t, cmd, tpe) } -func executeAndParseIDs(t testing.TB, cmd *CmdList, args ...string) backend.IDs { +func executeAndParseIDs(t testing.TB, cmd *CmdList, args ...string) restic.IDs { buf := bytes.NewBuffer(nil) cmd.global.stdout = buf OK(t, cmd.Execute(args)) return parseIDsFromReader(t, buf) } -func cmdRestore(t testing.TB, global GlobalOptions, dir string, snapshotID backend.ID) { +func cmdRestore(t testing.TB, global GlobalOptions, dir string, snapshotID restic.ID) { cmdRestoreExcludes(t, global, dir, snapshotID, nil) } @@ -87,12 +87,12 @@ func cmdRestoreLatest(t testing.TB, global GlobalOptions, dir string, paths []st OK(t, cmd.Execute([]string{"latest"})) } -func cmdRestoreExcludes(t testing.TB, global GlobalOptions, dir string, snapshotID backend.ID, excludes []string) { +func cmdRestoreExcludes(t testing.TB, global GlobalOptions, dir string, snapshotID restic.ID, excludes []string) { cmd := &CmdRestore{global: &global, Target: dir, Exclude: excludes} OK(t, cmd.Execute([]string{snapshotID.String()})) } -func cmdRestoreIncludes(t testing.TB, global GlobalOptions, dir string, snapshotID backend.ID, includes []string) { +func cmdRestoreIncludes(t testing.TB, global GlobalOptions, dir string, snapshotID restic.ID, includes []string) { cmd := &CmdRestore{global: &global, Target: dir, Include: includes} OK(t, cmd.Execute([]string{snapshotID.String()})) } diff --git a/src/restic/backend/generic.go b/src/restic/backend/generic.go deleted file mode 100644 index a267922f3..000000000 --- a/src/restic/backend/generic.go +++ /dev/null @@ -1,74 +0,0 @@ -package backend - -import ( - "restic" - - "github.com/pkg/errors" -) - -// ErrNoIDPrefixFound is returned by Find() when no ID for the given prefix -// could be found. -var ErrNoIDPrefixFound = errors.New("no ID found") - -// ErrMultipleIDMatches is returned by Find() when multiple IDs with the given -// prefix are found. -var ErrMultipleIDMatches = errors.New("multiple IDs with prefix found") - -// Find loads the list of all blobs of type t and searches for names which -// start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned. -// If more than one is found, nil and ErrMultipleIDMatches is returned. -func Find(be restic.Lister, t restic.FileType, prefix string) (string, error) { - done := make(chan struct{}) - defer close(done) - - match := "" - - // TODO: optimize by sorting list etc. - for name := range be.List(t, done) { - if prefix == name[:len(prefix)] { - if match == "" { - match = name - } else { - return "", ErrMultipleIDMatches - } - } - } - - if match != "" { - return match, nil - } - - return "", ErrNoIDPrefixFound -} - -const minPrefixLength = 8 - -// PrefixLength returns the number of bytes required so that all prefixes of -// all names of type t are unique. -func PrefixLength(be restic.Lister, t restic.FileType) (int, error) { - done := make(chan struct{}) - defer close(done) - - // load all IDs of the given type - list := make([]string, 0, 100) - for name := range be.List(t, done) { - list = append(list, name) - } - - // select prefixes of length l, test if the last one is the same as the current one -outer: - for l := minPrefixLength; l < restic.IDSize; l++ { - var last string - - for _, name := range list { - if last == name[:l] { - continue outer - } - last = name[:l] - } - - return l, nil - } - - return restic.IDSize, nil -} diff --git a/src/restic/backend/generic_test.go b/src/restic/backend/generic_test.go deleted file mode 100644 index 64b82a769..000000000 --- a/src/restic/backend/generic_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package backend_test - -import ( - "restic" - "testing" - - "restic/backend" - . "restic/test" -) - -type mockBackend struct { - list func(restic.FileType, <-chan struct{}) <-chan string -} - -func (m mockBackend) List(t restic.FileType, done <-chan struct{}) <-chan string { - return m.list(t, done) -} - -var samples = restic.IDs{ - ParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"), - ParseID("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"), - ParseID("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"), - ParseID("20ff988befa5fc40350f00d531a767606efefe242c837aaccb80673f286be53d"), - ParseID("326cb59dfe802304f96ee9b5b9af93bdee73a30f53981e5ec579aedb6f1d0f07"), - ParseID("86b60b9594d1d429c4aa98fa9562082cabf53b98c7dc083abe5dae31074dd15a"), - ParseID("96c8dbe225079e624b5ce509f5bd817d1453cd0a85d30d536d01b64a8669aeae"), - ParseID("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"), -} - -func TestPrefixLength(t *testing.T) { - list := samples - - m := mockBackend{} - m.list = func(t restic.FileType, done <-chan struct{}) <-chan string { - ch := make(chan string) - go func() { - defer close(ch) - for _, id := range list { - select { - case ch <- id.String(): - case <-done: - return - } - } - }() - return ch - } - - l, err := backend.PrefixLength(m, restic.SnapshotFile) - OK(t, err) - Equals(t, 19, l) - - list = samples[:3] - l, err = backend.PrefixLength(m, restic.SnapshotFile) - OK(t, err) - Equals(t, 19, l) - - list = samples[3:] - l, err = backend.PrefixLength(m, restic.SnapshotFile) - OK(t, err) - Equals(t, 8, l) -} diff --git a/src/restic/backend_find_test.go b/src/restic/backend_find_test.go new file mode 100644 index 000000000..cc86cd810 --- /dev/null +++ b/src/restic/backend_find_test.go @@ -0,0 +1,70 @@ +package restic + +import ( + "testing" +) + +type mockBackend struct { + list func(FileType, <-chan struct{}) <-chan string +} + +func (m mockBackend) List(t FileType, done <-chan struct{}) <-chan string { + return m.list(t, done) +} + +var samples = IDs{ + TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"), + TestParseID("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"), + TestParseID("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"), + TestParseID("20ff988befa5fc40350f00d531a767606efefe242c837aaccb80673f286be53d"), + TestParseID("326cb59dfe802304f96ee9b5b9af93bdee73a30f53981e5ec579aedb6f1d0f07"), + TestParseID("86b60b9594d1d429c4aa98fa9562082cabf53b98c7dc083abe5dae31074dd15a"), + TestParseID("96c8dbe225079e624b5ce509f5bd817d1453cd0a85d30d536d01b64a8669aeae"), + TestParseID("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"), +} + +func TestPrefixLength(t *testing.T) { + list := samples + + m := mockBackend{} + m.list = func(t FileType, done <-chan struct{}) <-chan string { + ch := make(chan string) + go func() { + defer close(ch) + for _, id := range list { + select { + case ch <- id.String(): + case <-done: + return + } + } + }() + return ch + } + + l, err := PrefixLength(m, SnapshotFile) + if err != nil { + t.Error(err) + } + if l != 19 { + t.Errorf("wrong prefix length returned, want %d, got %d", 19, l) + } + + list = samples[:3] + l, err = PrefixLength(m, SnapshotFile) + if err != nil { + t.Error(err) + } + if l != 19 { + t.Errorf("wrong prefix length returned, want %d, got %d", 19, l) + } + + list = samples[3:] + l, err = PrefixLength(m, SnapshotFile) + if err != nil { + t.Error(err) + } + if l != 8 { + t.Errorf("wrong prefix length returned, want %d, got %d", 8, l) + } +} diff --git a/src/restic/snapshot.go b/src/restic/snapshot.go index a81dfb82c..4775cbd7b 100644 --- a/src/restic/snapshot.go +++ b/src/restic/snapshot.go @@ -153,3 +153,16 @@ func FindLatestSnapshot(repo Repository, targets []string, source string) (ID, e return latestID, nil } + +// FindSnapshot takes a string and tries to find a snapshot whose ID matches +// the string as closely as possible. +func FindSnapshot(repo Repository, s string) (ID, error) { + + // find snapshot id with prefix + name, err := Find(repo.Backend(), SnapshotFile, s) + if err != nil { + return ID{}, err + } + + return ParseID(name) +}