diff --git a/archiver_test.go b/archiver_test.go index 9aa9bffb4..c639be6ab 100644 --- a/archiver_test.go +++ b/archiver_test.go @@ -3,7 +3,6 @@ package restic_test import ( "bytes" "crypto/sha256" - "flag" "io" "testing" @@ -15,7 +14,6 @@ import ( . "github.com/restic/restic/test" ) -var benchArchiveDirectory = flag.String("test.benchdir", ".", "benchmark archiving a real directory (default: .)") var testPol = chunker.Pol(0x3DA3358B4DC173) type Rdr interface { @@ -48,12 +46,12 @@ func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *crypto.K } func BenchmarkChunkEncrypt(b *testing.B) { + repo := SetupRepo() + defer TeardownRepo(repo) + data := Random(23, 10<<20) // 10MiB rd := bytes.NewReader(data) - s := SetupRepo(b) - defer TeardownRepo(b, s) - buf := make([]byte, chunker.MaxSize) buf2 := make([]byte, chunker.MaxSize) @@ -61,7 +59,7 @@ func BenchmarkChunkEncrypt(b *testing.B) { b.SetBytes(int64(len(data))) for i := 0; i < b.N; i++ { - benchmarkChunkEncrypt(b, buf, buf2, rd, s.Key()) + benchmarkChunkEncrypt(b, buf, buf2, rd, repo.Key()) } } @@ -82,8 +80,8 @@ func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *crypto.Key) } func BenchmarkChunkEncryptParallel(b *testing.B) { - s := SetupRepo(b) - defer TeardownRepo(b, s) + repo := SetupRepo() + defer TeardownRepo(repo) data := Random(23, 10<<20) // 10MiB @@ -95,25 +93,25 @@ func BenchmarkChunkEncryptParallel(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { rd := bytes.NewReader(data) - benchmarkChunkEncryptP(pb, buf, rd, s.Key()) + benchmarkChunkEncryptP(pb, buf, rd, repo.Key()) } }) } func archiveDirectory(b testing.TB) { - repo := SetupRepo(b) - defer TeardownRepo(b, repo) + repo := SetupRepo() + defer TeardownRepo(repo) arch := restic.NewArchiver(repo) - _, id, err := arch.Snapshot(nil, []string{*benchArchiveDirectory}, nil) + _, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil) OK(b, err) b.Logf("snapshot archived as %v", id) } func TestArchiveDirectory(t *testing.T) { - if *benchArchiveDirectory == "" { + if BenchArchiveDirectory == "" { t.Skip("benchdir not set, skipping TestArchiveDirectory") } @@ -121,7 +119,7 @@ func TestArchiveDirectory(t *testing.T) { } func BenchmarkArchiveDirectory(b *testing.B) { - if *benchArchiveDirectory == "" { + if BenchArchiveDirectory == "" { b.Skip("benchdir not set, skipping BenchmarkArchiveDirectory") } @@ -131,13 +129,13 @@ func BenchmarkArchiveDirectory(b *testing.B) { } func archiveWithDedup(t testing.TB) { - if *benchArchiveDirectory == "" { + repo := SetupRepo() + defer TeardownRepo(repo) + + if BenchArchiveDirectory == "" { t.Skip("benchdir not set, skipping TestArchiverDedup") } - repo := SetupRepo(t) - defer TeardownRepo(t, repo) - var cnt struct { before, after, after2 struct { packs, dataBlobs, treeBlobs uint @@ -145,7 +143,7 @@ func archiveWithDedup(t testing.TB) { } // archive a few files - sn := SnapshotDir(t, repo, *benchArchiveDirectory, nil) + sn := SnapshotDir(t, repo, BenchArchiveDirectory, nil) t.Logf("archived snapshot %v", sn.ID().Str()) // get archive stats @@ -156,7 +154,7 @@ func archiveWithDedup(t testing.TB) { cnt.before.packs, cnt.before.dataBlobs, cnt.before.treeBlobs) // archive the same files again, without parent snapshot - sn2 := SnapshotDir(t, repo, *benchArchiveDirectory, nil) + sn2 := SnapshotDir(t, repo, BenchArchiveDirectory, nil) t.Logf("archived snapshot %v", sn2.ID().Str()) // get archive stats again @@ -173,7 +171,7 @@ func archiveWithDedup(t testing.TB) { } // archive the same files again, with a parent snapshot - sn3 := SnapshotDir(t, repo, *benchArchiveDirectory, sn2.ID()) + sn3 := SnapshotDir(t, repo, BenchArchiveDirectory, sn2.ID()) t.Logf("archived snapshot %v, parent %v", sn3.ID().Str(), sn2.ID().Str()) // get archive stats again @@ -195,23 +193,23 @@ func TestArchiveDedup(t *testing.T) { } func BenchmarkLoadTree(t *testing.B) { - if *benchArchiveDirectory == "" { + repo := SetupRepo() + defer TeardownRepo(repo) + + if BenchArchiveDirectory == "" { t.Skip("benchdir not set, skipping TestArchiverDedup") } - s := SetupRepo(t) - defer TeardownRepo(t, s) - // archive a few files - arch := restic.NewArchiver(s) - sn, _, err := arch.Snapshot(nil, []string{*benchArchiveDirectory}, nil) + arch := restic.NewArchiver(repo) + sn, _, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil) OK(t, err) t.Logf("archived snapshot %v", sn.ID()) list := make([]backend.ID, 0, 10) done := make(chan struct{}) - for blob := range s.Index().Each(done) { + for blob := range repo.Index().Each(done) { if blob.Type != pack.Tree { continue } @@ -228,7 +226,7 @@ func BenchmarkLoadTree(t *testing.B) { for i := 0; i < t.N; i++ { for _, id := range list { - _, err := restic.LoadTree(s, id) + _, err := restic.LoadTree(repo, id) OK(t, err) } } diff --git a/cache_test.go b/cache_test.go index 6f342dc39..b480b5869 100644 --- a/cache_test.go +++ b/cache_test.go @@ -8,8 +8,8 @@ import ( ) func TestCache(t *testing.T) { - repo := SetupRepo(t) - defer TeardownRepo(t, repo) + repo := SetupRepo() + defer TeardownRepo(repo) _, err := restic.NewCache(repo, "") OK(t, err) @@ -17,7 +17,7 @@ func TestCache(t *testing.T) { arch := restic.NewArchiver(repo) // archive some files, this should automatically cache all blobs from the snapshot - _, _, err = arch.Snapshot(nil, []string{*benchArchiveDirectory}, nil) + _, _, err = arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil) // TODO: test caching index } diff --git a/chunker/chunker_test.go b/chunker/chunker_test.go index 5c38d2b0d..6a7f30a7d 100644 --- a/chunker/chunker_test.go +++ b/chunker/chunker_test.go @@ -5,12 +5,10 @@ import ( "crypto/md5" "crypto/sha256" "encoding/hex" - "flag" "hash" "io" "io/ioutil" "math/rand" - "os" "testing" "time" @@ -18,8 +16,6 @@ import ( . "github.com/restic/restic/test" ) -var benchmarkFile = flag.String("bench.file", "", "read from this file for benchmark") - func parseDigest(s string) []byte { d, err := hex.DecodeString(s) if err != nil { @@ -247,29 +243,8 @@ func TestChunkerWithoutHash(t *testing.T) { } func benchmarkChunker(b *testing.B, hash hash.Hash) { - var ( - rd io.ReadSeeker - size int - ) - - if *benchmarkFile != "" { - b.Logf("using file %q for benchmark", *benchmarkFile) - f, err := os.Open(*benchmarkFile) - if err != nil { - b.Fatalf("open(%q): %v", *benchmarkFile, err) - } - - fi, err := f.Stat() - if err != nil { - b.Fatalf("lstat(%q): %v", *benchmarkFile, err) - } - - size = int(fi.Size()) - rd = f - } else { - size = 10 * 1024 * 1024 - rd = bytes.NewReader(getRandom(23, size)) - } + size := 10 * 1024 * 1024 + rd := bytes.NewReader(getRandom(23, size)) b.ResetTimer() b.SetBytes(int64(size)) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 2ab9dc5e0..32d486329 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -215,12 +215,18 @@ func (cmd CmdBackup) Execute(args []string) error { target = append(target, d) } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() if err != nil { return err } - err = s.LoadIndex() + lock, err := lockRepo(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + + err = repo.LoadIndex() if err != nil { return err } @@ -229,7 +235,7 @@ func (cmd CmdBackup) Execute(args []string) error { // Force using a parent if !cmd.Force && cmd.Parent != "" { - parentSnapshotID, err = restic.FindSnapshot(s, cmd.Parent) + parentSnapshotID, err = restic.FindSnapshot(repo, cmd.Parent) if err != nil { return fmt.Errorf("invalid id %q: %v", cmd.Parent, err) } @@ -239,7 +245,7 @@ func (cmd CmdBackup) Execute(args []string) error { // Find last snapshot to set it as parent, if not already set if !cmd.Force && parentSnapshotID == nil { - parentSnapshotID, err = findLatestSnapshot(s, target) + parentSnapshotID, err = findLatestSnapshot(repo, target) if err != nil { return err } @@ -258,7 +264,7 @@ func (cmd CmdBackup) Execute(args []string) error { // return true // } - arch := restic.NewArchiver(s) + arch := restic.NewArchiver(repo) arch.Error = func(dir string, fi os.FileInfo, err error) error { // TODO: make ignoring errors configurable diff --git a/cmd/restic/cmd_cache.go b/cmd/restic/cmd_cache.go index 26a6b7970..39733b997 100644 --- a/cmd/restic/cmd_cache.go +++ b/cmd/restic/cmd_cache.go @@ -29,18 +29,24 @@ func (cmd CmdCache) Execute(args []string) error { // return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage()) // } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() if err != nil { return err } - cache, err := restic.NewCache(s, cmd.global.CacheDir) + lock, err := lockRepo(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + + cache, err := restic.NewCache(repo, cmd.global.CacheDir) if err != nil { return err } fmt.Printf("clear cache for old snapshots\n") - err = cache.Clear(s) + err = cache.Clear(repo) if err != nil { return err } diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 65345cfe7..6f22670ea 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -37,7 +37,13 @@ func (cmd CmdCat) Execute(args []string) error { return fmt.Errorf("type or ID not specified, Usage: %s", cmd.Usage()) } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() + if err != nil { + return err + } + + lock, err := lockRepo(repo) + defer unlockRepo(lock) if err != nil { return err } @@ -55,7 +61,7 @@ func (cmd CmdCat) Execute(args []string) error { } // find snapshot id with prefix - id, err = restic.FindSnapshot(s, args[1]) + id, err = restic.FindSnapshot(repo, args[1]) if err != nil { return err } @@ -65,7 +71,7 @@ func (cmd CmdCat) Execute(args []string) error { // handle all types that don't need an index switch tpe { case "config": - buf, err := json.MarshalIndent(s.Config, "", " ") + buf, err := json.MarshalIndent(repo.Config, "", " ") if err != nil { return err } @@ -73,7 +79,7 @@ func (cmd CmdCat) Execute(args []string) error { fmt.Println(string(buf)) return nil case "index": - buf, err := s.Load(backend.Index, id) + buf, err := repo.Load(backend.Index, id) if err != nil { return err } @@ -83,7 +89,7 @@ func (cmd CmdCat) Execute(args []string) error { case "snapshot": sn := &restic.Snapshot{} - err = s.LoadJSONUnpacked(backend.Snapshot, id, sn) + err = repo.LoadJSONUnpacked(backend.Snapshot, id, sn) if err != nil { return err } @@ -97,7 +103,7 @@ func (cmd CmdCat) Execute(args []string) error { return nil case "key": - rd, err := s.Backend().Get(backend.Key, id.String()) + rd, err := repo.Backend().Get(backend.Key, id.String()) if err != nil { return err } @@ -118,7 +124,7 @@ func (cmd CmdCat) Execute(args []string) error { fmt.Println(string(buf)) return nil case "masterkey": - buf, err := json.MarshalIndent(s.Key(), "", " ") + buf, err := json.MarshalIndent(repo.Key(), "", " ") if err != nil { return err } @@ -126,18 +132,30 @@ func (cmd CmdCat) Execute(args []string) error { fmt.Println(string(buf)) return nil case "lock": - return errors.New("not yet implemented") + lock, err := restic.LoadLock(repo, id) + if err != nil { + return err + } + + buf, err := json.MarshalIndent(&lock, "", " ") + if err != nil { + return err + } + + fmt.Println(string(buf)) + + return nil } // load index, handle all the other types - err = s.LoadIndex() + err = repo.LoadIndex() if err != nil { return err } switch tpe { case "pack": - rd, err := s.Backend().Get(backend.Data, id.String()) + rd, err := repo.Backend().Get(backend.Data, id.String()) if err != nil { return err } @@ -146,7 +164,7 @@ func (cmd CmdCat) Execute(args []string) error { return err case "blob": - data, err := s.LoadBlob(pack.Data, id) + data, err := repo.LoadBlob(pack.Data, id) if err == nil { _, err = os.Stdout.Write(data) return err @@ -158,7 +176,7 @@ func (cmd CmdCat) Execute(args []string) error { case "tree": debug.Log("cat", "cat tree %v", id.Str()) tree := restic.NewTree() - err = s.LoadJSONPack(pack.Tree, id, tree) + err = repo.LoadJSONPack(pack.Tree, id, tree) if err != nil { debug.Log("cat", "unable to load tree %v: %v", id.Str(), err) return err diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index b80e61509..9a12ea308 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -109,6 +109,12 @@ func (cmd CmdDump) Execute(args []string) error { return err } + lock, err := lockRepo(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + err = repo.LoadIndex() if err != nil { return err diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index c1a56ebc6..33b91fe43 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -157,7 +157,13 @@ func (c CmdFind) Execute(args []string) error { } } - s, err := c.global.OpenRepository() + repo, err := c.global.OpenRepository() + if err != nil { + return err + } + + lock, err := lockRepo(repo) + defer unlockRepo(lock) if err != nil { return err } @@ -165,18 +171,18 @@ func (c CmdFind) Execute(args []string) error { c.pattern = args[0] if c.Snapshot != "" { - snapshotID, err := restic.FindSnapshot(s, c.Snapshot) + snapshotID, err := restic.FindSnapshot(repo, c.Snapshot) if err != nil { return fmt.Errorf("invalid id %q: %v", args[1], err) } - return c.findInSnapshot(s, snapshotID) + return c.findInSnapshot(repo, snapshotID) } done := make(chan struct{}) defer close(done) - for snapshotID := range s.List(backend.Snapshot, done) { - err := c.findInSnapshot(s, snapshotID) + for snapshotID := range repo.List(backend.Snapshot, done) { + err := c.findInSnapshot(repo, snapshotID) if err != nil { return err diff --git a/cmd/restic/cmd_fsck.go b/cmd/restic/cmd_fsck.go index 17a0608ce..b4e8d3e0d 100644 --- a/cmd/restic/cmd_fsck.go +++ b/cmd/restic/cmd_fsck.go @@ -190,23 +190,29 @@ func (cmd CmdFsck) Execute(args []string) error { cmd.Orphaned = true } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() if err != nil { return err } - err = s.LoadIndex() + lock, err := lockRepoExclusive(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + + err = repo.LoadIndex() if err != nil { return err } if cmd.Snapshot != "" { - id, err := restic.FindSnapshot(s, cmd.Snapshot) + id, err := restic.FindSnapshot(repo, cmd.Snapshot) if err != nil { return fmt.Errorf("invalid id %q: %v", cmd.Snapshot, err) } - err = fsckSnapshot(cmd, s, id) + err = fsckSnapshot(cmd, repo, id) if err != nil { fmt.Fprintf(os.Stderr, "check for snapshot %v failed\n", id) } @@ -223,8 +229,8 @@ func (cmd CmdFsck) Execute(args []string) error { defer close(done) var firstErr error - for id := range s.List(backend.Snapshot, done) { - err = fsckSnapshot(cmd, s, id) + for id := range repo.List(backend.Snapshot, done) { + err = fsckSnapshot(cmd, repo, id) if err != nil { fmt.Fprintf(os.Stderr, "check for snapshot %v failed\n", id) firstErr = err @@ -241,7 +247,7 @@ func (cmd CmdFsck) Execute(args []string) error { cnt[pack.Data] = cmd.o_data cnt[pack.Tree] = cmd.o_trees - for blob := range s.Index().Each(done) { + for blob := range repo.Index().Each(done) { debug.Log("restic.fsck", "checking %v blob %v\n", blob.Type, blob.ID) err = cnt[blob.Type].Find(blob.ID) diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index ad1a358fd..c585ae4cd 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -116,25 +116,49 @@ func (cmd CmdKey) Execute(args []string) error { return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage()) } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() if err != nil { return err } switch args[0] { case "list": - return cmd.listKeys(s) - case "add": - return cmd.addKey(s) - case "rm": - id, err := backend.Find(s.Backend(), backend.Key, args[1]) + lock, err := lockRepo(repo) + defer unlockRepo(lock) if err != nil { return err } - return cmd.deleteKey(s, id) + return cmd.listKeys(repo) + case "add": + lock, err := lockRepo(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + + return cmd.addKey(repo) + case "rm": + lock, err := lockRepoExclusive(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + + id, err := backend.Find(repo.Backend(), backend.Key, args[1]) + if err != nil { + return err + } + + return cmd.deleteKey(repo, id) case "passwd": - return cmd.changePassword(s) + lock, err := lockRepoExclusive(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + + return cmd.changePassword(repo) } return nil diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index bd01e6eda..c8ea6c65c 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -30,7 +30,13 @@ func (cmd CmdList) Execute(args []string) error { return fmt.Errorf("type not specified, Usage: %s", cmd.Usage()) } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() + if err != nil { + return err + } + + lock, err := lockRepo(repo) + defer unlockRepo(lock) if err != nil { return err } @@ -38,12 +44,12 @@ func (cmd CmdList) Execute(args []string) error { var t backend.Type switch args[0] { case "blobs": - err = s.LoadIndex() + err = repo.LoadIndex() if err != nil { return err } - for blob := range s.Index().Each(nil) { + for blob := range repo.Index().Each(nil) { cmd.global.Printf("%s\n", blob.ID) } @@ -62,7 +68,7 @@ func (cmd CmdList) Execute(args []string) error { return errors.New("invalid type") } - for id := range s.List(t, nil) { + for id := range repo.List(t, nil) { cmd.global.Printf("%s\n", id) } diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 59144befc..8ec904bbd 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -69,27 +69,27 @@ func (cmd CmdLs) Execute(args []string) error { return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage()) } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() if err != nil { return err } - err = s.LoadIndex() + err = repo.LoadIndex() if err != nil { return err } - id, err := restic.FindSnapshot(s, args[0]) + id, err := restic.FindSnapshot(repo, args[0]) if err != nil { return err } - sn, err := restic.LoadSnapshot(s, id) + sn, err := restic.LoadSnapshot(repo, id) if err != nil { return err } fmt.Printf("snapshot of %v at %s:\n", sn.Paths, sn.Time) - return printTree("", s, sn.Tree) + return printTree("", repo, sn.Tree) } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 1f3697d4d..393d0347d 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -30,17 +30,23 @@ func (cmd CmdRestore) Execute(args []string) error { return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage()) } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() if err != nil { return err } - err = s.LoadIndex() + lock, err := lockRepo(repo) + defer unlockRepo(lock) if err != nil { return err } - id, err := restic.FindSnapshot(s, args[0]) + err = repo.LoadIndex() + if err != nil { + return err + } + + id, err := restic.FindSnapshot(repo, args[0]) if err != nil { cmd.global.Exitf(1, "invalid id %q: %v", args[0], err) } @@ -48,7 +54,7 @@ func (cmd CmdRestore) Execute(args []string) error { target := args[1] // create restorer - res, err := restic.NewRestorer(s, id) + res, err := restic.NewRestorer(repo, id) if err != nil { cmd.global.Exitf(2, "creating restorer failed: %v\n", err) } diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index f1f8ba31e..38e2046ad 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -94,7 +94,13 @@ func (cmd CmdSnapshots) Execute(args []string) error { return fmt.Errorf("wrong number of arguments, usage: %s", cmd.Usage()) } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() + if err != nil { + return err + } + + lock, err := lockRepo(repo) + defer unlockRepo(lock) if err != nil { return err } @@ -107,8 +113,8 @@ func (cmd CmdSnapshots) Execute(args []string) error { defer close(done) list := []*restic.Snapshot{} - for id := range s.List(backend.Snapshot, done) { - sn, err := restic.LoadSnapshot(s, id) + for id := range repo.List(backend.Snapshot, done) { + sn, err := restic.LoadSnapshot(repo, id) if err != nil { fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err) continue @@ -127,7 +133,7 @@ func (cmd CmdSnapshots) Execute(args []string) error { } } - plen, err := s.PrefixLength(backend.Snapshot) + plen, err := repo.PrefixLength(backend.Snapshot) if err != nil { return err } diff --git a/cmd/restic/cmd_unlock.go b/cmd/restic/cmd_unlock.go new file mode 100644 index 000000000..47345350c --- /dev/null +++ b/cmd/restic/cmd_unlock.go @@ -0,0 +1,43 @@ +package main + +import "github.com/restic/restic" + +type CmdUnlock struct { + RemoveAll bool `long:"remove-all" description:"Remove all locks, even stale ones"` + + global *GlobalOptions +} + +func init() { + _, err := parser.AddCommand("unlock", + "remove locks", + "The unlock command checks for stale locks and removes them", + &CmdUnlock{global: &globalOpts}) + if err != nil { + panic(err) + } +} + +func (cmd CmdUnlock) Usage() string { + return "[unlock-options]" +} + +func (cmd CmdUnlock) Execute(args []string) error { + repo, err := cmd.global.OpenRepository() + if err != nil { + return err + } + + fn := restic.RemoveStaleLocks + if cmd.RemoveAll { + fn = restic.RemoveAllLocks + } + + err = fn(repo) + if err != nil { + return err + } + + cmd.global.Verbosef("successfully removed locks\n") + return nil +} diff --git a/cmd/restic/lock.go b/cmd/restic/lock.go new file mode 100644 index 000000000..5f6c18802 --- /dev/null +++ b/cmd/restic/lock.go @@ -0,0 +1,93 @@ +package main + +import ( + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/restic/restic" + "github.com/restic/restic/debug" + "github.com/restic/restic/repository" +) + +var globalLocks []*restic.Lock + +func lockRepo(repo *repository.Repository) (*restic.Lock, error) { + return lockRepository(repo, false) +} + +func lockRepoExclusive(repo *repository.Repository) (*restic.Lock, error) { + return lockRepository(repo, true) +} + +func lockRepository(repo *repository.Repository, exclusive bool) (*restic.Lock, error) { + lockFn := restic.NewLock + if exclusive { + lockFn = restic.NewExclusiveLock + } + + lock, err := lockFn(repo) + if err != nil { + if restic.IsAlreadyLocked(err) { + tpe := "" + if exclusive { + tpe = " exclusive" + } + fmt.Fprintf(os.Stderr, "unable to acquire%s lock for operation:\n", tpe) + fmt.Fprintln(os.Stderr, err) + fmt.Fprintf(os.Stderr, "\nthe `unlock` command can be used to remove stale locks\n") + os.Exit(1) + } + + return nil, err + } + + globalLocks = append(globalLocks, lock) + + return lock, err +} + +func unlockRepo(lock *restic.Lock) error { + if err := lock.Unlock(); err != nil { + return err + } + + for i := 0; i < len(globalLocks); i++ { + if lock == globalLocks[i] { + globalLocks = append(globalLocks[:i], globalLocks[i+1:]...) + return nil + } + } + + return nil +} + +func unlockAll() error { + debug.Log("unlockAll", "unlocking %d locks", len(globalLocks)) + for _, lock := range globalLocks { + if err := lock.Unlock(); err != nil { + return err + } + } + + return nil +} + +func init() { + c := make(chan os.Signal) + signal.Notify(c, syscall.SIGINT) + + go CleanupHandler(c) +} + +// CleanupHandler handles the SIGINT signal. +func CleanupHandler(c <-chan os.Signal) { + for s := range c { + debug.Log("CleanupHandler", "signal %v received, cleaning up", s) + fmt.Println("\x1b[2KInterrupt received, cleaning up") + unlockAll() + fmt.Println("exiting") + os.Exit(0) + } +} diff --git a/crypto/crypto_test.go b/crypto/crypto_test.go index faba9970c..e8911c086 100644 --- a/crypto/crypto_test.go +++ b/crypto/crypto_test.go @@ -2,7 +2,6 @@ package crypto_test import ( "bytes" - "flag" "io" "io/ioutil" "os" @@ -13,13 +12,13 @@ import ( . "github.com/restic/restic/test" ) -var testLargeCrypto = flag.Bool("test.largecrypto", false, "also test crypto functions with large payloads") +const testLargeCrypto = false func TestEncryptDecrypt(t *testing.T) { k := crypto.NewRandomKey() tests := []int{5, 23, 2<<18 + 23, 1 << 20} - if *testLargeCrypto { + if testLargeCrypto { tests = append(tests, 7<<20+123) } @@ -117,7 +116,7 @@ func TestCornerCases(t *testing.T) { } func TestLargeEncrypt(t *testing.T) { - if !*testLargeCrypto { + if !testLargeCrypto { t.SkipNow() } @@ -252,7 +251,7 @@ func TestEncryptStreamWriter(t *testing.T) { k := crypto.NewRandomKey() tests := []int{5, 23, 2<<18 + 23, 1 << 20} - if *testLargeCrypto { + if testLargeCrypto { tests = append(tests, 7<<20+123) } @@ -286,7 +285,7 @@ func TestDecryptStreamReader(t *testing.T) { k := crypto.NewRandomKey() tests := []int{5, 23, 2<<18 + 23, 1 << 20} - if *testLargeCrypto { + if testLargeCrypto { tests = append(tests, 7<<20+123) } @@ -320,7 +319,7 @@ func TestEncryptWriter(t *testing.T) { k := crypto.NewRandomKey() tests := []int{5, 23, 2<<18 + 23, 1 << 20} - if *testLargeCrypto { + if testLargeCrypto { tests = append(tests, 7<<20+123) } diff --git a/doc/Design.md b/doc/Design.md index 483ce3621..e1ce4f275 100644 --- a/doc/Design.md +++ b/doc/Design.md @@ -375,6 +375,47 @@ As can be seen from the output of the program `sha256sum`, the hash matches the plaintext hash from the map included in the tree above, so the correct data has been returned. +Locks +----- + +The restic repository structure is designed in a way that allows parallel +access of multiple instance of restic and even parallel writes. However, there +are some functions that work more efficient or even require exclusive access of +the repository. In order to implement these functions, restic processes are +required to create a lock on the repository before doing anything. + +Locks come in two types: Exclusive and non-exclusive locks. At most one +process can have an exclusive lock on the repository, and during that time +there mustn't be any other locks (exclusive and non-exclusive). There may be +multiple non-exclusive locks in parallel. + +A lock is a file in the subdir `locks` whose filename is the storage ID of +the contents. It is encrypted and authenticated the same way as other files +in the repository and contains the following JSON structure: + + { + "time": "2015-06-27T12:18:51.759239612+02:00", + "exclusive": false, + "hostname": "kasimir", + "username": "fd0", + "pid": 13607, + "uid": 1000, + "gid": 100 + } + +The field `exclusive` defines the type of lock. When a new lock is to be +created, restic checks all locks in the repository. When a lock is found, it +is tested if the lock is stale, which is the case for locks with timestamps +older than 30 minutes. If the lock was created on the same machine, even for +younger locks it is tested whether the process is still alive by sending a +signal to it. If that fails, restic assumes that the process is dead and +considers the lock to be stale. + +When a new lock is to be created and no other conflicting locks are +detected, restic creates a new lock, waits, and checks if other locks +appeared in the repository. Depending on the type of the other locks and the +lock to be created, restic either continues or fails. + Backups and Deduplication ========================= diff --git a/lock.go b/lock.go new file mode 100644 index 000000000..65d102323 --- /dev/null +++ b/lock.go @@ -0,0 +1,282 @@ +package restic + +import ( + "fmt" + "os" + "os/signal" + "os/user" + "strconv" + "sync" + "syscall" + "time" + + "github.com/restic/restic/backend" + "github.com/restic/restic/debug" + "github.com/restic/restic/repository" +) + +// Lock represents a process locking the repository for an operation. +// +// There are two types of locks: exclusive and non-exclusive. There may be many +// different non-exclusive locks, but at most one exclusive lock, which can +// only be acquired while no non-exclusive lock is held. +type Lock struct { + Time time.Time `json:"time"` + Exclusive bool `json:"exclusive"` + Hostname string `json:"hostname"` + Username string `json:"username"` + PID int `json:"pid"` + UID uint32 `json:"uid,omitempty"` + GID uint32 `json:"gid,omitempty"` + + repo *repository.Repository + lockID backend.ID +} + +// ErrAlreadyLocked is returned when NewLock or NewExclusiveLock are unable to +// acquire the desired lock. +type ErrAlreadyLocked struct { + otherLock *Lock +} + +func (e ErrAlreadyLocked) Error() string { + return fmt.Sprintf("repository is already locked by %v", e.otherLock) +} + +// IsAlreadyLocked returns true iff err is an instance of ErrAlreadyLocked. +func IsAlreadyLocked(err error) bool { + if _, ok := err.(ErrAlreadyLocked); ok { + return true + } + + return false +} + +// NewLock returns a new, non-exclusive lock for the repository. If an +// exclusive lock is already held by another process, ErrAlreadyLocked is +// returned. +func NewLock(repo *repository.Repository) (*Lock, error) { + return newLock(repo, false) +} + +// NewExclusiveLock returns a new, exclusive lock for the repository. If +// another lock (normal and exclusive) is already held by another process, +// ErrAlreadyLocked is returned. +func NewExclusiveLock(repo *repository.Repository) (*Lock, error) { + return newLock(repo, true) +} + +const waitBeforeLockCheck = 200 * time.Millisecond + +func newLock(repo *repository.Repository, excl bool) (*Lock, error) { + lock := &Lock{ + Time: time.Now(), + PID: os.Getpid(), + Exclusive: excl, + repo: repo, + } + + hn, err := os.Hostname() + if err == nil { + lock.Hostname = hn + } + + if err = lock.fillUserInfo(); err != nil { + return nil, err + } + + if err = lock.checkForOtherLocks(); err != nil { + return nil, err + } + + err = lock.createLock() + if err != nil { + return nil, err + } + + time.Sleep(waitBeforeLockCheck) + + if err = lock.checkForOtherLocks(); err != nil { + lock.Unlock() + return nil, err + } + + return lock, nil +} + +func (l *Lock) fillUserInfo() error { + usr, err := user.Current() + if err != nil { + return nil + } + l.Username = usr.Username + + uid, err := strconv.ParseInt(usr.Uid, 10, 32) + if err != nil { + return err + } + l.UID = uint32(uid) + + gid, err := strconv.ParseInt(usr.Gid, 10, 32) + if err != nil { + return err + } + l.GID = uint32(gid) + + return nil +} + +// checkForOtherLocks looks for other locks that currently exist in the repository. +// +// If an exclusive lock is to be created, checkForOtherLocks returns an error +// if there are any other locks, regardless if exclusive or not. If a +// non-exclusive lock is to be created, an error is only returned when an +// exclusive lock is found. +func (l *Lock) checkForOtherLocks() error { + return eachLock(l.repo, func(id backend.ID, lock *Lock, err error) error { + if id.Equal(l.lockID) { + return nil + } + + // ignore locks that cannot be loaded + if err != nil { + return nil + } + + if l.Exclusive { + return ErrAlreadyLocked{otherLock: lock} + } + + if !l.Exclusive && lock.Exclusive { + return ErrAlreadyLocked{otherLock: lock} + } + + return nil + }) +} + +func eachLock(repo *repository.Repository, f func(backend.ID, *Lock, error) error) error { + done := make(chan struct{}) + defer close(done) + + for id := range repo.List(backend.Lock, done) { + lock, err := LoadLock(repo, id) + err = f(id, lock, err) + if err != nil { + return err + } + } + + return nil +} + +// createLock acquires the lock by creating a file in the repository. +func (l *Lock) createLock() error { + id, err := l.repo.SaveJSONUnpacked(backend.Lock, l) + if err != nil { + return err + } + + l.lockID = id + return nil +} + +// Unlock removes the lock from the repository. +func (l *Lock) Unlock() error { + if l == nil || l.lockID == nil { + return nil + } + + return l.repo.Backend().Remove(backend.Lock, l.lockID.String()) +} + +var staleTimeout = 30 * time.Minute + +// Stale returns true if the lock is stale. A lock is stale if the timestamp is +// older than 30 minutes or if it was created on the current machine and the +// process isn't alive any more. +func (l *Lock) Stale() bool { + debug.Log("Lock.Stale", "testing if lock %v for process %d is stale", l.lockID.Str(), l.PID) + if time.Now().Sub(l.Time) > staleTimeout { + debug.Log("Lock.Stale", "lock is stale, timestamp is too old: %v\n", l.Time) + return true + } + + proc, err := os.FindProcess(l.PID) + defer proc.Release() + if err != nil { + debug.Log("Lock.Stale", "error searching for process %d: %v\n", l.PID, err) + return true + } + + debug.Log("Lock.Stale", "sending SIGHUP to process %d\n", l.PID) + err = proc.Signal(syscall.SIGHUP) + if err != nil { + debug.Log("Lock.Stale", "signal error: %v, lock is probably stale\n", err) + return true + } + + debug.Log("Lock.Stale", "lock not stale\n") + return false +} + +func (l Lock) String() string { + text := fmt.Sprintf("PID %d on %s by %s (UID %d, GID %d)\nlock was created at %s (%s ago)\nstorage ID %v", + l.PID, l.Hostname, l.Username, l.UID, l.GID, + l.Time.Format("2006-01-02 15:04:05"), time.Since(l.Time), + l.lockID.Str()) + + if l.Stale() { + text += " (stale)" + } + + return text +} + +// listen for incoming SIGHUP and ignore +var ignoreSIGHUP sync.Once + +func init() { + ignoreSIGHUP.Do(func() { + go func() { + c := make(chan os.Signal) + signal.Notify(c, syscall.SIGHUP) + for s := range c { + debug.Log("lock.ignoreSIGHUP", "Signal received: %v\n", s) + } + }() + }) +} + +// LoadLock loads and unserializes a lock from a repository. +func LoadLock(repo *repository.Repository, id backend.ID) (*Lock, error) { + lock := &Lock{} + if err := repo.LoadJSONUnpacked(backend.Lock, id, lock); err != nil { + return nil, err + } + lock.lockID = id + + return lock, nil +} + +// RemoveStaleLocks deletes all locks detected as stale from the repository. +func RemoveStaleLocks(repo *repository.Repository) error { + return eachLock(repo, func(id backend.ID, lock *Lock, err error) error { + // ignore locks that cannot be loaded + if err != nil { + return nil + } + + if lock.Stale() { + return repo.Backend().Remove(backend.Lock, id.String()) + } + + return nil + }) +} + +func RemoveAllLocks(repo *repository.Repository) error { + return eachLock(repo, func(id backend.ID, lock *Lock, err error) error { + return repo.Backend().Remove(backend.Lock, id.String()) + }) +} diff --git a/lock_test.go b/lock_test.go new file mode 100644 index 000000000..988c5d440 --- /dev/null +++ b/lock_test.go @@ -0,0 +1,197 @@ +package restic_test + +import ( + "os" + "testing" + "time" + + "github.com/restic/restic" + "github.com/restic/restic/backend" + "github.com/restic/restic/repository" + . "github.com/restic/restic/test" +) + +func TestLock(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + lock, err := restic.NewLock(repo) + OK(t, err) + + OK(t, lock.Unlock()) +} + +func TestDoubleUnlock(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + lock, err := restic.NewLock(repo) + OK(t, err) + + OK(t, lock.Unlock()) + + err = lock.Unlock() + Assert(t, err != nil, + "double unlock didn't return an error, got %v", err) +} + +func TestMultipleLock(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + lock1, err := restic.NewLock(repo) + OK(t, err) + + lock2, err := restic.NewLock(repo) + OK(t, err) + + OK(t, lock1.Unlock()) + OK(t, lock2.Unlock()) +} + +func TestLockExclusive(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + elock, err := restic.NewExclusiveLock(repo) + OK(t, err) + OK(t, elock.Unlock()) +} + +func TestLockOnExclusiveLockedRepo(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + elock, err := restic.NewExclusiveLock(repo) + OK(t, err) + + lock, err := restic.NewLock(repo) + Assert(t, err != nil, + "create normal lock with exclusively locked repo didn't return an error") + Assert(t, restic.IsAlreadyLocked(err), + "create normal lock with exclusively locked repo didn't return the correct error") + + OK(t, lock.Unlock()) + OK(t, elock.Unlock()) +} + +func TestExclusiveLockOnLockedRepo(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + elock, err := restic.NewLock(repo) + OK(t, err) + + lock, err := restic.NewExclusiveLock(repo) + Assert(t, err != nil, + "create normal lock with exclusively locked repo didn't return an error") + Assert(t, restic.IsAlreadyLocked(err), + "create normal lock with exclusively locked repo didn't return the correct error") + + OK(t, lock.Unlock()) + OK(t, elock.Unlock()) +} + +func createFakeLock(repo *repository.Repository, t time.Time, pid int) (backend.ID, error) { + newLock := &restic.Lock{Time: t, PID: pid} + return repo.SaveJSONUnpacked(backend.Lock, &newLock) +} + +func removeLock(repo *repository.Repository, id backend.ID) error { + return repo.Backend().Remove(backend.Lock, id.String()) +} + +var staleLockTests = []struct { + timestamp time.Time + stale bool + pid int +}{ + { + timestamp: time.Now(), + stale: false, + pid: os.Getpid(), + }, + { + timestamp: time.Now().Add(-time.Hour), + stale: true, + pid: os.Getpid(), + }, + { + timestamp: time.Now().Add(3 * time.Minute), + stale: false, + pid: os.Getpid(), + }, + { + timestamp: time.Now(), + stale: true, + pid: os.Getpid() + 500, + }, +} + +func TestLockStale(t *testing.T) { + for i, test := range staleLockTests { + lock := restic.Lock{ + Time: test.timestamp, + PID: test.pid, + } + + Assert(t, lock.Stale() == test.stale, + "TestStaleLock: test %d failed: expected stale: %v, got %v", + i, test.stale, !test.stale) + } +} + +func lockExists(repo *repository.Repository, t testing.TB, id backend.ID) bool { + exists, err := repo.Backend().Test(backend.Lock, id.String()) + OK(t, err) + + return exists +} + +func TestLockWithStaleLock(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + id1, err := createFakeLock(repo, time.Now().Add(-time.Hour), os.Getpid()) + OK(t, err) + + id2, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()) + OK(t, err) + + id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500) + OK(t, err) + + OK(t, restic.RemoveStaleLocks(repo)) + + Assert(t, lockExists(repo, t, id1) == false, + "stale lock still exists after RemoveStaleLocks was called") + Assert(t, lockExists(repo, t, id2) == true, + "non-stale lock was removed by RemoveStaleLocks") + Assert(t, lockExists(repo, t, id3) == false, + "stale lock still exists after RemoveStaleLocks was called") + + OK(t, removeLock(repo, id2)) +} + +func TestRemoveAllLocks(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + id1, err := createFakeLock(repo, time.Now().Add(-time.Hour), os.Getpid()) + OK(t, err) + + id2, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()) + OK(t, err) + + id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500) + OK(t, err) + + OK(t, restic.RemoveAllLocks(repo)) + + Assert(t, lockExists(repo, t, id1) == false, + "lock still exists after RemoveAllLocks was called") + Assert(t, lockExists(repo, t, id2) == false, + "lock still exists after RemoveAllLocks was called") + Assert(t, lockExists(repo, t, id3) == false, + "lock still exists after RemoveAllLocks was called") +} diff --git a/pipe/pipe_test.go b/pipe/pipe_test.go index edb270e6f..42ff7c31d 100644 --- a/pipe/pipe_test.go +++ b/pipe/pipe_test.go @@ -1,7 +1,6 @@ package pipe_test import ( - "flag" "os" "path/filepath" "sync" @@ -12,9 +11,6 @@ import ( . "github.com/restic/restic/test" ) -var testWalkerPath = flag.String("test.walkerpath", ".", "pipeline walker testpath (default: .)") -var maxWorkers = flag.Int("test.workers", 100, "max concurrency (default: 100)") - func isFile(fi os.FileInfo) bool { return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0 } @@ -27,7 +23,7 @@ func statPath(path string) (stats, error) { var s stats // count files and directories with filepath.Walk() - err := filepath.Walk(*testWalkerPath, func(p string, fi os.FileInfo, err error) error { + err := filepath.Walk(TestWalkerPath, func(p string, fi os.FileInfo, err error) error { if fi == nil { return err } @@ -44,15 +40,17 @@ func statPath(path string) (stats, error) { return s, err } +const maxWorkers = 100 + func TestPipelineWalkerWithSplit(t *testing.T) { - if *testWalkerPath == "" { + if TestWalkerPath == "" { t.Skipf("walkerpath not set, skipping TestPipelineWalker") } - before, err := statPath(*testWalkerPath) + before, err := statPath(TestWalkerPath) OK(t, err) - t.Logf("walking path %s with %d dirs, %d files", *testWalkerPath, + t.Logf("walking path %s with %d dirs, %d files", TestWalkerPath, before.dirs, before.files) // account for top level dir @@ -105,7 +103,7 @@ func TestPipelineWalkerWithSplit(t *testing.T) { entCh := make(chan pipe.Entry) dirCh := make(chan pipe.Dir) - for i := 0; i < *maxWorkers; i++ { + for i := 0; i < maxWorkers; i++ { wg.Add(1) go worker(&wg, done, entCh, dirCh) } @@ -120,7 +118,7 @@ func TestPipelineWalkerWithSplit(t *testing.T) { }() resCh := make(chan pipe.Result, 1) - err = pipe.Walk([]string{*testWalkerPath}, done, jobs, resCh) + err = pipe.Walk([]string{TestWalkerPath}, done, jobs, resCh) OK(t, err) // wait for all workers to terminate @@ -129,21 +127,21 @@ func TestPipelineWalkerWithSplit(t *testing.T) { // wait for top-level blob <-resCh - t.Logf("walked path %s with %d dirs, %d files", *testWalkerPath, + t.Logf("walked path %s with %d dirs, %d files", TestWalkerPath, after.dirs, after.files) Assert(t, before == after, "stats do not match, expected %v, got %v", before, after) } func TestPipelineWalker(t *testing.T) { - if *testWalkerPath == "" { + if TestWalkerPath == "" { t.Skipf("walkerpath not set, skipping TestPipelineWalker") } - before, err := statPath(*testWalkerPath) + before, err := statPath(TestWalkerPath) OK(t, err) - t.Logf("walking path %s with %d dirs, %d files", *testWalkerPath, + t.Logf("walking path %s with %d dirs, %d files", TestWalkerPath, before.dirs, before.files) // account for top level dir @@ -194,13 +192,13 @@ func TestPipelineWalker(t *testing.T) { done := make(chan struct{}) jobs := make(chan pipe.Job) - for i := 0; i < *maxWorkers; i++ { + for i := 0; i < maxWorkers; i++ { wg.Add(1) go worker(&wg, done, jobs) } resCh := make(chan pipe.Result, 1) - err = pipe.Walk([]string{*testWalkerPath}, done, jobs, resCh) + err = pipe.Walk([]string{TestWalkerPath}, done, jobs, resCh) OK(t, err) // wait for all workers to terminate @@ -209,14 +207,14 @@ func TestPipelineWalker(t *testing.T) { // wait for top-level blob <-resCh - t.Logf("walked path %s with %d dirs, %d files", *testWalkerPath, + t.Logf("walked path %s with %d dirs, %d files", TestWalkerPath, after.dirs, after.files) Assert(t, before == after, "stats do not match, expected %v, got %v", before, after) } func BenchmarkPipelineWalker(b *testing.B) { - if *testWalkerPath == "" { + if TestWalkerPath == "" { b.Skipf("walkerpath not set, skipping BenchPipelineWalker") } @@ -283,8 +281,8 @@ func BenchmarkPipelineWalker(b *testing.B) { dirCh := make(chan pipe.Dir, 200) var wg sync.WaitGroup - b.Logf("starting %d workers", *maxWorkers) - for i := 0; i < *maxWorkers; i++ { + b.Logf("starting %d workers", maxWorkers) + for i := 0; i < maxWorkers; i++ { wg.Add(2) go dirWorker(&wg, done, dirCh) go fileWorker(&wg, done, entCh) @@ -300,7 +298,7 @@ func BenchmarkPipelineWalker(b *testing.B) { }() resCh := make(chan pipe.Result, 1) - err := pipe.Walk([]string{*testWalkerPath}, done, jobs, resCh) + err := pipe.Walk([]string{TestWalkerPath}, done, jobs, resCh) OK(b, err) // wait for all workers to terminate @@ -314,13 +312,13 @@ func BenchmarkPipelineWalker(b *testing.B) { } func TestPipelineWalkerMultiple(t *testing.T) { - if *testWalkerPath == "" { + if TestWalkerPath == "" { t.Skipf("walkerpath not set, skipping TestPipelineWalker") } - paths, err := filepath.Glob(filepath.Join(*testWalkerPath, "*")) + paths, err := filepath.Glob(filepath.Join(TestWalkerPath, "*")) - before, err := statPath(*testWalkerPath) + before, err := statPath(TestWalkerPath) OK(t, err) t.Logf("walking paths %v with %d dirs, %d files", paths, @@ -371,7 +369,7 @@ func TestPipelineWalkerMultiple(t *testing.T) { done := make(chan struct{}) jobs := make(chan pipe.Job) - for i := 0; i < *maxWorkers; i++ { + for i := 0; i < maxWorkers; i++ { wg.Add(1) go worker(&wg, done, jobs) } diff --git a/repository/repository.go b/repository/repository.go index aad5f3c9b..491ef6bd7 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -372,7 +372,7 @@ func (s *Repository) SaveJSONUnpacked(t backend.Type, item interface{}) (backend if err != nil { return nil, err } - debug.Log("Repo.SaveJSONUnpacked", "create new file %p", blob) + debug.Log("Repo.SaveJSONUnpacked", "create new blob %v", t) // hash hw := backend.NewHashingWriter(blob, sha256.New()) @@ -396,9 +396,12 @@ func (s *Repository) SaveJSONUnpacked(t backend.Type, item interface{}) (backend err = blob.Finalize(t, sid.String()) if err != nil { + debug.Log("Repo.SaveJSONUnpacked", "error saving blob %v as %v: %v", t, sid, err) return nil, err } + debug.Log("Repo.SaveJSONUnpacked", "new blob %v saved as %v", t, sid) + return sid, nil } diff --git a/repository/repository_test.go b/repository/repository_test.go index fe9f5c9d4..7691e596a 100644 --- a/repository/repository_test.go +++ b/repository/repository_test.go @@ -5,7 +5,6 @@ import ( "crypto/rand" "crypto/sha256" "encoding/json" - "flag" "io" "testing" @@ -15,8 +14,6 @@ import ( . "github.com/restic/restic/test" ) -var benchTestDir = flag.String("test.dir", ".", "dir used in benchmarks (default: .)") - type testJSONStruct struct { Foo uint32 Bar string @@ -28,8 +25,8 @@ var repoTests = []testJSONStruct{ } func TestSaveJSON(t *testing.T) { - repo := SetupRepo(t) - defer TeardownRepo(t, repo) + repo := SetupRepo() + defer TeardownRepo(repo) for _, obj := range repoTests { data, err := json.Marshal(obj) @@ -47,8 +44,8 @@ func TestSaveJSON(t *testing.T) { } func BenchmarkSaveJSON(t *testing.B) { - repo := SetupRepo(t) - defer TeardownRepo(t, repo) + repo := SetupRepo() + defer TeardownRepo(repo) obj := repoTests[0] @@ -72,8 +69,8 @@ func BenchmarkSaveJSON(t *testing.B) { var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20} func TestSave(t *testing.T) { - repo := SetupRepo(t) - defer TeardownRepo(t, repo) + repo := SetupRepo() + defer TeardownRepo(repo) for _, size := range testSizes { data := make([]byte, size) @@ -104,8 +101,8 @@ func TestSave(t *testing.T) { } func TestSaveFrom(t *testing.T) { - repo := SetupRepo(t) - defer TeardownRepo(t, repo) + repo := SetupRepo() + defer TeardownRepo(repo) for _, size := range testSizes { data := make([]byte, size) @@ -134,8 +131,8 @@ func TestSaveFrom(t *testing.T) { } func BenchmarkSaveFrom(t *testing.B) { - repo := SetupRepo(t) - defer TeardownRepo(t, repo) + repo := SetupRepo() + defer TeardownRepo(repo) size := 4 << 20 // 4MiB @@ -156,15 +153,15 @@ func BenchmarkSaveFrom(t *testing.B) { } func TestLoadJSONPack(t *testing.T) { - if *benchTestDir == "" { + repo := SetupRepo() + defer TeardownRepo(repo) + + if BenchArchiveDirectory == "" { t.Skip("benchdir not set, skipping") } - repo := SetupRepo(t) - defer TeardownRepo(t, repo) - // archive a few files - sn := SnapshotDir(t, repo, *benchTestDir, nil) + sn := SnapshotDir(t, repo, BenchArchiveDirectory, nil) OK(t, repo.Flush()) tree := restic.NewTree() @@ -173,13 +170,13 @@ func TestLoadJSONPack(t *testing.T) { } func TestLoadJSONUnpacked(t *testing.T) { - if *benchTestDir == "" { + repo := SetupRepo() + defer TeardownRepo(repo) + + if BenchArchiveDirectory == "" { t.Skip("benchdir not set, skipping") } - repo := SetupRepo(t) - defer TeardownRepo(t, repo) - // archive a snapshot sn := restic.Snapshot{} sn.Hostname = "foobar" diff --git a/snapshot_test.go b/snapshot_test.go index 00bb7a038..649e9a1f4 100644 --- a/snapshot_test.go +++ b/snapshot_test.go @@ -8,9 +8,6 @@ import ( ) func TestNewSnapshot(t *testing.T) { - s := SetupRepo(t) - defer TeardownRepo(t, s) - paths := []string{"/home/foobar"} _, err := restic.NewSnapshot(paths) diff --git a/test/backend.go b/test/backend.go index bcf874c0f..c2d1fe285 100644 --- a/test/backend.go +++ b/test/backend.go @@ -14,12 +14,13 @@ import ( ) var ( - TestPassword = getStringVar("RESTIC_TEST_PASSWORD", "geheim") - TestCleanup = getBoolVar("RESTIC_TEST_CLEANUP", true) - TestTempDir = getStringVar("RESTIC_TEST_TMPDIR", "") - RunIntegrationTest = getBoolVar("RESTIC_TEST_INTEGRATION", true) - TestSFTPPath = getStringVar("RESTIC_TEST_SFTPPATH", - "/usr/lib/ssh:/usr/lib/openssh") + TestPassword = getStringVar("RESTIC_TEST_PASSWORD", "geheim") + TestCleanup = getBoolVar("RESTIC_TEST_CLEANUP", true) + TestTempDir = getStringVar("RESTIC_TEST_TMPDIR", "") + RunIntegrationTest = getBoolVar("RESTIC_TEST_INTEGRATION", true) + TestSFTPPath = getStringVar("RESTIC_TEST_SFTPPATH", "/usr/lib/ssh:/usr/lib/openssh") + TestWalkerPath = getStringVar("RESTIC_TEST_PATH", ".") + BenchArchiveDirectory = getStringVar("RESTIC_BENCH_DIR", ".") ) func getStringVar(name, defaultValue string) string { @@ -45,27 +46,38 @@ func getBoolVar(name string, defaultValue bool) bool { return defaultValue } -func SetupRepo(t testing.TB) *repository.Repository { +func SetupRepo() *repository.Repository { tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-") - OK(t, err) + if err != nil { + panic(err) + } // create repository below temp dir b, err := local.Create(filepath.Join(tempdir, "repo")) - OK(t, err) + if err != nil { + panic(err) + } repo := repository.New(b) - OK(t, repo.Init(TestPassword)) + err = repo.Init(TestPassword) + if err != nil { + panic(err) + } + return repo } -func TeardownRepo(t testing.TB, repo *repository.Repository) { +func TeardownRepo(repo *repository.Repository) { if !TestCleanup { l := repo.Backend().(*local.Local) - t.Logf("leaving local backend at %s\n", l.Location()) + fmt.Printf("leaving local backend at %s\n", l.Location()) return } - OK(t, repo.Delete()) + err := repo.Delete() + if err != nil { + panic(err) + } } func SnapshotDir(t testing.TB, repo *repository.Repository, path string, parent backend.ID) *restic.Snapshot { @@ -74,3 +86,9 @@ func SnapshotDir(t testing.TB, repo *repository.Repository, path string, parent OK(t, err) return sn } + +func WithRepo(t testing.TB, f func(*repository.Repository)) { + repo := SetupRepo() + f(repo) + TeardownRepo(repo) +} diff --git a/tree_test.go b/tree_test.go index b46f01b0a..2d9cb7b7e 100644 --- a/tree_test.go +++ b/tree_test.go @@ -93,8 +93,8 @@ func TestNodeComparison(t *testing.T) { } func TestLoadTree(t *testing.T) { - repo := SetupRepo(t) - defer TeardownRepo(t, repo) + repo := SetupRepo() + defer TeardownRepo(repo) // save tree tree := restic.NewTree() diff --git a/walk_test.go b/walk_test.go index 384e605df..397655978 100644 --- a/walk_test.go +++ b/walk_test.go @@ -1,7 +1,6 @@ package restic_test import ( - "flag" "path/filepath" "testing" @@ -10,14 +9,12 @@ import ( . "github.com/restic/restic/test" ) -var testWalkDirectory = flag.String("test.walkdir", ".", "test walking a directory (globbing pattern, default: .)") - func TestWalkTree(t *testing.T) { - dirs, err := filepath.Glob(*testWalkDirectory) - OK(t, err) + repo := SetupRepo() + defer TeardownRepo(repo) - repo := SetupRepo(t) - defer TeardownRepo(t, repo) + dirs, err := filepath.Glob(TestWalkerPath) + OK(t, err) // archive a few files arch := restic.NewArchiver(repo)