From 46b7a270a6de16b1d4e0831ca2fd747644a5c985 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 5 Jun 2017 23:56:59 +0200 Subject: [PATCH] Add context parameters to tests --- src/restic/archiver/archive_reader_test.go | 18 +++--- .../archiver/archiver_duplication_test.go | 21 +++---- src/restic/archiver/archiver_int_test.go | 17 +++--- src/restic/archiver/archiver_test.go | 18 +++--- src/restic/checker/checker_test.go | 52 +++++++++-------- src/restic/config_test.go | 11 ++-- src/restic/find_test.go | 5 +- src/restic/fuse/file_test.go | 10 ++-- src/restic/index/index_test.go | 38 ++++++------- src/restic/lock_test.go | 37 ++++++------ src/restic/node_test.go | 3 +- src/restic/pipe/pipe_test.go | 57 ++++++++++--------- src/restic/repository/packer_manager_test.go | 5 +- src/restic/repository/parallel_test.go | 13 +++-- src/restic/repository/repack_test.go | 32 +++++------ src/restic/repository/repository_test.go | 51 +++++++++-------- src/restic/testing_test.go | 3 +- src/restic/tree_test.go | 5 +- src/restic/walk/walk_test.go | 21 ++++--- src/restic/worker/pool.go | 9 ++- 20 files changed, 215 insertions(+), 211 deletions(-) diff --git a/src/restic/archiver/archive_reader_test.go b/src/restic/archiver/archive_reader_test.go index a8ab18668..03c644846 100644 --- a/src/restic/archiver/archive_reader_test.go +++ b/src/restic/archiver/archive_reader_test.go @@ -2,6 +2,7 @@ package archiver import ( "bytes" + "context" "errors" "io" "math/rand" @@ -12,7 +13,7 @@ import ( ) func loadBlob(t *testing.T, repo restic.Repository, id restic.ID, buf []byte) int { - n, err := repo.LoadBlob(restic.DataBlob, id, buf) + n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf) if err != nil { t.Fatalf("LoadBlob(%v) returned error %v", id, err) } @@ -21,7 +22,7 @@ func loadBlob(t *testing.T, repo restic.Repository, id restic.ID, buf []byte) in } func checkSavedFile(t *testing.T, repo restic.Repository, treeID restic.ID, name string, rd io.Reader) { - tree, err := repo.LoadTree(treeID) + tree, err := repo.LoadTree(context.TODO(), treeID) if err != nil { t.Fatalf("LoadTree() returned error %v", err) } @@ -85,7 +86,7 @@ func TestArchiveReader(t *testing.T) { Tags: []string{"test"}, } - sn, id, err := r.Archive("fakefile", f, nil) + sn, id, err := r.Archive(context.TODO(), "fakefile", f, nil) if err != nil { t.Fatalf("ArchiveReader() returned error %v", err) } @@ -111,7 +112,7 @@ func TestArchiveReaderNull(t *testing.T) { Tags: []string{"test"}, } - sn, id, err := r.Archive("fakefile", bytes.NewReader(nil), nil) + sn, id, err := r.Archive(context.TODO(), "fakefile", bytes.NewReader(nil), nil) if err != nil { t.Fatalf("ArchiveReader() returned error %v", err) } @@ -132,11 +133,8 @@ func (e errReader) Read([]byte) (int, error) { } func countSnapshots(t testing.TB, repo restic.Repository) int { - done := make(chan struct{}) - defer close(done) - snapshots := 0 - for range repo.List(restic.SnapshotFile, done) { + for range repo.List(context.TODO(), restic.SnapshotFile) { snapshots++ } return snapshots @@ -152,7 +150,7 @@ func TestArchiveReaderError(t *testing.T) { Tags: []string{"test"}, } - sn, id, err := r.Archive("fakefile", errReader("error returned by reading stdin"), nil) + sn, id, err := r.Archive(context.TODO(), "fakefile", errReader("error returned by reading stdin"), nil) if err == nil { t.Errorf("expected error not returned") } @@ -195,7 +193,7 @@ func BenchmarkArchiveReader(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - _, _, err := r.Archive("fakefile", bytes.NewReader(buf), nil) + _, _, err := r.Archive(context.TODO(), "fakefile", bytes.NewReader(buf), nil) if err != nil { t.Fatal(err) } diff --git a/src/restic/archiver/archiver_duplication_test.go b/src/restic/archiver/archiver_duplication_test.go index c7de1cc49..9c01d5035 100644 --- a/src/restic/archiver/archiver_duplication_test.go +++ b/src/restic/archiver/archiver_duplication_test.go @@ -1,6 +1,7 @@ package archiver_test import ( + "context" "crypto/rand" "io" mrand "math/rand" @@ -39,33 +40,33 @@ func randomID() restic.ID { func forgetfulBackend() restic.Backend { be := &mock.Backend{} - be.TestFn = func(h restic.Handle) (bool, error) { + be.TestFn = func(ctx context.Context, h restic.Handle) (bool, error) { return false, nil } - be.LoadFn = func(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + be.LoadFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { return nil, errors.New("not found") } - be.SaveFn = func(h restic.Handle, rd io.Reader) error { + be.SaveFn = func(ctx context.Context, h restic.Handle, rd io.Reader) error { return nil } - be.StatFn = func(h restic.Handle) (restic.FileInfo, error) { + be.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { return restic.FileInfo{}, errors.New("not found") } - be.RemoveFn = func(h restic.Handle) error { + be.RemoveFn = func(ctx context.Context, h restic.Handle) error { return nil } - be.ListFn = func(t restic.FileType, done <-chan struct{}) <-chan string { + be.ListFn = func(ctx context.Context, t restic.FileType) <-chan string { ch := make(chan string) close(ch) return ch } - be.DeleteFn = func() error { + be.DeleteFn = func(ctx context.Context) error { return nil } @@ -80,7 +81,7 @@ func testArchiverDuplication(t *testing.T) { repo := repository.New(forgetfulBackend()) - err = repo.Init("foo") + err = repo.Init(context.TODO(), "foo") if err != nil { t.Fatal(err) } @@ -108,7 +109,7 @@ func testArchiverDuplication(t *testing.T) { buf := make([]byte, 50) - err := arch.Save(restic.DataBlob, buf, id) + err := arch.Save(context.TODO(), restic.DataBlob, buf, id) if err != nil { t.Fatal(err) } @@ -127,7 +128,7 @@ func testArchiverDuplication(t *testing.T) { case <-done: return case <-ticker.C: - err := repo.SaveFullIndex() + err := repo.SaveFullIndex(context.TODO()) if err != nil { t.Fatal(err) } diff --git a/src/restic/archiver/archiver_int_test.go b/src/restic/archiver/archiver_int_test.go index c4014f5b0..eb135436e 100644 --- a/src/restic/archiver/archiver_int_test.go +++ b/src/restic/archiver/archiver_int_test.go @@ -1,6 +1,7 @@ package archiver import ( + "context" "os" "testing" @@ -83,10 +84,10 @@ func (j testPipeJob) Error() error { return j.err } func (j testPipeJob) Info() os.FileInfo { return j.fi } func (j testPipeJob) Result() chan<- pipe.Result { return j.res } -func testTreeWalker(done <-chan struct{}, out chan<- walk.TreeJob) { +func testTreeWalker(ctx context.Context, out chan<- walk.TreeJob) { for _, e := range treeJobs { select { - case <-done: + case <-ctx.Done(): return case out <- walk.TreeJob{Path: e}: } @@ -95,10 +96,10 @@ func testTreeWalker(done <-chan struct{}, out chan<- walk.TreeJob) { close(out) } -func testPipeWalker(done <-chan struct{}, out chan<- pipe.Job) { +func testPipeWalker(ctx context.Context, out chan<- pipe.Job) { for _, e := range pipeJobs { select { - case <-done: + case <-ctx.Done(): return case out <- testPipeJob{path: e}: } @@ -108,19 +109,19 @@ func testPipeWalker(done <-chan struct{}, out chan<- pipe.Job) { } func TestArchivePipe(t *testing.T) { - done := make(chan struct{}) + ctx := context.TODO() treeCh := make(chan walk.TreeJob) pipeCh := make(chan pipe.Job) - go testTreeWalker(done, treeCh) - go testPipeWalker(done, pipeCh) + go testTreeWalker(ctx, treeCh) + go testPipeWalker(ctx, pipeCh) p := archivePipe{Old: treeCh, New: pipeCh} ch := make(chan pipe.Job) - go p.compare(done, ch) + go p.compare(ctx, ch) i := 0 for job := range ch { diff --git a/src/restic/archiver/archiver_test.go b/src/restic/archiver/archiver_test.go index 3536ff727..f4928089b 100644 --- a/src/restic/archiver/archiver_test.go +++ b/src/restic/archiver/archiver_test.go @@ -2,6 +2,7 @@ package archiver_test import ( "bytes" + "context" "io" "testing" "time" @@ -104,7 +105,7 @@ func archiveDirectory(b testing.TB) { arch := archiver.New(repo) - _, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil, "localhost", nil) + _, id, err := arch.Snapshot(context.TODO(), nil, []string{BenchArchiveDirectory}, nil, "localhost", nil) OK(b, err) b.Logf("snapshot archived as %v", id) @@ -129,7 +130,7 @@ func BenchmarkArchiveDirectory(b *testing.B) { } func countPacks(repo restic.Repository, t restic.FileType) (n uint) { - for range repo.Backend().List(t, nil) { + for range repo.Backend().List(context.TODO(), t) { n++ } @@ -234,7 +235,7 @@ func testParallelSaveWithDuplication(t *testing.T, seed int) { id := restic.Hash(c.Data) time.Sleep(time.Duration(id[0])) - err := arch.Save(restic.DataBlob, c.Data, id) + err := arch.Save(context.TODO(), restic.DataBlob, c.Data, id) <-barrier errChan <- err }(c, errChan) @@ -246,7 +247,7 @@ func testParallelSaveWithDuplication(t *testing.T, seed int) { } OK(t, repo.Flush()) - OK(t, repo.SaveIndex()) + OK(t, repo.SaveIndex(context.TODO())) chkr := createAndInitChecker(t, repo) assertNoUnreferencedPacks(t, chkr) @@ -271,7 +272,7 @@ func getRandomData(seed int, size int) []chunker.Chunk { func createAndInitChecker(t *testing.T, repo restic.Repository) *checker.Checker { chkr := checker.New(repo) - hints, errs := chkr.LoadIndex() + hints, errs := chkr.LoadIndex(context.TODO()) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -284,11 +285,8 @@ func createAndInitChecker(t *testing.T, repo restic.Repository) *checker.Checker } func assertNoUnreferencedPacks(t *testing.T, chkr *checker.Checker) { - done := make(chan struct{}) - defer close(done) - errChan := make(chan error) - go chkr.Packs(errChan, done) + go chkr.Packs(context.TODO(), errChan) for err := range errChan { OK(t, err) @@ -301,7 +299,7 @@ func TestArchiveEmptySnapshot(t *testing.T) { arch := archiver.New(repo) - sn, id, err := arch.Snapshot(nil, []string{"file-does-not-exist-123123213123", "file2-does-not-exist-too-123123123"}, nil, "localhost", nil) + sn, id, err := arch.Snapshot(context.TODO(), nil, []string{"file-does-not-exist-123123213123", "file2-does-not-exist-too-123123123"}, nil, "localhost", nil) if err == nil { t.Errorf("expected error for empty snapshot, got nil") } diff --git a/src/restic/checker/checker_test.go b/src/restic/checker/checker_test.go index 65e764137..ee345a97b 100644 --- a/src/restic/checker/checker_test.go +++ b/src/restic/checker/checker_test.go @@ -1,6 +1,7 @@ package checker_test import ( + "context" "io" "math/rand" "path/filepath" @@ -16,13 +17,13 @@ import ( var checkerTestData = filepath.Join("testdata", "checker-test-repo.tar.gz") -func collectErrors(f func(chan<- error, <-chan struct{})) (errs []error) { - done := make(chan struct{}) - defer close(done) +func collectErrors(ctx context.Context, f func(context.Context, chan<- error)) (errs []error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() errChan := make(chan error) - go f(errChan, done) + go f(ctx, errChan) for err := range errChan { errs = append(errs, err) @@ -32,17 +33,18 @@ func collectErrors(f func(chan<- error, <-chan struct{})) (errs []error) { } func checkPacks(chkr *checker.Checker) []error { - return collectErrors(chkr.Packs) + return collectErrors(context.TODO(), chkr.Packs) } func checkStruct(chkr *checker.Checker) []error { - return collectErrors(chkr.Structure) + return collectErrors(context.TODO(), chkr.Structure) } func checkData(chkr *checker.Checker) []error { return collectErrors( - func(errCh chan<- error, done <-chan struct{}) { - chkr.ReadData(nil, errCh, done) + context.TODO(), + func(ctx context.Context, errCh chan<- error) { + chkr.ReadData(ctx, nil, errCh) }, ) } @@ -54,7 +56,7 @@ func TestCheckRepo(t *testing.T) { repo := repository.TestOpenLocal(t, repodir) chkr := checker.New(repo) - hints, errs := chkr.LoadIndex() + hints, errs := chkr.LoadIndex(context.TODO()) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -77,10 +79,10 @@ func TestMissingPack(t *testing.T) { Type: restic.DataFile, Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6", } - test.OK(t, repo.Backend().Remove(packHandle)) + test.OK(t, repo.Backend().Remove(context.TODO(), packHandle)) chkr := checker.New(repo) - hints, errs := chkr.LoadIndex() + hints, errs := chkr.LoadIndex(context.TODO()) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -113,10 +115,10 @@ func TestUnreferencedPack(t *testing.T) { Type: restic.IndexFile, Name: "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44", } - test.OK(t, repo.Backend().Remove(indexHandle)) + test.OK(t, repo.Backend().Remove(context.TODO(), indexHandle)) chkr := checker.New(repo) - hints, errs := chkr.LoadIndex() + hints, errs := chkr.LoadIndex(context.TODO()) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -147,7 +149,7 @@ func TestUnreferencedBlobs(t *testing.T) { Type: restic.SnapshotFile, Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02", } - test.OK(t, repo.Backend().Remove(snapshotHandle)) + test.OK(t, repo.Backend().Remove(context.TODO(), snapshotHandle)) unusedBlobsBySnapshot := restic.IDs{ restic.TestParseID("58c748bbe2929fdf30c73262bd8313fe828f8925b05d1d4a87fe109082acb849"), @@ -161,7 +163,7 @@ func TestUnreferencedBlobs(t *testing.T) { sort.Sort(unusedBlobsBySnapshot) chkr := checker.New(repo) - hints, errs := chkr.LoadIndex() + hints, errs := chkr.LoadIndex(context.TODO()) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -192,7 +194,7 @@ func TestModifiedIndex(t *testing.T) { Type: restic.IndexFile, Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd", } - f, err := repo.Backend().Load(h, 0, 0) + f, err := repo.Backend().Load(context.TODO(), h, 0, 0) test.OK(t, err) // save the index again with a modified name so that the hash doesn't match @@ -201,13 +203,13 @@ func TestModifiedIndex(t *testing.T) { Type: restic.IndexFile, Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd", } - err = repo.Backend().Save(h2, f) + err = repo.Backend().Save(context.TODO(), h2, f) test.OK(t, err) test.OK(t, f.Close()) chkr := checker.New(repo) - hints, errs := chkr.LoadIndex() + hints, errs := chkr.LoadIndex(context.TODO()) if len(errs) == 0 { t.Fatalf("expected errors not found") } @@ -230,7 +232,7 @@ func TestDuplicatePacksInIndex(t *testing.T) { repo := repository.TestOpenLocal(t, repodir) chkr := checker.New(repo) - hints, errs := chkr.LoadIndex() + hints, errs := chkr.LoadIndex(context.TODO()) if len(hints) == 0 { t.Fatalf("did not get expected checker hints for duplicate packs in indexes") } @@ -259,8 +261,8 @@ type errorBackend struct { ProduceErrors bool } -func (b errorBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - rd, err := b.Backend.Load(h, length, offset) +func (b errorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + rd, err := b.Backend.Load(ctx, h, length, offset) if err != nil { return rd, err } @@ -303,17 +305,17 @@ func TestCheckerModifiedData(t *testing.T) { defer cleanup() arch := archiver.New(repo) - _, id, err := arch.Snapshot(nil, []string{"."}, nil, "localhost", nil) + _, id, err := arch.Snapshot(context.TODO(), nil, []string{"."}, nil, "localhost", nil) test.OK(t, err) t.Logf("archived as %v", id.Str()) beError := &errorBackend{Backend: repo.Backend()} checkRepo := repository.New(beError) - test.OK(t, checkRepo.SearchKey(test.TestPassword, 5)) + test.OK(t, checkRepo.SearchKey(context.TODO(), test.TestPassword, 5)) chkr := checker.New(checkRepo) - hints, errs := chkr.LoadIndex() + hints, errs := chkr.LoadIndex(context.TODO()) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -349,7 +351,7 @@ func BenchmarkChecker(t *testing.B) { repo := repository.TestOpenLocal(t, repodir) chkr := checker.New(repo) - hints, errs := chkr.LoadIndex() + hints, errs := chkr.LoadIndex(context.TODO()) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } diff --git a/src/restic/config_test.go b/src/restic/config_test.go index c5d2166e3..c287ae196 100644 --- a/src/restic/config_test.go +++ b/src/restic/config_test.go @@ -1,6 +1,7 @@ package restic_test import ( + "context" "restic" "testing" @@ -13,10 +14,10 @@ func (s saver) SaveJSONUnpacked(t restic.FileType, arg interface{}) (restic.ID, return s(t, arg) } -type loader func(restic.FileType, restic.ID, interface{}) error +type loader func(context.Context, restic.FileType, restic.ID, interface{}) error -func (l loader) LoadJSONUnpacked(t restic.FileType, id restic.ID, arg interface{}) error { - return l(t, id, arg) +func (l loader) LoadJSONUnpacked(ctx context.Context, t restic.FileType, id restic.ID, arg interface{}) error { + return l(ctx, t, id, arg) } func TestConfig(t *testing.T) { @@ -36,7 +37,7 @@ func TestConfig(t *testing.T) { _, err = saver(save).SaveJSONUnpacked(restic.ConfigFile, cfg1) - load := func(tpe restic.FileType, id restic.ID, arg interface{}) error { + load := func(ctx context.Context, tpe restic.FileType, id restic.ID, arg interface{}) error { Assert(t, tpe == restic.ConfigFile, "wrong backend type: got %v, wanted %v", tpe, restic.ConfigFile) @@ -46,7 +47,7 @@ func TestConfig(t *testing.T) { return nil } - cfg2, err := restic.LoadConfig(loader(load)) + cfg2, err := restic.LoadConfig(context.TODO(), loader(load)) OK(t, err) Assert(t, cfg1 == cfg2, diff --git a/src/restic/find_test.go b/src/restic/find_test.go index f4d7266ee..272472ffa 100644 --- a/src/restic/find_test.go +++ b/src/restic/find_test.go @@ -2,6 +2,7 @@ package restic_test import ( "bufio" + "context" "encoding/json" "flag" "fmt" @@ -92,7 +93,7 @@ func TestFindUsedBlobs(t *testing.T) { for i, sn := range snapshots { usedBlobs := restic.NewBlobSet() - err := restic.FindUsedBlobs(repo, *sn.Tree, usedBlobs, restic.NewBlobSet()) + err := restic.FindUsedBlobs(context.TODO(), repo, *sn.Tree, usedBlobs, restic.NewBlobSet()) if err != nil { t.Errorf("FindUsedBlobs returned error: %v", err) continue @@ -128,7 +129,7 @@ func BenchmarkFindUsedBlobs(b *testing.B) { for i := 0; i < b.N; i++ { seen := restic.NewBlobSet() blobs := restic.NewBlobSet() - err := restic.FindUsedBlobs(repo, *sn.Tree, blobs, seen) + err := restic.FindUsedBlobs(context.TODO(), repo, *sn.Tree, blobs, seen) if err != nil { b.Error(err) } diff --git a/src/restic/fuse/file_test.go b/src/restic/fuse/file_test.go index 9b2e0982d..dcb959fec 100644 --- a/src/restic/fuse/file_test.go +++ b/src/restic/fuse/file_test.go @@ -34,9 +34,7 @@ func testRead(t testing.TB, f *file, offset, length int, data []byte) { } func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) { - done := make(chan struct{}) - defer close(done) - for id := range repo.List(restic.SnapshotFile, done) { + for id := range repo.List(context.TODO(), restic.SnapshotFile) { if first.IsNull() { first = id } @@ -46,13 +44,13 @@ func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) { func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot { id := firstSnapshotID(t, repo) - sn, err := restic.LoadSnapshot(repo, id) + sn, err := restic.LoadSnapshot(context.TODO(), repo, id) OK(t, err) return sn } func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree { - tree, err := repo.LoadTree(id) + tree, err := repo.LoadTree(context.TODO(), id) OK(t, err) return tree } @@ -87,7 +85,7 @@ func TestFuseFile(t *testing.T) { filesize += uint64(size) buf := restic.NewBlobBuffer(int(size)) - n, err := repo.LoadBlob(restic.DataBlob, id, buf) + n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf) OK(t, err) if uint(n) != size { diff --git a/src/restic/index/index_test.go b/src/restic/index/index_test.go index 1984c2cb6..11d0cc08a 100644 --- a/src/restic/index/index_test.go +++ b/src/restic/index/index_test.go @@ -1,6 +1,7 @@ package index import ( + "context" "math/rand" "restic" "restic/checker" @@ -26,7 +27,7 @@ func createFilledRepo(t testing.TB, snapshots int, dup float32) (restic.Reposito } func validateIndex(t testing.TB, repo restic.Repository, idx *Index) { - for id := range repo.List(restic.DataFile, nil) { + for id := range repo.List(context.TODO(), restic.DataFile) { p, ok := idx.Packs[id] if !ok { t.Errorf("pack %v missing from index", id.Str()) @@ -42,7 +43,7 @@ func TestIndexNew(t *testing.T) { repo, cleanup := createFilledRepo(t, 3, 0) defer cleanup() - idx, err := New(repo, nil) + idx, err := New(context.TODO(), repo, nil) if err != nil { t.Fatalf("New() returned error %v", err) } @@ -58,7 +59,7 @@ func TestIndexLoad(t *testing.T) { repo, cleanup := createFilledRepo(t, 3, 0) defer cleanup() - loadIdx, err := Load(repo, nil) + loadIdx, err := Load(context.TODO(), repo, nil) if err != nil { t.Fatalf("Load() returned error %v", err) } @@ -69,7 +70,7 @@ func TestIndexLoad(t *testing.T) { validateIndex(t, repo, loadIdx) - newIdx, err := New(repo, nil) + newIdx, err := New(context.TODO(), repo, nil) if err != nil { t.Fatalf("New() returned error %v", err) } @@ -133,7 +134,7 @@ func BenchmarkIndexNew(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - idx, err := New(repo, nil) + idx, err := New(context.TODO(), repo, nil) if err != nil { b.Fatalf("New() returned error %v", err) @@ -150,7 +151,7 @@ func BenchmarkIndexSave(b *testing.B) { repo, cleanup := repository.TestRepository(b) defer cleanup() - idx, err := New(repo, nil) + idx, err := New(context.TODO(), repo, nil) test.OK(b, err) for i := 0; i < 8000; i++ { @@ -170,7 +171,7 @@ func BenchmarkIndexSave(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - id, err := idx.Save(repo, nil) + id, err := idx.Save(context.TODO(), repo, nil) if err != nil { b.Fatalf("New() returned error %v", err) } @@ -183,7 +184,7 @@ func TestIndexDuplicateBlobs(t *testing.T) { repo, cleanup := createFilledRepo(t, 3, 0.01) defer cleanup() - idx, err := New(repo, nil) + idx, err := New(context.TODO(), repo, nil) if err != nil { t.Fatal(err) } @@ -202,7 +203,7 @@ func TestIndexDuplicateBlobs(t *testing.T) { } func loadIndex(t testing.TB, repo restic.Repository) *Index { - idx, err := Load(repo, nil) + idx, err := Load(context.TODO(), repo, nil) if err != nil { t.Fatalf("Load() returned error %v", err) } @@ -225,7 +226,7 @@ func TestSave(t *testing.T) { t.Logf("save %d/%d packs in a new index\n", len(packs), len(idx.Packs)) - id, err := Save(repo, packs, idx.IndexIDs.List()) + id, err := Save(context.TODO(), repo, packs, idx.IndexIDs.List()) if err != nil { t.Fatalf("unable to save new index: %v", err) } @@ -235,7 +236,7 @@ func TestSave(t *testing.T) { for id := range idx.IndexIDs { t.Logf("remove index %v", id.Str()) h := restic.Handle{Type: restic.IndexFile, Name: id.String()} - err = repo.Backend().Remove(h) + err = repo.Backend().Remove(context.TODO(), h) if err != nil { t.Errorf("error removing index %v: %v", id, err) } @@ -267,7 +268,7 @@ func TestIndexSave(t *testing.T) { idx := loadIndex(t, repo) - id, err := idx.Save(repo, idx.IndexIDs.List()) + id, err := idx.Save(context.TODO(), repo, idx.IndexIDs.List()) if err != nil { t.Fatalf("unable to save new index: %v", err) } @@ -277,7 +278,7 @@ func TestIndexSave(t *testing.T) { for id := range idx.IndexIDs { t.Logf("remove index %v", id.Str()) h := restic.Handle{Type: restic.IndexFile, Name: id.String()} - err = repo.Backend().Remove(h) + err = repo.Backend().Remove(context.TODO(), h) if err != nil { t.Errorf("error removing index %v: %v", id, err) } @@ -287,7 +288,7 @@ func TestIndexSave(t *testing.T) { t.Logf("load new index with %d packs", len(idx2.Packs)) checker := checker.New(repo) - hints, errs := checker.LoadIndex() + hints, errs := checker.LoadIndex(context.TODO()) for _, h := range hints { t.Logf("hint: %v\n", h) } @@ -301,15 +302,12 @@ func TestIndexAddRemovePack(t *testing.T) { repo, cleanup := createFilledRepo(t, 3, 0) defer cleanup() - idx, err := Load(repo, nil) + idx, err := Load(context.TODO(), repo, nil) if err != nil { t.Fatalf("Load() returned error %v", err) } - done := make(chan struct{}) - defer close(done) - - packID := <-repo.List(restic.DataFile, done) + packID := <-repo.List(context.TODO(), restic.DataFile) t.Logf("selected pack %v", packID.Str()) @@ -367,7 +365,7 @@ func TestIndexLoadDocReference(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() - id, err := repo.SaveUnpacked(restic.IndexFile, docExample) + id, err := repo.SaveUnpacked(context.TODO(), restic.IndexFile, docExample) if err != nil { t.Fatalf("SaveUnpacked() returned error %v", err) } diff --git a/src/restic/lock_test.go b/src/restic/lock_test.go index b8288d6cd..d5fd179a1 100644 --- a/src/restic/lock_test.go +++ b/src/restic/lock_test.go @@ -1,6 +1,7 @@ package restic_test import ( + "context" "os" "testing" "time" @@ -14,7 +15,7 @@ func TestLock(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() - lock, err := restic.NewLock(repo) + lock, err := restic.NewLock(context.TODO(), repo) OK(t, err) OK(t, lock.Unlock()) @@ -24,7 +25,7 @@ func TestDoubleUnlock(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() - lock, err := restic.NewLock(repo) + lock, err := restic.NewLock(context.TODO(), repo) OK(t, err) OK(t, lock.Unlock()) @@ -38,10 +39,10 @@ func TestMultipleLock(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() - lock1, err := restic.NewLock(repo) + lock1, err := restic.NewLock(context.TODO(), repo) OK(t, err) - lock2, err := restic.NewLock(repo) + lock2, err := restic.NewLock(context.TODO(), repo) OK(t, err) OK(t, lock1.Unlock()) @@ -52,7 +53,7 @@ func TestLockExclusive(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() - elock, err := restic.NewExclusiveLock(repo) + elock, err := restic.NewExclusiveLock(context.TODO(), repo) OK(t, err) OK(t, elock.Unlock()) } @@ -61,10 +62,10 @@ func TestLockOnExclusiveLockedRepo(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() - elock, err := restic.NewExclusiveLock(repo) + elock, err := restic.NewExclusiveLock(context.TODO(), repo) OK(t, err) - lock, err := restic.NewLock(repo) + lock, err := restic.NewLock(context.TODO(), repo) Assert(t, err != nil, "create normal lock with exclusively locked repo didn't return an error") Assert(t, restic.IsAlreadyLocked(err), @@ -78,10 +79,10 @@ func TestExclusiveLockOnLockedRepo(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() - elock, err := restic.NewLock(repo) + elock, err := restic.NewLock(context.TODO(), repo) OK(t, err) - lock, err := restic.NewExclusiveLock(repo) + lock, err := restic.NewExclusiveLock(context.TODO(), repo) Assert(t, err != nil, "create normal lock with exclusively locked repo didn't return an error") Assert(t, restic.IsAlreadyLocked(err), @@ -98,12 +99,12 @@ func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, er } newLock := &restic.Lock{Time: t, PID: pid, Hostname: hostname} - return repo.SaveJSONUnpacked(restic.LockFile, &newLock) + return repo.SaveJSONUnpacked(context.TODO(), restic.LockFile, &newLock) } func removeLock(repo restic.Repository, id restic.ID) error { h := restic.Handle{Type: restic.LockFile, Name: id.String()} - return repo.Backend().Remove(h) + return repo.Backend().Remove(context.TODO(), h) } var staleLockTests = []struct { @@ -164,7 +165,7 @@ func TestLockStale(t *testing.T) { func lockExists(repo restic.Repository, t testing.TB, id restic.ID) bool { h := restic.Handle{Type: restic.LockFile, Name: id.String()} - exists, err := repo.Backend().Test(h) + exists, err := repo.Backend().Test(context.TODO(), h) OK(t, err) return exists @@ -183,7 +184,7 @@ func TestLockWithStaleLock(t *testing.T) { id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500000) OK(t, err) - OK(t, restic.RemoveStaleLocks(repo)) + OK(t, restic.RemoveStaleLocks(context.TODO(), repo)) Assert(t, lockExists(repo, t, id1) == false, "stale lock still exists after RemoveStaleLocks was called") @@ -208,7 +209,7 @@ func TestRemoveAllLocks(t *testing.T) { id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500000) OK(t, err) - OK(t, restic.RemoveAllLocks(repo)) + OK(t, restic.RemoveAllLocks(context.TODO(), repo)) Assert(t, lockExists(repo, t, id1) == false, "lock still exists after RemoveAllLocks was called") @@ -222,21 +223,21 @@ func TestLockRefresh(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() - lock, err := restic.NewLock(repo) + lock, err := restic.NewLock(context.TODO(), repo) OK(t, err) var lockID *restic.ID - for id := range repo.List(restic.LockFile, nil) { + for id := range repo.List(context.TODO(), restic.LockFile) { if lockID != nil { t.Error("more than one lock found") } lockID = &id } - OK(t, lock.Refresh()) + OK(t, lock.Refresh(context.TODO())) var lockID2 *restic.ID - for id := range repo.List(restic.LockFile, nil) { + for id := range repo.List(context.TODO(), restic.LockFile) { if lockID2 != nil { t.Error("more than one lock found") } diff --git a/src/restic/node_test.go b/src/restic/node_test.go index f0c16b368..f357adab2 100644 --- a/src/restic/node_test.go +++ b/src/restic/node_test.go @@ -1,6 +1,7 @@ package restic_test import ( + "context" "io/ioutil" "os" "path/filepath" @@ -180,7 +181,7 @@ func TestNodeRestoreAt(t *testing.T) { for _, test := range nodeTests { nodePath := filepath.Join(tempdir, test.Name) - OK(t, test.CreateAt(nodePath, nil, idx)) + OK(t, test.CreateAt(context.TODO(), nodePath, nil, idx)) if test.Type == "symlink" && runtime.GOOS == "windows" { continue diff --git a/src/restic/pipe/pipe_test.go b/src/restic/pipe/pipe_test.go index dd2c5e02d..197a1c428 100644 --- a/src/restic/pipe/pipe_test.go +++ b/src/restic/pipe/pipe_test.go @@ -1,6 +1,7 @@ package pipe_test import ( + "context" "io/ioutil" "os" "path/filepath" @@ -127,7 +128,7 @@ func TestPipelineWalkerWithSplit(t *testing.T) { }() resCh := make(chan pipe.Result, 1) - pipe.Walk([]string{TestWalkerPath}, acceptAll, done, jobs, resCh) + pipe.Walk(context.TODO(), []string{TestWalkerPath}, acceptAll, jobs, resCh) // wait for all workers to terminate wg.Wait() @@ -146,6 +147,9 @@ func TestPipelineWalker(t *testing.T) { t.Skipf("walkerpath not set, skipping TestPipelineWalker") } + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + var err error if !filepath.IsAbs(TestWalkerPath) { TestWalkerPath, err = filepath.Abs(TestWalkerPath) @@ -164,7 +168,7 @@ func TestPipelineWalker(t *testing.T) { after := stats{} m := sync.Mutex{} - worker := func(wg *sync.WaitGroup, done <-chan struct{}, jobs <-chan pipe.Job) { + worker := func(ctx context.Context, wg *sync.WaitGroup, jobs <-chan pipe.Job) { defer wg.Done() for { select { @@ -195,7 +199,7 @@ func TestPipelineWalker(t *testing.T) { j.Result() <- true } - case <-done: + case <-ctx.Done(): // pipeline was cancelled return } @@ -203,16 +207,15 @@ func TestPipelineWalker(t *testing.T) { } var wg sync.WaitGroup - done := make(chan struct{}) jobs := make(chan pipe.Job) for i := 0; i < maxWorkers; i++ { wg.Add(1) - go worker(&wg, done, jobs) + go worker(ctx, &wg, jobs) } resCh := make(chan pipe.Result, 1) - pipe.Walk([]string{TestWalkerPath}, acceptAll, done, jobs, resCh) + pipe.Walk(ctx, []string{TestWalkerPath}, acceptAll, jobs, resCh) // wait for all workers to terminate wg.Wait() @@ -286,11 +289,12 @@ func TestPipeWalkerError(t *testing.T) { OK(t, os.RemoveAll(testdir)) }) - done := make(chan struct{}) + ctx, cancel := context.WithCancel(context.TODO()) + ch := make(chan pipe.Job) resCh := make(chan pipe.Result, 1) - go pipe.Walk([]string{dir}, acceptAll, done, ch, resCh) + go pipe.Walk(ctx, []string{dir}, acceptAll, ch, resCh) i := 0 for job := range ch { @@ -321,7 +325,7 @@ func TestPipeWalkerError(t *testing.T) { t.Errorf("expected %d jobs, got %d", len(testjobs), i) } - close(done) + cancel() Assert(t, ranHook, "hook did not run") OK(t, os.RemoveAll(dir)) @@ -335,7 +339,7 @@ func BenchmarkPipelineWalker(b *testing.B) { var max time.Duration m := sync.Mutex{} - fileWorker := func(wg *sync.WaitGroup, done <-chan struct{}, ch <-chan pipe.Entry) { + fileWorker := func(ctx context.Context, wg *sync.WaitGroup, ch <-chan pipe.Entry) { defer wg.Done() for { select { @@ -349,14 +353,14 @@ func BenchmarkPipelineWalker(b *testing.B) { //time.Sleep(10 * time.Millisecond) e.Result() <- true - case <-done: + case <-ctx.Done(): // pipeline was cancelled return } } } - dirWorker := func(wg *sync.WaitGroup, done <-chan struct{}, ch <-chan pipe.Dir) { + dirWorker := func(ctx context.Context, wg *sync.WaitGroup, ch <-chan pipe.Dir) { defer wg.Done() for { select { @@ -381,16 +385,18 @@ func BenchmarkPipelineWalker(b *testing.B) { m.Unlock() dir.Result() <- true - case <-done: + case <-ctx.Done(): // pipeline was cancelled return } } } + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + for i := 0; i < b.N; i++ { max = 0 - done := make(chan struct{}) entCh := make(chan pipe.Entry, 200) dirCh := make(chan pipe.Dir, 200) @@ -398,8 +404,8 @@ func BenchmarkPipelineWalker(b *testing.B) { b.Logf("starting %d workers", maxWorkers) for i := 0; i < maxWorkers; i++ { wg.Add(2) - go dirWorker(&wg, done, dirCh) - go fileWorker(&wg, done, entCh) + go dirWorker(ctx, &wg, dirCh) + go fileWorker(ctx, &wg, entCh) } jobs := make(chan pipe.Job, 200) @@ -412,7 +418,7 @@ func BenchmarkPipelineWalker(b *testing.B) { }() resCh := make(chan pipe.Result, 1) - pipe.Walk([]string{TestWalkerPath}, acceptAll, done, jobs, resCh) + pipe.Walk(ctx, []string{TestWalkerPath}, acceptAll, jobs, resCh) // wait for all workers to terminate wg.Wait() @@ -429,6 +435,9 @@ func TestPipelineWalkerMultiple(t *testing.T) { t.Skipf("walkerpath not set, skipping TestPipelineWalker") } + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + paths, err := filepath.Glob(filepath.Join(TestWalkerPath, "*")) OK(t, err) @@ -441,7 +450,7 @@ func TestPipelineWalkerMultiple(t *testing.T) { after := stats{} m := sync.Mutex{} - worker := func(wg *sync.WaitGroup, done <-chan struct{}, jobs <-chan pipe.Job) { + worker := func(ctx context.Context, wg *sync.WaitGroup, jobs <-chan pipe.Job) { defer wg.Done() for { select { @@ -472,7 +481,7 @@ func TestPipelineWalkerMultiple(t *testing.T) { j.Result() <- true } - case <-done: + case <-ctx.Done(): // pipeline was cancelled return } @@ -480,16 +489,15 @@ func TestPipelineWalkerMultiple(t *testing.T) { } var wg sync.WaitGroup - done := make(chan struct{}) jobs := make(chan pipe.Job) for i := 0; i < maxWorkers; i++ { wg.Add(1) - go worker(&wg, done, jobs) + go worker(ctx, &wg, jobs) } resCh := make(chan pipe.Result, 1) - pipe.Walk(paths, acceptAll, done, jobs, resCh) + pipe.Walk(ctx, paths, acceptAll, jobs, resCh) // wait for all workers to terminate wg.Wait() @@ -547,9 +555,6 @@ func testPipeWalkerRootWithPath(path string, t *testing.T) { t.Logf("paths in %v (pattern %q) expanded to %v items", path, pattern, len(rootPaths)) - done := make(chan struct{}) - defer close(done) - jobCh := make(chan pipe.Job) var jobs []pipe.Job @@ -571,7 +576,7 @@ func testPipeWalkerRootWithPath(path string, t *testing.T) { } resCh := make(chan pipe.Result, 1) - pipe.Walk([]string{path}, filter, done, jobCh, resCh) + pipe.Walk(context.TODO(), []string{path}, filter, jobCh, resCh) wg.Wait() diff --git a/src/restic/repository/packer_manager_test.go b/src/restic/repository/packer_manager_test.go index 2ca44de0b..3b49655bd 100644 --- a/src/restic/repository/packer_manager_test.go +++ b/src/restic/repository/packer_manager_test.go @@ -1,6 +1,7 @@ package repository import ( + "context" "io" "math/rand" "os" @@ -52,7 +53,7 @@ func saveFile(t testing.TB, be Saver, f *os.File, id restic.ID) { h := restic.Handle{Type: restic.DataFile, Name: id.String()} t.Logf("save file %v", h) - if err := be.Save(h, f); err != nil { + if err := be.Save(context.TODO(), h, f); err != nil { t.Fatal(err) } @@ -145,7 +146,7 @@ func BenchmarkPackerManager(t *testing.B) { rnd := newRandReader(rand.NewSource(23)) be := &mock.Backend{ - SaveFn: func(restic.Handle, io.Reader) error { return nil }, + SaveFn: func(context.Context, restic.Handle, io.Reader) error { return nil }, } blobBuf := make([]byte, maxBlobSize) diff --git a/src/restic/repository/parallel_test.go b/src/restic/repository/parallel_test.go index cfa384a01..aa15e79e7 100644 --- a/src/restic/repository/parallel_test.go +++ b/src/restic/repository/parallel_test.go @@ -1,6 +1,7 @@ package repository_test import ( + "context" "math/rand" "restic" "testing" @@ -73,7 +74,7 @@ var lister = testIDs{ "34dd044c228727f2226a0c9c06a3e5ceb5e30e31cb7854f8fa1cde846b395a58", } -func (tests testIDs) List(t restic.FileType, done <-chan struct{}) <-chan string { +func (tests testIDs) List(ctx context.Context, t restic.FileType) <-chan string { ch := make(chan string) go func() { @@ -83,7 +84,7 @@ func (tests testIDs) List(t restic.FileType, done <-chan struct{}) <-chan string for _, id := range tests { select { case ch <- id: - case <-done: + case <-ctx.Done(): return } } @@ -94,13 +95,13 @@ func (tests testIDs) List(t restic.FileType, done <-chan struct{}) <-chan string } func TestFilesInParallel(t *testing.T) { - f := func(id string, done <-chan struct{}) error { + f := func(ctx context.Context, id string) error { time.Sleep(1 * time.Millisecond) return nil } for n := uint(1); n < 5; n++ { - err := repository.FilesInParallel(lister, restic.DataFile, n*100, f) + err := repository.FilesInParallel(context.TODO(), lister, restic.DataFile, n*100, f) OK(t, err) } } @@ -109,7 +110,7 @@ var errTest = errors.New("test error") func TestFilesInParallelWithError(t *testing.T) { - f := func(id string, done <-chan struct{}) error { + f := func(ctx context.Context, id string) error { time.Sleep(1 * time.Millisecond) if rand.Float32() < 0.01 { @@ -120,7 +121,7 @@ func TestFilesInParallelWithError(t *testing.T) { } for n := uint(1); n < 5; n++ { - err := repository.FilesInParallel(lister, restic.DataFile, n*100, f) + err := repository.FilesInParallel(context.TODO(), lister, restic.DataFile, n*100, f) Equals(t, errTest, err) } } diff --git a/src/restic/repository/repack_test.go b/src/restic/repository/repack_test.go index 622b3ba52..d339cf2b8 100644 --- a/src/restic/repository/repack_test.go +++ b/src/restic/repository/repack_test.go @@ -1,6 +1,7 @@ package repository_test import ( + "context" "io" "math/rand" "restic" @@ -47,7 +48,7 @@ func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData fl continue } - _, err := repo.SaveBlob(tpe, buf, id) + _, err := repo.SaveBlob(context.TODO(), tpe, buf, id) if err != nil { t.Fatalf("SaveFrom() error %v", err) } @@ -67,16 +68,13 @@ func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData fl // selectBlobs splits the list of all blobs randomly into two lists. A blob // will be contained in the firstone ith probability p. func selectBlobs(t *testing.T, repo restic.Repository, p float32) (list1, list2 restic.BlobSet) { - done := make(chan struct{}) - defer close(done) - list1 = restic.NewBlobSet() list2 = restic.NewBlobSet() blobs := restic.NewBlobSet() - for id := range repo.List(restic.DataFile, done) { - entries, _, err := repo.ListPack(id) + for id := range repo.List(context.TODO(), restic.DataFile) { + entries, _, err := repo.ListPack(context.TODO(), id) if err != nil { t.Fatalf("error listing pack %v: %v", id, err) } @@ -102,11 +100,8 @@ func selectBlobs(t *testing.T, repo restic.Repository, p float32) (list1, list2 } func listPacks(t *testing.T, repo restic.Repository) restic.IDSet { - done := make(chan struct{}) - defer close(done) - list := restic.NewIDSet() - for id := range repo.List(restic.DataFile, done) { + for id := range repo.List(context.TODO(), restic.DataFile) { list.Insert(id) } @@ -132,35 +127,36 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe } func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs restic.BlobSet) { - err := repository.Repack(repo, packs, blobs, nil) + err := repository.Repack(context.TODO(), repo, packs, blobs, nil) if err != nil { t.Fatal(err) } } func saveIndex(t *testing.T, repo restic.Repository) { - if err := repo.SaveIndex(); err != nil { + if err := repo.SaveIndex(context.TODO()); err != nil { t.Fatalf("repo.SaveIndex() %v", err) } } func rebuildIndex(t *testing.T, repo restic.Repository) { - idx, err := index.New(repo, nil) + idx, err := index.New(context.TODO(), repo, nil) if err != nil { t.Fatal(err) } - for id := range repo.List(restic.IndexFile, nil) { - err = repo.Backend().Remove(restic.Handle{ + for id := range repo.List(context.TODO(), restic.IndexFile) { + h := restic.Handle{ Type: restic.IndexFile, Name: id.String(), - }) + } + err = repo.Backend().Remove(context.TODO(), h) if err != nil { t.Fatal(err) } } - _, err = idx.Save(repo, nil) + _, err = idx.Save(context.TODO(), repo, nil) if err != nil { t.Fatal(err) } @@ -168,7 +164,7 @@ func rebuildIndex(t *testing.T, repo restic.Repository) { func reloadIndex(t *testing.T, repo restic.Repository) { repo.SetIndex(repository.NewMasterIndex()) - if err := repo.LoadIndex(); err != nil { + if err := repo.LoadIndex(context.TODO()); err != nil { t.Fatalf("error loading new index: %v", err) } } diff --git a/src/restic/repository/repository_test.go b/src/restic/repository/repository_test.go index 6ee99f265..cc9e0ab64 100644 --- a/src/restic/repository/repository_test.go +++ b/src/restic/repository/repository_test.go @@ -2,6 +2,7 @@ package repository_test import ( "bytes" + "context" "crypto/sha256" "io" "math/rand" @@ -31,7 +32,7 @@ func TestSave(t *testing.T) { id := restic.Hash(data) // save - sid, err := repo.SaveBlob(restic.DataBlob, data, restic.ID{}) + sid, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, restic.ID{}) OK(t, err) Equals(t, id, sid) @@ -41,7 +42,7 @@ func TestSave(t *testing.T) { // read back buf := restic.NewBlobBuffer(size) - n, err := repo.LoadBlob(restic.DataBlob, id, buf) + n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf) OK(t, err) Equals(t, len(buf), n) @@ -67,7 +68,7 @@ func TestSaveFrom(t *testing.T) { id := restic.Hash(data) // save - id2, err := repo.SaveBlob(restic.DataBlob, data, id) + id2, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, id) OK(t, err) Equals(t, id, id2) @@ -75,7 +76,7 @@ func TestSaveFrom(t *testing.T) { // read back buf := restic.NewBlobBuffer(size) - n, err := repo.LoadBlob(restic.DataBlob, id, buf) + n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf) OK(t, err) Equals(t, len(buf), n) @@ -106,7 +107,7 @@ func BenchmarkSaveAndEncrypt(t *testing.B) { for i := 0; i < t.N; i++ { // save - _, err = repo.SaveBlob(restic.DataBlob, data, id) + _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, data, id) OK(t, err) } } @@ -123,7 +124,7 @@ func TestLoadTree(t *testing.T) { sn := archiver.TestSnapshot(t, repo, BenchArchiveDirectory, nil) OK(t, repo.Flush()) - _, err := repo.LoadTree(*sn.Tree) + _, err := repo.LoadTree(context.TODO(), *sn.Tree) OK(t, err) } @@ -142,7 +143,7 @@ func BenchmarkLoadTree(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - _, err := repo.LoadTree(*sn.Tree) + _, err := repo.LoadTree(context.TODO(), *sn.Tree) OK(t, err) } } @@ -156,14 +157,14 @@ func TestLoadBlob(t *testing.T) { _, err := io.ReadFull(rnd, buf) OK(t, err) - id, err := repo.SaveBlob(restic.DataBlob, buf, restic.ID{}) + id, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}) OK(t, err) OK(t, repo.Flush()) // first, test with buffers that are too small for _, testlength := range []int{length - 20, length, restic.CiphertextLength(length) - 1} { buf = make([]byte, 0, testlength) - n, err := repo.LoadBlob(restic.DataBlob, id, buf) + n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf) if err == nil { t.Errorf("LoadBlob() did not return an error for a buffer that is too small to hold the blob") continue @@ -179,7 +180,7 @@ func TestLoadBlob(t *testing.T) { base := restic.CiphertextLength(length) for _, testlength := range []int{base, base + 7, base + 15, base + 1000} { buf = make([]byte, 0, testlength) - n, err := repo.LoadBlob(restic.DataBlob, id, buf) + n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf) if err != nil { t.Errorf("LoadBlob() returned an error for buffer size %v: %v", testlength, err) continue @@ -201,7 +202,7 @@ func BenchmarkLoadBlob(b *testing.B) { _, err := io.ReadFull(rnd, buf) OK(b, err) - id, err := repo.SaveBlob(restic.DataBlob, buf, restic.ID{}) + id, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}) OK(b, err) OK(b, repo.Flush()) @@ -209,7 +210,7 @@ func BenchmarkLoadBlob(b *testing.B) { b.SetBytes(int64(length)) for i := 0; i < b.N; i++ { - n, err := repo.LoadBlob(restic.DataBlob, id, buf) + n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf) OK(b, err) if n != length { b.Errorf("wanted %d bytes, got %d", length, n) @@ -233,7 +234,7 @@ func BenchmarkLoadAndDecrypt(b *testing.B) { dataID := restic.Hash(buf) - storageID, err := repo.SaveUnpacked(restic.DataFile, buf) + storageID, err := repo.SaveUnpacked(context.TODO(), restic.DataFile, buf) OK(b, err) // OK(b, repo.Flush()) @@ -241,7 +242,7 @@ func BenchmarkLoadAndDecrypt(b *testing.B) { b.SetBytes(int64(length)) for i := 0; i < b.N; i++ { - data, err := repo.LoadAndDecrypt(restic.DataFile, storageID) + data, err := repo.LoadAndDecrypt(context.TODO(), restic.DataFile, storageID) OK(b, err) if len(data) != length { b.Errorf("wanted %d bytes, got %d", length, len(data)) @@ -267,13 +268,13 @@ func TestLoadJSONUnpacked(t *testing.T) { sn.Hostname = "foobar" sn.Username = "test!" - id, err := repo.SaveJSONUnpacked(restic.SnapshotFile, &sn) + id, err := repo.SaveJSONUnpacked(context.TODO(), restic.SnapshotFile, &sn) OK(t, err) var sn2 restic.Snapshot // restore - err = repo.LoadJSONUnpacked(restic.SnapshotFile, id, &sn2) + err = repo.LoadJSONUnpacked(context.TODO(), restic.SnapshotFile, id, &sn2) OK(t, err) Equals(t, sn.Hostname, sn2.Hostname) @@ -287,7 +288,7 @@ func TestRepositoryLoadIndex(t *testing.T) { defer cleanup() repo := repository.TestOpenLocal(t, repodir) - OK(t, repo.LoadIndex()) + OK(t, repo.LoadIndex(context.TODO())) } func BenchmarkLoadIndex(b *testing.B) { @@ -310,18 +311,18 @@ func BenchmarkLoadIndex(b *testing.B) { }) } - id, err := repository.SaveIndex(repo, idx) + id, err := repository.SaveIndex(context.TODO(), repo, idx) OK(b, err) b.Logf("index saved as %v (%v entries)", id.Str(), idx.Count(restic.DataBlob)) - fi, err := repo.Backend().Stat(restic.Handle{Type: restic.IndexFile, Name: id.String()}) + fi, err := repo.Backend().Stat(context.TODO(), restic.Handle{Type: restic.IndexFile, Name: id.String()}) OK(b, err) b.Logf("filesize is %v", fi.Size) b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := repository.LoadIndex(repo, id) + _, err := repository.LoadIndex(context.TODO(), repo, id) OK(b, err) } } @@ -335,7 +336,7 @@ func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax _, err := io.ReadFull(rnd, buf) OK(t, err) - _, err = repo.SaveBlob(restic.DataBlob, buf, restic.ID{}) + _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}) OK(t, err) } } @@ -354,7 +355,7 @@ func TestRepositoryIncrementalIndex(t *testing.T) { OK(t, repo.Flush()) } - OK(t, repo.SaveFullIndex()) + OK(t, repo.SaveFullIndex(context.TODO())) } // add another 5 packs @@ -364,12 +365,12 @@ func TestRepositoryIncrementalIndex(t *testing.T) { } // save final index - OK(t, repo.SaveIndex()) + OK(t, repo.SaveIndex(context.TODO())) packEntries := make(map[restic.ID]map[restic.ID]struct{}) - for id := range repo.List(restic.IndexFile, nil) { - idx, err := repository.LoadIndex(repo, id) + for id := range repo.List(context.TODO(), restic.IndexFile) { + idx, err := repository.LoadIndex(context.TODO(), repo, id) OK(t, err) for pb := range idx.Each(nil) { diff --git a/src/restic/testing_test.go b/src/restic/testing_test.go index 86b18a001..29b426623 100644 --- a/src/restic/testing_test.go +++ b/src/restic/testing_test.go @@ -1,6 +1,7 @@ package restic_test import ( + "context" "restic" "restic/checker" "restic/repository" @@ -23,7 +24,7 @@ func TestCreateSnapshot(t *testing.T) { restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0) } - snapshots, err := restic.LoadAllSnapshots(repo) + snapshots, err := restic.LoadAllSnapshots(context.TODO(), repo) if err != nil { t.Fatal(err) } diff --git a/src/restic/tree_test.go b/src/restic/tree_test.go index 0bf7cfddc..dbdd20d20 100644 --- a/src/restic/tree_test.go +++ b/src/restic/tree_test.go @@ -1,6 +1,7 @@ package restic_test import ( + "context" "encoding/json" "io/ioutil" "os" @@ -98,14 +99,14 @@ func TestLoadTree(t *testing.T) { // save tree tree := restic.NewTree() - id, err := repo.SaveTree(tree) + id, err := repo.SaveTree(context.TODO(), tree) OK(t, err) // save packs OK(t, repo.Flush()) // load tree again - tree2, err := repo.LoadTree(id) + tree2, err := repo.LoadTree(context.TODO(), id) OK(t, err) Assert(t, tree.Equals(tree2), diff --git a/src/restic/walk/walk_test.go b/src/restic/walk/walk_test.go index 6a824827d..bdb5cf459 100644 --- a/src/restic/walk/walk_test.go +++ b/src/restic/walk/walk_test.go @@ -1,6 +1,7 @@ package walk_test import ( + "context" "os" "path/filepath" "strings" @@ -24,17 +25,15 @@ func TestWalkTree(t *testing.T) { // archive a few files arch := archiver.New(repo) - sn, _, err := arch.Snapshot(nil, dirs, nil, "localhost", nil) + sn, _, err := arch.Snapshot(context.TODO(), nil, dirs, nil, "localhost", nil) OK(t, err) // flush repo, write all packs OK(t, repo.Flush()) - done := make(chan struct{}) - // start tree walker treeJobs := make(chan walk.TreeJob) - go walk.Tree(repo, *sn.Tree, done, treeJobs) + go walk.Tree(context.TODO(), repo, *sn.Tree, treeJobs) // start filesystem walker fsJobs := make(chan pipe.Job) @@ -43,7 +42,7 @@ func TestWalkTree(t *testing.T) { f := func(string, os.FileInfo) bool { return true } - go pipe.Walk(dirs, f, done, fsJobs, resCh) + go pipe.Walk(context.TODO(), dirs, f, fsJobs, resCh) for { // receive fs job @@ -95,9 +94,9 @@ type delayRepo struct { delay time.Duration } -func (d delayRepo) LoadTree(id restic.ID) (*restic.Tree, error) { +func (d delayRepo) LoadTree(ctx context.Context, id restic.ID) (*restic.Tree, error) { time.Sleep(d.delay) - return d.repo.LoadTree(id) + return d.repo.LoadTree(ctx, id) } var repoFixture = filepath.Join("testdata", "walktree-test-repo.tar.gz") @@ -1345,7 +1344,7 @@ func TestDelayedWalkTree(t *testing.T) { defer cleanup() repo := repository.TestOpenLocal(t, repodir) - OK(t, repo.LoadIndex()) + OK(t, repo.LoadIndex(context.TODO())) root, err := restic.ParseID("937a2f64f736c64ee700c6ab06f840c68c94799c288146a0e81e07f4c94254da") OK(t, err) @@ -1354,7 +1353,7 @@ func TestDelayedWalkTree(t *testing.T) { // start tree walker treeJobs := make(chan walk.TreeJob) - go walk.Tree(dr, root, nil, treeJobs) + go walk.Tree(context.TODO(), dr, root, treeJobs) i := 0 for job := range treeJobs { @@ -1375,7 +1374,7 @@ func BenchmarkDelayedWalkTree(t *testing.B) { defer cleanup() repo := repository.TestOpenLocal(t, repodir) - OK(t, repo.LoadIndex()) + OK(t, repo.LoadIndex(context.TODO())) root, err := restic.ParseID("937a2f64f736c64ee700c6ab06f840c68c94799c288146a0e81e07f4c94254da") OK(t, err) @@ -1387,7 +1386,7 @@ func BenchmarkDelayedWalkTree(t *testing.B) { for i := 0; i < t.N; i++ { // start tree walker treeJobs := make(chan walk.TreeJob) - go walk.Tree(dr, root, nil, treeJobs) + go walk.Tree(context.TODO(), dr, root, treeJobs) for range treeJobs { } diff --git a/src/restic/worker/pool.go b/src/restic/worker/pool.go index 2d98a0d70..870548378 100644 --- a/src/restic/worker/pool.go +++ b/src/restic/worker/pool.go @@ -16,7 +16,6 @@ type Func func(ctx context.Context, job Job) (result interface{}, err error) // Pool implements a worker pool. type Pool struct { f Func - ctx context.Context jobCh <-chan Job resCh chan<- Job @@ -38,7 +37,7 @@ func New(ctx context.Context, n int, f Func, jobChan <-chan Job, resultChan chan } for i := 0; i < n; i++ { - go p.runWorker(i) + go p.runWorker(ctx, i) } go p.waitForExit() @@ -59,7 +58,7 @@ func (p *Pool) waitForExit() { } // runWorker runs a worker function. -func (p *Pool) runWorker(numWorker int) { +func (p *Pool) runWorker(ctx context.Context, numWorker int) { defer func() { p.workersExit <- struct{}{} }() @@ -76,7 +75,7 @@ func (p *Pool) runWorker(numWorker int) { for { select { - case <-p.ctx.Done(): + case <-ctx.Done(): return case job, ok = <-inCh: @@ -84,7 +83,7 @@ func (p *Pool) runWorker(numWorker int) { return } - job.Result, job.Error = p.f(p.ctx, job) + job.Result, job.Error = p.f(ctx, job) inCh = nil outCh = p.resCh