From 3695ba5882a6d20b73611b9aa3d47c2a520b0508 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 31 Aug 2016 23:07:50 +0200 Subject: [PATCH] Tests pass for restic/ --- .../archiver/archiver_duplication_test.go | 15 ++-- src/restic/archiver/archiver_int_test.go | 1 + src/restic/archiver/archiver_test.go | 53 ++---------- src/restic/checker/checker.go | 14 ++-- src/restic/checker/checker_test.go | 8 +- src/restic/index/index.go | 80 +++++++++--------- src/restic/index/index_test.go | 12 ++- src/restic/pack/pack_test.go | 25 +++--- src/restic/repository.go | 6 ++ src/restic/repository/index.go | 2 +- src/restic/repository/index_test.go | 82 ++++++++++--------- src/restic/repository/packer_manager_test.go | 5 +- src/restic/repository/repack_test.go | 27 +++--- src/restic/repository/repository_test.go | 21 +++-- 14 files changed, 156 insertions(+), 195 deletions(-) diff --git a/src/restic/archiver/archiver_duplication_test.go b/src/restic/archiver/archiver_duplication_test.go index 37bed0830..9f0867d5e 100644 --- a/src/restic/archiver/archiver_duplication_test.go +++ b/src/restic/archiver/archiver_duplication_test.go @@ -11,9 +11,8 @@ import ( "github.com/pkg/errors" "restic" - "restic/backend" + "restic/archiver" "restic/mock" - "restic/pack" "restic/repository" ) @@ -21,14 +20,14 @@ const parallelSaves = 50 const testSaveIndexTime = 100 * time.Millisecond const testTimeout = 2 * time.Second -var DupID backend.ID +var DupID restic.ID -func randomID() backend.ID { +func randomID() restic.ID { if mrand.Float32() < 0.5 { return DupID } - id := backend.ID{} + id := restic.ID{} _, err := io.ReadFull(rand.Reader, id[:]) if err != nil { panic(err) @@ -52,8 +51,8 @@ func forgetfulBackend() restic.Backend { return nil } - be.StatFn = func(h restic.Handle) (restic.BlobInfo, error) { - return restic.BlobInfo{}, errors.New("not found") + be.StatFn = func(h restic.Handle) (restic.FileInfo, error) { + return restic.FileInfo{}, errors.New("not found") } be.RemoveFn = func(t restic.FileType, name string) error { @@ -86,7 +85,7 @@ func testArchiverDuplication(t *testing.T) { t.Fatal(err) } - arch := restic.NewArchiver(repo) + arch := archiver.New(repo) wg := &sync.WaitGroup{} done := make(chan struct{}) diff --git a/src/restic/archiver/archiver_int_test.go b/src/restic/archiver/archiver_int_test.go index 28305b38f..3b5309a20 100644 --- a/src/restic/archiver/archiver_int_test.go +++ b/src/restic/archiver/archiver_int_test.go @@ -2,6 +2,7 @@ package archiver import ( "os" + "restic" "testing" "restic/pipe" diff --git a/src/restic/archiver/archiver_test.go b/src/restic/archiver/archiver_test.go index 57903931a..176a01802 100644 --- a/src/restic/archiver/archiver_test.go +++ b/src/restic/archiver/archiver_test.go @@ -7,10 +7,9 @@ import ( "time" "restic" - "restic/backend" + "restic/archiver" "restic/checker" "restic/crypto" - "restic/pack" . "restic/test" "github.com/pkg/errors" @@ -101,7 +100,7 @@ func archiveDirectory(b testing.TB) { repo := SetupRepo() defer TeardownRepo(repo) - arch := restic.NewArchiver(repo) + arch := archiver.New(repo) _, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil) OK(b, err) @@ -191,48 +190,6 @@ func TestArchiveDedup(t *testing.T) { archiveWithDedup(t) } -func BenchmarkLoadTree(t *testing.B) { - repo := SetupRepo() - defer TeardownRepo(repo) - - if BenchArchiveDirectory == "" { - t.Skip("benchdir not set, skipping TestArchiverDedup") - } - - // archive a few files - arch := restic.NewArchiver(repo) - sn, _, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil) - OK(t, err) - t.Logf("archived snapshot %v", sn.ID()) - - list := make([]backend.ID, 0, 10) - done := make(chan struct{}) - - for _, idx := range repo.Index().All() { - for blob := range idx.Each(done) { - if blob.Type != restic.TreeBlob { - continue - } - - list = append(list, blob.ID) - if len(list) == cap(list) { - close(done) - break - } - } - } - - // start benchmark - t.ResetTimer() - - for i := 0; i < t.N; i++ { - for _, id := range list { - _, err := restic.LoadTree(repo, id) - OK(t, err) - } - } -} - // Saves several identical chunks concurrently and later checks that there are no // unreferenced packs in the repository. See also #292 and #358. func TestParallelSaveWithDuplication(t *testing.T) { @@ -248,7 +205,7 @@ func testParallelSaveWithDuplication(t *testing.T, seed int) { dataSizeMb := 128 duplication := 7 - arch := restic.NewArchiver(repo) + arch := archiver.New(repo) chunks := getRandomData(seed, dataSizeMb*1024*1024) errChannels := [](<-chan error){} @@ -265,7 +222,7 @@ func testParallelSaveWithDuplication(t *testing.T, seed int) { go func(c chunker.Chunk, errChan chan<- error) { barrier <- struct{}{} - id := backend.Hash(c.Data) + id := restic.Hash(c.Data) time.Sleep(time.Duration(id[0])) err := arch.Save(restic.DataBlob, c.Data, id) <-barrier @@ -301,7 +258,7 @@ func getRandomData(seed int, size int) []chunker.Chunk { return chunks } -func createAndInitChecker(t *testing.T, repo Repository) *checker.Checker { +func createAndInitChecker(t *testing.T, repo restic.Repository) *checker.Checker { chkr := checker.New(repo) hints, errs := chkr.LoadIndex() diff --git a/src/restic/checker/checker.go b/src/restic/checker/checker.go index 6bcea9b81..3b488cced 100644 --- a/src/restic/checker/checker.go +++ b/src/restic/checker/checker.go @@ -32,11 +32,11 @@ type Checker struct { masterIndex *repository.MasterIndex - repo *repository.Repository + repo restic.Repository } // New returns a new checker which runs on repo. -func New(repo *repository.Repository) *Checker { +func New(repo restic.Repository) *Checker { c := &Checker{ packs: restic.NewIDSet(), blobs: restic.NewIDSet(), @@ -180,7 +180,7 @@ func (e PackError) Error() string { return "pack " + e.ID.String() + ": " + e.Err.Error() } -func packIDTester(repo *repository.Repository, inChan <-chan restic.ID, errChan chan<- error, wg *sync.WaitGroup, done <-chan struct{}) { +func packIDTester(repo restic.Repository, inChan <-chan restic.ID, errChan chan<- error, wg *sync.WaitGroup, done <-chan struct{}) { debug.Log("Checker.testPackID", "worker start") defer debug.Log("Checker.testPackID", "worker done") @@ -273,7 +273,7 @@ func (e Error) Error() string { return e.Err.Error() } -func loadTreeFromSnapshot(repo *repository.Repository, id restic.ID) (restic.ID, error) { +func loadTreeFromSnapshot(repo restic.Repository, id restic.ID) (restic.ID, error) { sn, err := restic.LoadSnapshot(repo, id) if err != nil { debug.Log("Checker.loadTreeFromSnapshot", "error loading snapshot %v: %v", id.Str(), err) @@ -289,7 +289,7 @@ func loadTreeFromSnapshot(repo *repository.Repository, id restic.ID) (restic.ID, } // loadSnapshotTreeIDs loads all snapshots from backend and returns the tree IDs. -func loadSnapshotTreeIDs(repo *repository.Repository) (restic.IDs, []error) { +func loadSnapshotTreeIDs(repo restic.Repository) (restic.IDs, []error) { var trees struct { IDs restic.IDs sync.Mutex @@ -349,7 +349,7 @@ type treeJob struct { } // loadTreeWorker loads trees from repo and sends them to out. -func loadTreeWorker(repo *repository.Repository, +func loadTreeWorker(repo restic.Repository, in <-chan restic.ID, out chan<- treeJob, done <-chan struct{}, wg *sync.WaitGroup) { @@ -660,7 +660,7 @@ func (c *Checker) CountPacks() uint64 { } // checkPack reads a pack and checks the integrity of all blobs. -func checkPack(r *repository.Repository, id restic.ID) error { +func checkPack(r restic.Repository, id restic.ID) error { debug.Log("Checker.checkPack", "checking pack %v", id.Str()) h := restic.Handle{FileType: restic.DataFile, Name: id.String()} buf, err := backend.LoadAll(r.Backend(), h, nil) diff --git a/src/restic/checker/checker_test.go b/src/restic/checker/checker_test.go index e719cd229..6e9f29d06 100644 --- a/src/restic/checker/checker_test.go +++ b/src/restic/checker/checker_test.go @@ -8,7 +8,7 @@ import ( "testing" "restic" - "restic/backend" + "restic/archiver" "restic/backend/mem" "restic/checker" "restic/repository" @@ -147,7 +147,7 @@ func TestUnreferencedBlobs(t *testing.T) { snID := "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02" OK(t, repo.Backend().Remove(restic.SnapshotFile, snID)) - unusedBlobsBySnapshot := backend.IDs{ + unusedBlobsBySnapshot := restic.IDs{ ParseID("58c748bbe2929fdf30c73262bd8313fe828f8925b05d1d4a87fe109082acb849"), ParseID("988a272ab9768182abfd1fe7d7a7b68967825f0b861d3b36156795832c772235"), ParseID("c01952de4d91da1b1b80bc6e06eaa4ec21523f4853b69dc8231708b9b7ec62d8"), @@ -212,7 +212,7 @@ func TestDuplicatePacksInIndex(t *testing.T) { // errorBackend randomly modifies data after reading. type errorBackend struct { - backend.Backend + restic.Backend ProduceErrors bool } @@ -244,7 +244,7 @@ func TestCheckerModifiedData(t *testing.T) { repo := repository.New(be) OK(t, repo.Init(TestPassword)) - arch := restic.NewArchiver(repo) + arch := archiver.New(repo) _, id, err := arch.Snapshot(nil, []string{"."}, nil) OK(t, err) t.Logf("archived as %v", id.Str()) diff --git a/src/restic/index/index.go b/src/restic/index/index.go index 7d8f47250..e2f7f8317 100644 --- a/src/restic/index/index.go +++ b/src/restic/index/index.go @@ -5,10 +5,8 @@ import ( "fmt" "os" "restic" - "restic/backend" "restic/debug" "restic/list" - "restic/pack" "restic/worker" "github.com/pkg/errors" @@ -17,27 +15,27 @@ import ( // Pack contains information about the contents of a pack. type Pack struct { Size int64 - Entries []pack.Blob + Entries []restic.Blob } // Blob contains information about a blob. type Blob struct { Size int64 - Packs backend.IDSet + Packs restic.IDSet } // Index contains information about blobs and packs stored in a repo. type Index struct { - Packs map[backend.ID]Pack - Blobs map[pack.Handle]Blob - IndexIDs backend.IDSet + Packs map[restic.ID]Pack + Blobs map[restic.BlobHandle]Blob + IndexIDs restic.IDSet } func newIndex() *Index { return &Index{ - Packs: make(map[backend.ID]Pack), - Blobs: make(map[pack.Handle]Blob), - IndexIDs: backend.NewIDSet(), + Packs: make(map[restic.ID]Pack), + Blobs: make(map[restic.BlobHandle]Blob), + IndexIDs: restic.NewIDSet(), } } @@ -57,7 +55,7 @@ func New(repo restic.Repository, p *restic.Progress) (*Index, error) { for job := range ch { p.Report(restic.Stat{Blobs: 1}) - packID := job.Data.(backend.ID) + packID := job.Data.(restic.ID) if job.Error != nil { fmt.Fprintf(os.Stderr, "unable to list pack %v: %v\n", packID.Str(), job.Error) continue @@ -82,23 +80,23 @@ func New(repo restic.Repository, p *restic.Progress) (*Index, error) { const loadIndexParallelism = 20 type packJSON struct { - ID backend.ID `json:"id"` + ID restic.ID `json:"id"` Blobs []blobJSON `json:"blobs"` } type blobJSON struct { - ID backend.ID `json:"id"` - Type pack.BlobType `json:"type"` - Offset uint `json:"offset"` - Length uint `json:"length"` + ID restic.ID `json:"id"` + Type restic.BlobType `json:"type"` + Offset uint `json:"offset"` + Length uint `json:"length"` } type indexJSON struct { - Supersedes backend.IDs `json:"supersedes,omitempty"` + Supersedes restic.IDs `json:"supersedes,omitempty"` Packs []*packJSON `json:"packs"` } -func loadIndexJSON(repo restic.Repository, id backend.ID) (*indexJSON, error) { +func loadIndexJSON(repo restic.Repository, id restic.ID) (*indexJSON, error) { debug.Log("index.loadIndexJSON", "process index %v\n", id.Str()) var idx indexJSON @@ -120,8 +118,8 @@ func Load(repo restic.Repository, p *restic.Progress) (*Index, error) { done := make(chan struct{}) defer close(done) - supersedes := make(map[backend.ID]backend.IDSet) - results := make(map[backend.ID]map[backend.ID]Pack) + supersedes := make(map[restic.ID]restic.IDSet) + results := make(map[restic.ID]map[restic.ID]Pack) index := newIndex() @@ -134,17 +132,17 @@ func Load(repo restic.Repository, p *restic.Progress) (*Index, error) { return nil, err } - res := make(map[backend.ID]Pack) - supersedes[id] = backend.NewIDSet() + res := make(map[restic.ID]Pack) + supersedes[id] = restic.NewIDSet() for _, sid := range idx.Supersedes { debug.Log("index.Load", " index %v supersedes %v", id.Str(), sid) supersedes[id].Insert(sid) } for _, jpack := range idx.Packs { - entries := make([]pack.Blob, 0, len(jpack.Blobs)) + entries := make([]restic.Blob, 0, len(jpack.Blobs)) for _, blob := range jpack.Blobs { - entry := pack.Blob{ + entry := restic.Blob{ ID: blob.ID, Type: blob.Type, Offset: blob.Offset, @@ -178,7 +176,7 @@ func Load(repo restic.Repository, p *restic.Progress) (*Index, error) { // AddPack adds a pack to the index. If this pack is already in the index, an // error is returned. -func (idx *Index) AddPack(id backend.ID, size int64, entries []pack.Blob) error { +func (idx *Index) AddPack(id restic.ID, size int64, entries []restic.Blob) error { if _, ok := idx.Packs[id]; ok { return errors.Errorf("pack %v already present in the index", id.Str()) } @@ -186,11 +184,11 @@ func (idx *Index) AddPack(id backend.ID, size int64, entries []pack.Blob) error idx.Packs[id] = Pack{Size: size, Entries: entries} for _, entry := range entries { - h := pack.Handle{ID: entry.ID, Type: entry.Type} + h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} if _, ok := idx.Blobs[h]; !ok { idx.Blobs[h] = Blob{ Size: int64(entry.Length), - Packs: backend.NewIDSet(), + Packs: restic.NewIDSet(), } } @@ -201,13 +199,13 @@ func (idx *Index) AddPack(id backend.ID, size int64, entries []pack.Blob) error } // RemovePack deletes a pack from the index. -func (idx *Index) RemovePack(id backend.ID) error { +func (idx *Index) RemovePack(id restic.ID) error { if _, ok := idx.Packs[id]; !ok { return errors.Errorf("pack %v not found in the index", id.Str()) } for _, blob := range idx.Packs[id].Entries { - h := pack.Handle{ID: blob.ID, Type: blob.Type} + h := restic.BlobHandle{ID: blob.ID, Type: blob.Type} idx.Blobs[h].Packs.Delete(id) if len(idx.Blobs[h].Packs) == 0 { @@ -222,13 +220,13 @@ func (idx *Index) RemovePack(id backend.ID) error { // DuplicateBlobs returns a list of blobs that are stored more than once in the // repo. -func (idx *Index) DuplicateBlobs() (dups pack.BlobSet) { - dups = pack.NewBlobSet() - seen := pack.NewBlobSet() +func (idx *Index) DuplicateBlobs() (dups restic.BlobSet) { + dups = restic.NewBlobSet() + seen := restic.NewBlobSet() for _, p := range idx.Packs { for _, entry := range p.Entries { - h := pack.Handle{ID: entry.ID, Type: entry.Type} + h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} if seen.Has(h) { dups.Insert(h) } @@ -240,8 +238,8 @@ func (idx *Index) DuplicateBlobs() (dups pack.BlobSet) { } // PacksForBlobs returns the set of packs in which the blobs are contained. -func (idx *Index) PacksForBlobs(blobs pack.BlobSet) (packs backend.IDSet) { - packs = backend.NewIDSet() +func (idx *Index) PacksForBlobs(blobs restic.BlobSet) (packs restic.IDSet) { + packs = restic.NewIDSet() for h := range blobs { blob, ok := idx.Blobs[h] @@ -259,8 +257,8 @@ func (idx *Index) PacksForBlobs(blobs pack.BlobSet) (packs backend.IDSet) { // Location describes the location of a blob in a pack. type Location struct { - PackID backend.ID - pack.Blob + PackID restic.ID + restic.Blob } // ErrBlobNotFound is return by FindBlob when the blob could not be found in @@ -268,7 +266,7 @@ type Location struct { var ErrBlobNotFound = errors.New("blob not found in index") // FindBlob returns a list of packs and positions the blob can be found in. -func (idx *Index) FindBlob(h pack.Handle) ([]Location, error) { +func (idx *Index) FindBlob(h restic.BlobHandle) ([]Location, error) { blob, ok := idx.Blobs[h] if !ok { return nil, ErrBlobNotFound @@ -299,8 +297,8 @@ func (idx *Index) FindBlob(h pack.Handle) ([]Location, error) { } // Save writes the complete index to the repo. -func (idx *Index) Save(repo restic.Repository, supersedes backend.IDs) (backend.ID, error) { - packs := make(map[backend.ID][]pack.Blob, len(idx.Packs)) +func (idx *Index) Save(repo restic.Repository, supersedes restic.IDs) (restic.ID, error) { + packs := make(map[restic.ID][]restic.Blob, len(idx.Packs)) for id, p := range idx.Packs { packs[id] = p.Entries } @@ -309,7 +307,7 @@ func (idx *Index) Save(repo restic.Repository, supersedes backend.IDs) (backend. } // Save writes a new index containing the given packs. -func Save(repo restic.Repository, packs map[backend.ID][]pack.Blob, supersedes backend.IDs) (backend.ID, error) { +func Save(repo restic.Repository, packs map[restic.ID][]restic.Blob, supersedes restic.IDs) (restic.ID, error) { idx := &indexJSON{ Supersedes: supersedes, Packs: make([]*packJSON, 0, len(packs)), diff --git a/src/restic/index/index_test.go b/src/restic/index/index_test.go index 0f273a1c5..521d0c0b0 100644 --- a/src/restic/index/index_test.go +++ b/src/restic/index/index_test.go @@ -3,8 +3,6 @@ package index import ( "math/rand" "restic" - "restic/backend" - "restic/pack" "restic/repository" . "restic/test" "testing" @@ -179,7 +177,7 @@ func TestIndexSave(t *testing.T) { idx := loadIndex(t, repo) - packs := make(map[backend.ID][]pack.Blob) + packs := make(map[restic.ID][]restic.Blob) for id := range idx.Packs { if rand.Float32() < 0.5 { packs[id] = idx.Packs[id].Entries @@ -248,7 +246,7 @@ func TestIndexAddRemovePack(t *testing.T) { } for _, blob := range blobs { - h := pack.Handle{ID: blob.ID, Type: blob.Type} + h := restic.BlobHandle{ID: blob.ID, Type: blob.Type} _, err := idx.FindBlob(h) if err == nil { t.Errorf("removed blob %v found in index", h) @@ -308,7 +306,7 @@ func TestIndexLoadDocReference(t *testing.T) { idx := loadIndex(t, repo) blobID := ParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66") - locs, err := idx.FindBlob(pack.Handle{ID: blobID, Type: pack.Data}) + locs, err := idx.FindBlob(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) if err != nil { t.Errorf("FindBlob() returned error %v", err) } @@ -322,8 +320,8 @@ func TestIndexLoadDocReference(t *testing.T) { t.Errorf("blob IDs are not equal: %v != %v", l.ID, blobID) } - if l.Type != pack.Data { - t.Errorf("want type %v, got %v", pack.Data, l.Type) + if l.Type != restic.DataBlob { + t.Errorf("want type %v, got %v", restic.DataBlob, l.Type) } if l.Offset != 150 { diff --git a/src/restic/pack/pack_test.go b/src/restic/pack/pack_test.go index bfafa0631..5e5b1fc1e 100644 --- a/src/restic/pack/pack_test.go +++ b/src/restic/pack/pack_test.go @@ -10,7 +10,6 @@ import ( "restic" "testing" - "restic/backend" "restic/backend/mem" "restic/crypto" "restic/pack" @@ -21,7 +20,7 @@ var testLens = []int{23, 31650, 25860, 10928, 13769, 19862, 5211, 127, 13690, 30 type Buf struct { data []byte - id backend.ID + id restic.ID } func newPack(t testing.TB, k *crypto.Key, lengths []int) ([]Buf, []byte, uint) { @@ -38,7 +37,7 @@ func newPack(t testing.TB, k *crypto.Key, lengths []int) ([]Buf, []byte, uint) { // pack blobs p := pack.NewPacker(k, nil) for _, b := range bufs { - p.Add(pack.Tree, b.id, b.data) + p.Add(restic.TreeBlob, b.id, b.data) } _, err := p.Finalize() @@ -56,7 +55,7 @@ func verifyBlobs(t testing.TB, bufs []Buf, k *crypto.Key, rd io.ReaderAt, packSi // header length written += binary.Size(uint32(0)) // header - written += len(bufs) * (binary.Size(pack.BlobType(0)) + binary.Size(uint32(0)) + backend.IDSize) + written += len(bufs) * (binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + restic.IDSize) // header crypto written += crypto.Extension @@ -96,11 +95,11 @@ func TestCreatePack(t *testing.T) { } var blobTypeJSON = []struct { - t pack.BlobType + t restic.BlobType res string }{ - {pack.Data, `"data"`}, - {pack.Tree, `"tree"`}, + {restic.DataBlob, `"data"`}, + {restic.TreeBlob, `"tree"`}, } func TestBlobTypeJSON(t *testing.T) { @@ -111,7 +110,7 @@ func TestBlobTypeJSON(t *testing.T) { Equals(t, test.res, string(buf)) // test unserialize - var v pack.BlobType + var v restic.BlobType err = json.Unmarshal([]byte(test.res), &v) OK(t, err) Equals(t, test.t, v) @@ -125,9 +124,9 @@ func TestUnpackReadSeeker(t *testing.T) { bufs, packData, packSize := newPack(t, k, testLens) b := mem.New() - id := backend.Hash(packData) + id := restic.Hash(packData) - handle := restic.Handle{Type: restic.DataFile, Name: id.String()} + handle := restic.Handle{FileType: restic.DataFile, Name: id.String()} OK(t, b.Save(handle, packData)) verifyBlobs(t, bufs, k, restic.ReaderAt(b, handle), packSize) } @@ -138,9 +137,9 @@ func TestShortPack(t *testing.T) { bufs, packData, packSize := newPack(t, k, []int{23}) b := mem.New() - id := backend.Hash(packData) + id := restic.Hash(packData) - handle := restic.Handle{Type: restic.DataFile, Name: id.String()} + handle := restic.Handle{FileType: restic.DataFile, Name: id.String()} OK(t, b.Save(handle, packData)) - verifyBlobs(t, bufs, k, backend.ReaderAt(b, handle), packSize) + verifyBlobs(t, bufs, k, restic.ReaderAt(b, handle), packSize) } diff --git a/src/restic/repository.go b/src/restic/repository.go index 9bd22dabb..060a8f3a9 100644 --- a/src/restic/repository.go +++ b/src/restic/repository.go @@ -1,5 +1,7 @@ package restic +import "restic/crypto" + // Repository stores data in a backend. It provides high-level functions and // transparently encrypts/decrypts data. type Repository interface { @@ -7,6 +9,8 @@ type Repository interface { // Backend returns the backend used by the repository Backend() Backend + Key() *crypto.Key + SetIndex(Index) Index() Index @@ -24,6 +28,7 @@ type Repository interface { LoadJSONPack(BlobType, ID, interface{}) error LoadJSONUnpacked(FileType, ID, interface{}) error LoadBlob(ID, BlobType, []byte) ([]byte, error) + LoadAndDecrypt(FileType, ID) ([]byte, error) LookupBlobSize(ID, BlobType) (uint, error) @@ -47,4 +52,5 @@ type Lister interface { type Index interface { Has(ID, BlobType) bool Lookup(ID, BlobType) ([]PackedBlob, error) + Count(BlobType) uint } diff --git a/src/restic/repository/index.go b/src/restic/repository/index.go index 266b5d79a..f543a25ec 100644 --- a/src/restic/repository/index.go +++ b/src/restic/repository/index.go @@ -534,7 +534,7 @@ func DecodeOldIndex(rd io.Reader) (idx *Index, err error) { } // LoadIndexWithDecoder loads the index and decodes it with fn. -func LoadIndexWithDecoder(repo *Repository, id restic.ID, fn func(io.Reader) (*Index, error)) (idx *Index, err error) { +func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func(io.Reader) (*Index, error)) (idx *Index, err error) { debug.Log("LoadIndexWithDecoder", "Loading index %v", id[:8]) buf, err := repo.LoadAndDecrypt(restic.IndexFile, id) diff --git a/src/restic/repository/index_test.go b/src/restic/repository/index_test.go index 6b45d5990..65e8cdbd7 100644 --- a/src/restic/repository/index_test.go +++ b/src/restic/repository/index_test.go @@ -5,8 +5,6 @@ import ( "restic" "testing" - "restic/backend" - "restic/pack" "restic/repository" . "restic/test" ) @@ -24,24 +22,26 @@ func TestIndexSerialize(t *testing.T) { // create 50 packs with 20 blobs each for i := 0; i < 50; i++ { - packID := backend.RandomID() + packID := restic.TestRandomID() pos := uint(0) for j := 0; j < 20; j++ { - id := backend.RandomID() + id := restic.TestRandomID() length := uint(i*100 + j) - idx.Store(repository.PackedBlob{ - Type: pack.Data, - ID: id, + idx.Store(restic.PackedBlob{ + Blob: restic.Blob{ + Type: restic.DataBlob, + ID: id, + Offset: pos, + Length: length, + }, PackID: packID, - Offset: pos, - Length: length, }) tests = append(tests, testEntry{ id: id, pack: packID, - tpe: pack.Data, + tpe: restic.DataBlob, offset: pos, length: length, }) @@ -94,24 +94,26 @@ func TestIndexSerialize(t *testing.T) { // add more blobs to idx newtests := []testEntry{} for i := 0; i < 10; i++ { - packID := backend.RandomID() + packID := restic.TestRandomID() pos := uint(0) for j := 0; j < 10; j++ { - id := backend.RandomID() + id := restic.TestRandomID() length := uint(i*100 + j) - idx.Store(repository.PackedBlob{ - Type: pack.Data, - ID: id, + idx.Store(restic.PackedBlob{ + Blob: restic.Blob{ + Type: restic.DataBlob, + ID: id, + Offset: pos, + Length: length, + }, PackID: packID, - Offset: pos, - Length: length, }) newtests = append(newtests, testEntry{ id: id, pack: packID, - tpe: pack.Data, + tpe: restic.DataBlob, offset: pos, length: length, }) @@ -128,7 +130,7 @@ func TestIndexSerialize(t *testing.T) { Assert(t, idx.Final(), "index not final after encoding") - id := backend.RandomID() + id := restic.TestRandomID() OK(t, idx.SetID(id)) id2, err := idx.ID() Assert(t, id2.Equal(id), @@ -165,18 +167,20 @@ func TestIndexSize(t *testing.T) { packs := 200 blobs := 100 for i := 0; i < packs; i++ { - packID := backend.RandomID() + packID := restic.TestRandomID() pos := uint(0) for j := 0; j < blobs; j++ { - id := backend.RandomID() + id := restic.TestRandomID() length := uint(i*100 + j) - idx.Store(repository.PackedBlob{ - Type: pack.Data, - ID: id, + idx.Store(restic.PackedBlob{ + Blob: restic.Blob{ + Type: restic.DataBlob, + ID: id, + Offset: pos, + Length: length, + }, PackID: packID, - Offset: pos, - Length: length, }) pos += length @@ -257,15 +261,15 @@ var exampleTests = []struct { { ParseID("3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce"), ParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"), - pack.Data, 0, 25, + restic.DataBlob, 0, 25, }, { ParseID("9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae"), ParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"), - pack.Tree, 38, 100, + restic.TreeBlob, 38, 100, }, { ParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66"), ParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"), - pack.Data, 150, 123, + restic.DataBlob, 150, 123, }, } @@ -275,9 +279,9 @@ var exampleLookupTest = struct { }{ ParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"), map[restic.ID]restic.BlobType{ - ParseID("3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce"): pack.Data, - ParseID("9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae"): pack.Tree, - ParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66"): pack.Data, + ParseID("3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce"): restic.DataBlob, + ParseID("9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae"): restic.TreeBlob, + ParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66"): restic.DataBlob, }, } @@ -349,13 +353,15 @@ func TestIndexPacks(t *testing.T) { packs := restic.NewIDSet() for i := 0; i < 20; i++ { - packID := backend.RandomID() - idx.Store(repository.PackedBlob{ - Type: pack.Data, - ID: backend.RandomID(), + packID := restic.TestRandomID() + idx.Store(restic.PackedBlob{ + Blob: restic.Blob{ + Type: restic.DataBlob, + ID: restic.TestRandomID(), + Offset: 0, + Length: 23, + }, PackID: packID, - Offset: 0, - Length: 23, }) packs.Insert(packID) diff --git a/src/restic/repository/packer_manager_test.go b/src/restic/repository/packer_manager_test.go index 0e99e2e90..323051612 100644 --- a/src/restic/repository/packer_manager_test.go +++ b/src/restic/repository/packer_manager_test.go @@ -7,7 +7,6 @@ import ( "restic" "restic/backend/mem" "restic/crypto" - "restic/pack" "testing" ) @@ -64,7 +63,7 @@ func saveFile(t testing.TB, be Saver, filename string, n int) { t.Fatal(err) } - h := restic.Handle{Type: restic.DataFile, Name: restic.Hash(data).String()} + h := restic.Handle{FileType: restic.DataFile, Name: restic.Hash(data).String()} err = be.Save(h, data) if err != nil { @@ -95,7 +94,7 @@ func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf [ t.Fatal(err) } - n, err := packer.Add(pack.Data, id, buf) + n, err := packer.Add(restic.DataBlob, id, buf) if n != l { t.Errorf("Add() returned invalid number of bytes: want %v, got %v", n, l) } diff --git a/src/restic/repository/repack_test.go b/src/restic/repository/repack_test.go index 804cf77f1..026e43cbc 100644 --- a/src/restic/repository/repack_test.go +++ b/src/restic/repository/repack_test.go @@ -4,7 +4,6 @@ import ( "io" "math/rand" "restic" - "restic/pack" "restic/repository" "testing" ) @@ -32,18 +31,18 @@ func createRandomBlobs(t testing.TB, repo *repository.Repository, blobs int, pDa ) if rand.Float32() < pData { - tpe = pack.Data + tpe = restic.DataBlob length = randomSize(10*1024, 1024*1024) // 10KiB to 1MiB of data } else { - tpe = pack.Tree + tpe = restic.TreeBlob length = randomSize(1*1024, 20*1024) // 1KiB to 20KiB } buf := random(t, length) id := restic.Hash(buf) - if repo.Index().Has(id, pack.Data) { - t.Errorf("duplicate blob %v/%v ignored", id, pack.Data) + if repo.Index().Has(id, restic.DataBlob) { + t.Errorf("duplicate blob %v/%v ignored", id, restic.DataBlob) continue } @@ -66,14 +65,14 @@ func createRandomBlobs(t testing.TB, repo *repository.Repository, blobs int, pDa // selectBlobs splits the list of all blobs randomly into two lists. A blob // will be contained in the firstone ith probability p. -func selectBlobs(t *testing.T, repo *repository.Repository, p float32) (list1, list2 pack.BlobSet) { +func selectBlobs(t *testing.T, repo *repository.Repository, p float32) (list1, list2 restic.BlobSet) { done := make(chan struct{}) defer close(done) - list1 = pack.NewBlobSet() - list2 = pack.NewBlobSet() + list1 = restic.NewBlobSet() + list2 = restic.NewBlobSet() - blobs := pack.NewBlobSet() + blobs := restic.NewBlobSet() for id := range repo.List(restic.DataFile, done) { entries, _, err := repo.ListPack(id) @@ -82,7 +81,7 @@ func selectBlobs(t *testing.T, repo *repository.Repository, p float32) (list1, l } for _, entry := range entries { - h := pack.Handle{ID: entry.ID, Type: entry.Type} + h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} if blobs.Has(h) { t.Errorf("ignoring duplicate blob %v", h) continue @@ -90,9 +89,9 @@ func selectBlobs(t *testing.T, repo *repository.Repository, p float32) (list1, l blobs.Insert(h) if rand.Float32() <= p { - list1.Insert(pack.Handle{ID: entry.ID, Type: entry.Type}) + list1.Insert(restic.BlobHandle{ID: entry.ID, Type: entry.Type}) } else { - list2.Insert(pack.Handle{ID: entry.ID, Type: entry.Type}) + list2.Insert(restic.BlobHandle{ID: entry.ID, Type: entry.Type}) } } @@ -113,7 +112,7 @@ func listPacks(t *testing.T, repo *repository.Repository) restic.IDSet { return list } -func findPacksForBlobs(t *testing.T, repo *repository.Repository, blobs pack.BlobSet) restic.IDSet { +func findPacksForBlobs(t *testing.T, repo *repository.Repository, blobs restic.BlobSet) restic.IDSet { packs := restic.NewIDSet() idx := repo.Index() @@ -131,7 +130,7 @@ func findPacksForBlobs(t *testing.T, repo *repository.Repository, blobs pack.Blo return packs } -func repack(t *testing.T, repo *repository.Repository, packs restic.IDSet, blobs pack.BlobSet) { +func repack(t *testing.T, repo *repository.Repository, packs restic.IDSet, blobs restic.BlobSet) { err := repository.Repack(repo, packs, blobs) if err != nil { t.Fatal(err) diff --git a/src/restic/repository/repository_test.go b/src/restic/repository/repository_test.go index 4e27b8943..98b8edd84 100644 --- a/src/restic/repository/repository_test.go +++ b/src/restic/repository/repository_test.go @@ -11,7 +11,6 @@ import ( "testing" "restic" - "restic/pack" "restic/repository" . "restic/test" ) @@ -36,7 +35,7 @@ func TestSaveJSON(t *testing.T) { data = append(data, '\n') h := sha256.Sum256(data) - id, err := repo.SaveJSON(pack.Tree, obj) + id, err := repo.SaveJSON(restic.TreeBlob, obj) OK(t, err) Assert(t, h == id, @@ -59,7 +58,7 @@ func BenchmarkSaveJSON(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - id, err := repo.SaveJSON(pack.Tree, obj) + id, err := repo.SaveJSON(restic.TreeBlob, obj) OK(t, err) Assert(t, h == id, @@ -82,7 +81,7 @@ func TestSave(t *testing.T) { id := restic.Hash(data) // save - sid, err := repo.SaveAndEncrypt(pack.Data, data, nil) + sid, err := repo.SaveAndEncrypt(restic.DataBlob, data, nil) OK(t, err) Equals(t, id, sid) @@ -91,7 +90,7 @@ func TestSave(t *testing.T) { // OK(t, repo.SaveIndex()) // read back - buf, err := repo.LoadBlob(id, pack.Data, make([]byte, size)) + buf, err := repo.LoadBlob(id, restic.DataBlob, make([]byte, size)) OK(t, err) Assert(t, len(buf) == len(data), @@ -116,14 +115,14 @@ func TestSaveFrom(t *testing.T) { id := restic.Hash(data) // save - id2, err := repo.SaveAndEncrypt(pack.Data, data, &id) + id2, err := repo.SaveAndEncrypt(restic.DataBlob, data, &id) OK(t, err) Equals(t, id, id2) OK(t, repo.Flush()) // read back - buf, err := repo.LoadBlob(id, pack.Data, make([]byte, size)) + buf, err := repo.LoadBlob(id, restic.DataBlob, make([]byte, size)) OK(t, err) Assert(t, len(buf) == len(data), @@ -153,7 +152,7 @@ func BenchmarkSaveAndEncrypt(t *testing.B) { for i := 0; i < t.N; i++ { // save - _, err = repo.SaveAndEncrypt(pack.Data, data, &id) + _, err = repo.SaveAndEncrypt(restic.DataBlob, data, &id) OK(t, err) } } @@ -171,7 +170,7 @@ func TestLoadJSONPack(t *testing.T) { OK(t, repo.Flush()) tree := restic.NewTree() - err := repo.LoadJSONPack(pack.Tree, *sn.Tree, &tree) + err := repo.LoadJSONPack(restic.TreeBlob, *sn.Tree, &tree) OK(t, err) } @@ -192,7 +191,7 @@ func BenchmarkLoadJSONPack(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - err := repo.LoadJSONPack(pack.Tree, *sn.Tree, &tree) + err := repo.LoadJSONPack(restic.TreeBlob, *sn.Tree, &tree) OK(t, err) } } @@ -253,7 +252,7 @@ func saveRandomDataBlobs(t testing.TB, repo *repository.Repository, num int, siz _, err := io.ReadFull(rand.Reader, buf) OK(t, err) - _, err = repo.SaveAndEncrypt(pack.Data, buf, nil) + _, err = repo.SaveAndEncrypt(restic.DataBlob, buf, nil) OK(t, err) } }