From 9607cad267ceaf53bd6c13bbc77248da41023a68 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Sat, 10 Oct 2020 21:51:31 +0200 Subject: [PATCH] Remove internal/index --- cmd/restic/cmd_rebuild_index.go | 29 +- internal/index/index.go | 373 ------------------------ internal/index/index_test.go | 497 -------------------------------- 3 files changed, 13 insertions(+), 886 deletions(-) delete mode 100644 internal/index/index.go delete mode 100644 internal/index/index_test.go diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_rebuild_index.go index b57d29200..850c03dd2 100644 --- a/cmd/restic/cmd_rebuild_index.go +++ b/cmd/restic/cmd_rebuild_index.go @@ -58,15 +58,15 @@ func runRebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions) error { func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error { ctx := gopts.ctx - var obsolete restic.IDs + var obsoleteIndexes restic.IDs packSizeFromList := make(map[restic.ID]int64) - packs := restic.NewIDSet() + removePacks := restic.NewIDSet() totalPacks := 0 if opts.ReadAllPacks { // get old index files err := repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { - obsolete = append(obsolete, id) + obsoleteIndexes = append(obsoleteIndexes, id) return nil }) if err != nil { @@ -76,7 +76,7 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor Verbosef("finding pack files in repo...\n") err = repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { packSizeFromList[id] = size - packs.Insert(id) + removePacks.Insert(id) totalPacks++ return nil }) @@ -90,18 +90,17 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor return err } - packSizeFromIndex := make(map[restic.ID]int64) - Verbosef("getting pack files to read...\n") - // iterate over all blobs in index + + // Compute size of each pack from index entries + packSizeFromIndex := make(map[restic.ID]int64) for blob := range repo.Index().Each(ctx) { size, ok := packSizeFromIndex[blob.PackID] if !ok { size = pack.HeaderSize } - size += int64(pack.PackedSizeOfBlob(blob.Length)) // update packSizeFromIndex - packSizeFromIndex[blob.PackID] = size + packSizeFromIndex[blob.PackID] = size + int64(pack.PackedSizeOfBlob(blob.Length)) } err = repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error { @@ -109,7 +108,7 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor if !ok || size != packSize { // Pack was not referenced in index or size does not match packSizeFromList[id] = size - packs.Insert(id) + removePacks.Insert(id) } totalPacks++ delete(packSizeFromIndex, id) @@ -119,16 +118,16 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor return err } for id := range packSizeFromIndex { - // ignore pack files that are referenced in the index but do not exist + // forget pack files that are referenced in the index but do not exist // when rebuilding the index - packs.Insert(id) + removePacks.Insert(id) } } if len(packSizeFromList) > 0 { Verbosef("reading pack files\n") bar := newProgressMax(!globalOptions.Quiet, uint64(len(packSizeFromList)), "packs") - invalidFiles, err := repo.LoadIndexFromPacks(ctx, packSizeFromList, bar) + invalidFiles, err := repo.CreateIndexFromPacks(ctx, packSizeFromList, bar) if err != nil { return err } @@ -137,11 +136,9 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor Verboseff("skipped incomplete pack file: %v\n", id) totalPacks-- } - } else { - Verbosef("no need to read any pack file\n") } - err := rebuildIndexFiles(gopts, repo, packs, obsolete, uint64(totalPacks)) + err := rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes, uint64(totalPacks)) if err != nil { return err } diff --git a/internal/index/index.go b/internal/index/index.go deleted file mode 100644 index 4c0f5fee7..000000000 --- a/internal/index/index.go +++ /dev/null @@ -1,373 +0,0 @@ -// Package index contains various data structures for indexing content in a repository or backend. -package index - -import ( - "context" - "fmt" - "os" - "sync" - - "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/pack" - "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/ui/progress" - - "golang.org/x/sync/errgroup" -) - -// Pack contains information about the contents of a pack. -type Pack struct { - ID restic.ID - Size int64 - Entries []restic.Blob -} - -// Index contains information about blobs and packs stored in a repo. -type Index struct { - Packs map[restic.ID]Pack - IndexIDs restic.IDSet -} - -func newIndex() *Index { - return &Index{ - Packs: make(map[restic.ID]Pack), - IndexIDs: restic.NewIDSet(), - } -} - -const listPackWorkers = 10 - -// Lister lists files and their contents -type Lister interface { - // List runs fn for all files of type t in the repo. - List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error - - // ListPack returns the list of blobs saved in the pack id and the length - // of the file as stored in the backend. - ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, int64, error) -} - -// New creates a new index for repo from scratch. InvalidFiles contains all IDs -// of files that cannot be listed successfully. -func New(ctx context.Context, repo Lister, ignorePacks restic.IDSet, p *progress.Counter) (idx *Index, invalidFiles restic.IDs, err error) { - type Job struct { - PackID restic.ID - Size int64 - } - - type Result struct { - Error error - PackID restic.ID - Size int64 - Entries []restic.Blob - } - - inputCh := make(chan Job) - outputCh := make(chan Result) - wg, ctx := errgroup.WithContext(ctx) - - // list the files in the repo, send to inputCh - wg.Go(func() error { - defer close(inputCh) - return repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { - if ignorePacks.Has(id) { - return nil - } - - job := Job{ - PackID: id, - Size: size, - } - - select { - case inputCh <- job: - case <-ctx.Done(): - } - return nil - }) - }) - - // run the workers listing the files, read from inputCh, send to outputCh - var workers sync.WaitGroup - for i := 0; i < listPackWorkers; i++ { - workers.Add(1) - go func() { - defer workers.Done() - for job := range inputCh { - res := Result{PackID: job.PackID} - res.Entries, res.Size, res.Error = repo.ListPack(ctx, job.PackID, job.Size) - - select { - case outputCh <- res: - case <-ctx.Done(): - return - } - } - }() - } - - // wait until all the workers are done, then close outputCh - wg.Go(func() error { - workers.Wait() - close(outputCh) - return nil - }) - - idx = newIndex() - - for res := range outputCh { - p.Add(1) - if res.Error != nil { - cause := errors.Cause(res.Error) - if _, ok := cause.(pack.InvalidFileError); ok { - invalidFiles = append(invalidFiles, res.PackID) - continue - } - - fmt.Fprintf(os.Stderr, "pack file cannot be listed %v: %v\n", res.PackID, res.Error) - continue - } - - debug.Log("pack %v contains %d blobs", res.PackID, len(res.Entries)) - - err := idx.AddPack(res.PackID, res.Size, res.Entries) - if err != nil { - return nil, nil, err - } - - select { - case <-ctx.Done(): // an error occurred - default: - } - } - - err = wg.Wait() - if err != nil { - return nil, nil, err - } - - return idx, invalidFiles, nil -} - -type packJSON struct { - ID restic.ID `json:"id"` - Blobs []blobJSON `json:"blobs"` -} - -type blobJSON struct { - ID restic.ID `json:"id"` - Type restic.BlobType `json:"type"` - Offset uint `json:"offset"` - Length uint `json:"length"` -} - -type indexJSON struct { - Supersedes restic.IDs `json:"supersedes,omitempty"` - Packs []packJSON `json:"packs"` -} - -// ListLoader allows listing files and their content, in addition to loading and unmarshaling JSON files. -type ListLoader interface { - Lister - LoadJSONUnpacked(context.Context, restic.FileType, restic.ID, interface{}) error -} - -func loadIndexJSON(ctx context.Context, repo ListLoader, id restic.ID) (*indexJSON, error) { - debug.Log("process index %v\n", id) - - var idx indexJSON - err := repo.LoadJSONUnpacked(ctx, restic.IndexFile, id, &idx) - if err != nil { - return nil, err - } - - return &idx, nil -} - -// Load creates an index by loading all index files from the repo. -func Load(ctx context.Context, repo ListLoader, p *progress.Counter) (*Index, error) { - debug.Log("loading indexes") - - supersedes := make(map[restic.ID]restic.IDSet) - results := make(map[restic.ID]map[restic.ID]Pack) - - index := newIndex() - - err := repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { - p.Add(1) - - debug.Log("Load index %v", id) - idx, err := loadIndexJSON(ctx, repo, id) - if err != nil { - return err - } - - res := make(map[restic.ID]Pack) - supersedes[id] = restic.NewIDSet() - for _, sid := range idx.Supersedes { - debug.Log(" index %v supersedes %v", id, sid) - supersedes[id].Insert(sid) - } - - for _, jpack := range idx.Packs { - entries := make([]restic.Blob, 0, len(jpack.Blobs)) - for _, blob := range jpack.Blobs { - entry := restic.Blob{ - ID: blob.ID, - Type: blob.Type, - Offset: blob.Offset, - Length: blob.Length, - } - entries = append(entries, entry) - } - - if err = index.AddPack(jpack.ID, 0, entries); err != nil { - return err - } - } - - results[id] = res - index.IndexIDs.Insert(id) - - return nil - }) - - if err != nil { - return nil, err - } - - for superID, list := range supersedes { - for indexID := range list { - if _, ok := results[indexID]; !ok { - continue - } - debug.Log(" removing index %v, superseded by %v", indexID, superID) - fmt.Fprintf(os.Stderr, "index %v can be removed, superseded by index %v\n", indexID.Str(), superID.Str()) - delete(results, indexID) - } - } - - return index, nil -} - -// AddPack adds a pack to the index. If this pack is already in the index, an -// error is returned. -func (idx *Index) AddPack(id restic.ID, size int64, entries []restic.Blob) error { - if _, ok := idx.Packs[id]; ok { - return errors.Errorf("pack %v already present in the index", id.Str()) - } - - idx.Packs[id] = Pack{ID: id, Size: size, Entries: entries} - - return nil -} - -// RemovePack deletes a pack from the index. -func (idx *Index) RemovePack(id restic.ID) error { - if _, ok := idx.Packs[id]; !ok { - return errors.Errorf("pack %v not found in the index", id.Str()) - } - - delete(idx.Packs, id) - - return nil -} - -// DuplicateBlobs returns a list of blobs that are stored more than once in the -// repo. -func (idx *Index) DuplicateBlobs() (dups restic.BlobSet) { - dups = restic.NewBlobSet() - seen := restic.NewBlobSet() - - for _, p := range idx.Packs { - for _, entry := range p.Entries { - h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} - if seen.Has(h) { - dups.Insert(h) - } - seen.Insert(h) - } - } - - return dups -} - -// PacksForBlobs returns the set of packs in which the blobs are contained. -func (idx *Index) PacksForBlobs(blobs restic.BlobSet) (packs restic.IDSet) { - packs = restic.NewIDSet() - - for id, p := range idx.Packs { - for _, entry := range p.Entries { - if blobs.Has(restic.BlobHandle{ID: entry.ID, Type: entry.Type}) { - packs.Insert(id) - } - } - } - - return packs -} - -const maxEntries = 3000 - -// Saver saves structures as JSON. -type Saver interface { - SaveJSONUnpacked(ctx context.Context, t restic.FileType, item interface{}) (restic.ID, error) -} - -// Save writes the complete index to the repo. -func (idx *Index) Save(ctx context.Context, repo Saver, supersedes restic.IDs) (restic.IDs, error) { - debug.Log("pack files: %d\n", len(idx.Packs)) - - var indexIDs []restic.ID - - packs := 0 - jsonIDX := &indexJSON{ - Supersedes: supersedes, - Packs: make([]packJSON, 0, maxEntries), - } - - for packID, pack := range idx.Packs { - debug.Log("%04d add pack %v with %d entries", packs, packID, len(pack.Entries)) - b := make([]blobJSON, 0, len(pack.Entries)) - for _, blob := range pack.Entries { - b = append(b, blobJSON{ - ID: blob.ID, - Type: blob.Type, - Offset: blob.Offset, - Length: blob.Length, - }) - } - - p := packJSON{ - ID: packID, - Blobs: b, - } - - jsonIDX.Packs = append(jsonIDX.Packs, p) - - packs++ - if packs == maxEntries { - id, err := repo.SaveJSONUnpacked(ctx, restic.IndexFile, jsonIDX) - if err != nil { - return nil, err - } - debug.Log("saved new index as %v", id) - - indexIDs = append(indexIDs, id) - packs = 0 - jsonIDX.Packs = jsonIDX.Packs[:0] - } - } - - if packs > 0 { - id, err := repo.SaveJSONUnpacked(ctx, restic.IndexFile, jsonIDX) - if err != nil { - return nil, err - } - debug.Log("saved new index as %v", id) - indexIDs = append(indexIDs, id) - } - - return indexIDs, nil -} diff --git a/internal/index/index_test.go b/internal/index/index_test.go deleted file mode 100644 index f4ff53f7a..000000000 --- a/internal/index/index_test.go +++ /dev/null @@ -1,497 +0,0 @@ -package index - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/restic/restic/internal/checker" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/test" -) - -var ( - snapshotTime = time.Unix(1470492820, 207401672) - depth = 3 -) - -func createFilledRepo(t testing.TB, snapshots int, dup float32) (restic.Repository, func()) { - repo, cleanup := repository.TestRepository(t) - - for i := 0; i < 3; i++ { - restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth, dup) - } - - return repo, cleanup -} - -func validateIndex(t testing.TB, repo restic.Repository, idx *Index) { - err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { - p, ok := idx.Packs[id] - if !ok { - t.Errorf("pack %v missing from index", id.Str()) - } - - if !p.ID.Equal(id) { - t.Errorf("pack %v has invalid ID: want %v, got %v", id.Str(), id, p.ID) - } - return nil - }) - - if err != nil { - t.Fatal(err) - } -} - -func TestIndexNew(t *testing.T) { - repo, cleanup := createFilledRepo(t, 3, 0) - defer cleanup() - - idx, invalid, err := New(context.TODO(), repo, restic.NewIDSet(), nil) - if err != nil { - t.Fatalf("New() returned error %v", err) - } - - if idx == nil { - t.Fatalf("New() returned nil index") - } - - if len(invalid) > 0 { - t.Fatalf("New() returned invalid files: %v", invalid) - } - - validateIndex(t, repo, idx) -} - -type ErrorRepo struct { - restic.Repository - MaxListFiles int - - MaxPacks int - MaxPacksMutex sync.Mutex -} - -// List returns an error after repo.MaxListFiles files. -func (repo *ErrorRepo) List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error { - if repo.MaxListFiles == 0 { - return errors.New("test error, max is zero") - } - - max := repo.MaxListFiles - return repo.Repository.List(ctx, t, func(id restic.ID, size int64) error { - if max == 0 { - return errors.New("test error, max reached zero") - } - - max-- - return fn(id, size) - }) -} - -// ListPack returns an error after repo.MaxPacks files. -func (repo *ErrorRepo) ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, int64, error) { - repo.MaxPacksMutex.Lock() - max := repo.MaxPacks - if max > 0 { - repo.MaxPacks-- - } - repo.MaxPacksMutex.Unlock() - - if max == 0 { - return nil, 0, errors.New("test list pack error") - } - - return repo.Repository.ListPack(ctx, id, size) -} - -func TestIndexNewListErrors(t *testing.T) { - repo, cleanup := createFilledRepo(t, 3, 0) - defer cleanup() - - for _, max := range []int{0, 3, 5} { - errRepo := &ErrorRepo{ - Repository: repo, - MaxListFiles: max, - } - idx, invalid, err := New(context.TODO(), errRepo, restic.NewIDSet(), nil) - if err == nil { - t.Errorf("expected error not found, got nil") - } - - if idx != nil { - t.Errorf("expected nil index, got %v", idx) - } - - if len(invalid) != 0 { - t.Errorf("expected empty invalid list, got %v", invalid) - } - } -} - -func TestIndexNewPackErrors(t *testing.T) { - repo, cleanup := createFilledRepo(t, 3, 0) - defer cleanup() - - for _, max := range []int{0, 3, 5} { - errRepo := &ErrorRepo{ - Repository: repo, - MaxPacks: max, - } - idx, invalid, err := New(context.TODO(), errRepo, restic.NewIDSet(), nil) - if err == nil { - t.Errorf("expected error not found, got nil") - } - - if idx != nil { - t.Errorf("expected nil index, got %v", idx) - } - - if len(invalid) != 0 { - t.Errorf("expected empty invalid list, got %v", invalid) - } - } -} - -func TestIndexLoad(t *testing.T) { - repo, cleanup := createFilledRepo(t, 3, 0) - defer cleanup() - - loadIdx, err := Load(context.TODO(), repo, nil) - if err != nil { - t.Fatalf("Load() returned error %v", err) - } - - if loadIdx == nil { - t.Fatalf("Load() returned nil index") - } - - validateIndex(t, repo, loadIdx) - - newIdx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil) - if err != nil { - t.Fatalf("New() returned error %v", err) - } - - if len(loadIdx.Packs) != len(newIdx.Packs) { - t.Errorf("number of packs does not match: want %v, got %v", - len(loadIdx.Packs), len(newIdx.Packs)) - } - - validateIndex(t, repo, newIdx) - - for packID, packNew := range newIdx.Packs { - packLoad, ok := loadIdx.Packs[packID] - - if !ok { - t.Errorf("loaded index does not list pack %v", packID.Str()) - continue - } - - if len(packNew.Entries) != len(packLoad.Entries) { - t.Errorf(" number of entries in pack %v does not match: %d != %d\n %v\n %v", - packID.Str(), len(packNew.Entries), len(packLoad.Entries), - packNew.Entries, packLoad.Entries) - continue - } - - for _, entryNew := range packNew.Entries { - found := false - for _, entryLoad := range packLoad.Entries { - if !entryLoad.ID.Equal(entryNew.ID) { - continue - } - - if entryLoad.Type != entryNew.Type { - continue - } - - if entryLoad.Offset != entryNew.Offset { - continue - } - - if entryLoad.Length != entryNew.Length { - continue - } - - found = true - break - } - - if !found { - t.Errorf("blob not found in loaded index: %v", entryNew) - } - } - } -} - -func BenchmarkIndexNew(b *testing.B) { - repo, cleanup := createFilledRepo(b, 3, 0) - defer cleanup() - - b.ResetTimer() - - for i := 0; i < b.N; i++ { - idx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil) - - if err != nil { - b.Fatalf("New() returned error %v", err) - } - - if idx == nil { - b.Fatalf("New() returned nil index") - } - b.Logf("idx %v packs", len(idx.Packs)) - } -} - -func BenchmarkIndexSave(b *testing.B) { - repo, cleanup := repository.TestRepository(b) - defer cleanup() - - idx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil) - test.OK(b, err) - - for i := 0; i < 8000; i++ { - entries := make([]restic.Blob, 0, 200) - for j := 0; j < cap(entries); j++ { - entries = append(entries, restic.Blob{ - ID: restic.NewRandomID(), - Length: 1000, - Offset: 5, - Type: restic.DataBlob, - }) - } - - idx.AddPack(restic.NewRandomID(), 10000, entries) - } - - b.ResetTimer() - - for i := 0; i < b.N; i++ { - ids, err := idx.Save(context.TODO(), repo, nil) - if err != nil { - b.Fatalf("New() returned error %v", err) - } - - b.Logf("saved as %v", ids) - } -} - -func TestIndexDuplicateBlobs(t *testing.T) { - repo, cleanup := createFilledRepo(t, 3, 0.05) - defer cleanup() - - idx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil) - if err != nil { - t.Fatal(err) - } - - dups := idx.DuplicateBlobs() - if len(dups) == 0 { - t.Errorf("no duplicate blobs found") - } - t.Logf("%d packs, %d duplicate blobs", len(idx.Packs), len(dups)) - - packs := idx.PacksForBlobs(dups) - if len(packs) == 0 { - t.Errorf("no packs with duplicate blobs found") - } - t.Logf("%d packs with duplicate blobs", len(packs)) -} - -func loadIndex(t testing.TB, repo restic.Repository) *Index { - idx, err := Load(context.TODO(), repo, nil) - if err != nil { - t.Fatalf("Load() returned error %v", err) - } - - return idx -} - -func TestIndexSave(t *testing.T) { - repo, cleanup := createFilledRepo(t, 3, 0) - defer cleanup() - - idx := loadIndex(t, repo) - - ids, err := idx.Save(context.TODO(), repo, idx.IndexIDs.List()) - if err != nil { - t.Fatalf("unable to save new index: %v", err) - } - - t.Logf("new index saved as %v", ids) - - for id := range idx.IndexIDs { - t.Logf("remove index %v", id.Str()) - h := restic.Handle{Type: restic.IndexFile, Name: id.String()} - err = repo.Backend().Remove(context.TODO(), h) - if err != nil { - t.Errorf("error removing index %v: %v", id, err) - } - } - - idx2 := loadIndex(t, repo) - t.Logf("load new index with %d packs", len(idx2.Packs)) - - checker := checker.New(repo) - hints, errs := checker.LoadIndex(context.TODO()) - for _, h := range hints { - t.Logf("hint: %v\n", h) - } - - for _, err := range errs { - t.Errorf("checker found error: %v", err) - } - - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - - errCh := make(chan error) - go checker.Structure(ctx, errCh) - i := 0 - for err := range errCh { - t.Errorf("checker returned error: %v", err) - i++ - if i == 10 { - t.Errorf("more than 10 errors returned, skipping the rest") - cancel() - break - } - } -} - -// Location describes the location of a blob in a pack. -type location struct { - PackID restic.ID - restic.Blob -} - -// FindBlob returns a list of packs and positions the blob can be found in. -func (idx *Index) findBlob(h restic.BlobHandle) (result []location) { - for id, p := range idx.Packs { - for _, entry := range p.Entries { - if entry.ID.Equal(h.ID) && entry.Type == h.Type { - result = append(result, location{ - PackID: id, - Blob: entry, - }) - } - } - } - - return result -} - -func TestIndexAddRemovePack(t *testing.T) { - repo, cleanup := createFilledRepo(t, 3, 0) - defer cleanup() - - idx, err := Load(context.TODO(), repo, nil) - if err != nil { - t.Fatalf("Load() returned error %v", err) - } - - var packID restic.ID - err = repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { - packID = id - return nil - }) - if err != nil { - t.Fatal(err) - } - - t.Logf("selected pack %v", packID.Str()) - - blobs := idx.Packs[packID].Entries - - idx.RemovePack(packID) - - if _, ok := idx.Packs[packID]; ok { - t.Errorf("removed pack %v found in index.Packs", packID.Str()) - } - - for _, blob := range blobs { - h := restic.BlobHandle{ID: blob.ID, Type: blob.Type} - locs := idx.findBlob(h) - if len(locs) != 0 { - t.Errorf("removed blob %v found in index", h) - } - } -} - -// example index serialization from doc/Design.rst -var docExample = []byte(` -{ - "supersedes": [ - "ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452" - ], - "packs": [ - { - "id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c", - "blobs": [ - { - "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce", - "type": "data", - "offset": 0, - "length": 25 - },{ - "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae", - "type": "tree", - "offset": 38, - "length": 100 - }, - { - "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66", - "type": "data", - "offset": 150, - "length": 123 - } - ] - } - ] -} -`) - -func TestIndexLoadDocReference(t *testing.T) { - repo, cleanup := repository.TestRepository(t) - defer cleanup() - - id, err := repo.SaveUnpacked(context.TODO(), restic.IndexFile, docExample) - if err != nil { - t.Fatalf("SaveUnpacked() returned error %v", err) - } - - t.Logf("index saved as %v", id.Str()) - - idx := loadIndex(t, repo) - - blobID := restic.TestParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66") - locs := idx.findBlob(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) - if len(locs) == 0 { - t.Error("blob not found in index") - } - - if len(locs) != 1 { - t.Errorf("blob found %d times, expected just one", len(locs)) - } - - l := locs[0] - if !l.ID.Equal(blobID) { - t.Errorf("blob IDs are not equal: %v != %v", l.ID, blobID) - } - - if l.Type != restic.DataBlob { - t.Errorf("want type %v, got %v", restic.DataBlob, l.Type) - } - - if l.Offset != 150 { - t.Errorf("wrong offset, want %d, got %v", 150, l.Offset) - } - - if l.Length != 123 { - t.Errorf("wrong length, want %d, got %v", 123, l.Length) - } -}