diff --git a/internal/checker/checker_test.go b/internal/checker/checker_test.go index f2ee0c732..bbb538d7a 100644 --- a/internal/checker/checker_test.go +++ b/internal/checker/checker_test.go @@ -508,7 +508,6 @@ func TestCheckerBlobTypeConfusion(t *testing.T) { test.OK(t, err) test.OK(t, repo.Flush(ctx)) - test.OK(t, repo.SaveIndex(ctx)) snapshot, err := restic.NewSnapshot([]string{"/damaged"}, []string{"test"}, "foo", time.Now()) test.OK(t, err) diff --git a/internal/repository/master_index.go b/internal/repository/master_index.go index 22e763c35..f52060d51 100644 --- a/internal/repository/master_index.go +++ b/internal/repository/master_index.go @@ -1,6 +1,7 @@ package repository import ( + "bytes" "context" "fmt" "sync" @@ -405,6 +406,44 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.Repository, packBla return obsolete, err } +// SaveIndex saves an index in the repository. +func SaveIndex(ctx context.Context, repo restic.SaverUnpacked, index *Index) (restic.ID, error) { + buf := bytes.NewBuffer(nil) + + err := index.Encode(buf) + if err != nil { + return restic.ID{}, err + } + + return repo.SaveUnpacked(ctx, restic.IndexFile, buf.Bytes()) +} + +// saveIndex saves all indexes in the backend. +func (mi *MasterIndex) saveIndex(ctx context.Context, r restic.SaverUnpacked, indexes ...*Index) error { + for i, idx := range indexes { + debug.Log("Saving index %d", i) + + sid, err := SaveIndex(ctx, r, idx) + if err != nil { + return err + } + + debug.Log("Saved index %d as %v", i, sid) + } + + return mi.MergeFinalIndexes() +} + +// SaveIndex saves all new indexes in the backend. +func (mi *MasterIndex) SaveIndex(ctx context.Context, r restic.SaverUnpacked) error { + return mi.saveIndex(ctx, r, mi.FinalizeNotFinalIndexes()...) +} + +// SaveFullIndex saves all full indexes in the backend. +func (mi *MasterIndex) SaveFullIndex(ctx context.Context, r restic.SaverUnpacked) error { + return mi.saveIndex(ctx, r, mi.FinalizeFullIndexes()...) +} + // ListPacks returns the blobs of the specified pack files grouped by pack file. func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs { out := make(chan restic.PackBlobs) diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go index 7347a0e1a..7d0306a03 100644 --- a/internal/repository/packer_manager.go +++ b/internal/repository/packer_manager.go @@ -154,7 +154,7 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe if r.noAutoIndexUpdate { return nil } - return r.SaveFullIndex(ctx) + return r.idx.SaveFullIndex(ctx, r) } // countPacker returns the number of open (unfinished) packers. diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index b86c8c95d..8106a2f0c 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -155,8 +155,8 @@ func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs rest } } -func saveIndex(t *testing.T, repo restic.Repository) { - if err := repo.SaveIndex(context.TODO()); err != nil { +func flush(t *testing.T, repo restic.Repository) { + if err := repo.Flush(context.TODO()); err != nil { t.Fatalf("repo.SaveIndex() %v", err) } } @@ -237,7 +237,7 @@ func testRepack(t *testing.T, version uint) { packsBefore, packsAfter) } - saveIndex(t, repo) + flush(t, repo) removeBlobs, keepBlobs := selectBlobs(t, repo, 0.2) @@ -297,7 +297,7 @@ func testRepackCopy(t *testing.T, version uint) { t.Logf("rand seed is %v", seed) createRandomBlobs(t, repo, 100, 0.7) - saveIndex(t, repo) + flush(t, repo) _, keepBlobs := selectBlobs(t, repo, 0.2) copyPacks := findPacksForBlobs(t, repo, keepBlobs) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 6c095dcc5..1bc4d3245 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -527,7 +527,7 @@ func (r *Repository) Flush(ctx context.Context) error { if r.noAutoIndexUpdate { return nil } - return r.SaveIndex(ctx) + return r.idx.SaveIndex(ctx, r) } // FlushPacks saves all remaining packs. @@ -573,44 +573,6 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error { return r.PrepareCache() } -// SaveIndex saves an index in the repository. -func SaveIndex(ctx context.Context, repo restic.Repository, index *Index) (restic.ID, error) { - buf := bytes.NewBuffer(nil) - - err := index.Encode(buf) - if err != nil { - return restic.ID{}, err - } - - return repo.SaveUnpacked(ctx, restic.IndexFile, buf.Bytes()) -} - -// saveIndex saves all indexes in the backend. -func (r *Repository) saveIndex(ctx context.Context, indexes ...*Index) error { - for i, idx := range indexes { - debug.Log("Saving index %d", i) - - sid, err := SaveIndex(ctx, r, idx) - if err != nil { - return err - } - - debug.Log("Saved index %d as %v", i, sid) - } - - return r.idx.MergeFinalIndexes() -} - -// SaveIndex saves all new indexes in the backend. -func (r *Repository) SaveIndex(ctx context.Context) error { - return r.saveIndex(ctx, r.idx.FinalizeNotFinalIndexes()...) -} - -// SaveFullIndex saves all full indexes in the backend. -func (r *Repository) SaveFullIndex(ctx context.Context) error { - return r.saveIndex(ctx, r.idx.FinalizeFullIndexes()...) -} - // LoadIndex loads all index files from the backend in parallel and stores them // in the master index. The first error that occurred is returned. func (r *Repository) LoadIndex(ctx context.Context) error { diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 497fd2906..81604dea3 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -421,8 +421,7 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) { saveRandomDataBlobs(t, repo, 5, 1<<15) rtest.OK(t, repo.FlushPacks(context.Background())) } - - rtest.OK(t, repo.SaveFullIndex(context.TODO())) + rtest.OK(t, repo.Flush(context.TODO())) } // add another 5 packs @@ -432,7 +431,7 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) { } // save final index - rtest.OK(t, repo.SaveIndex(context.TODO())) + rtest.OK(t, repo.Flush(context.TODO())) packEntries := make(map[restic.ID]map[restic.ID]struct{}) diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 61654fa1d..0d608e9bf 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -15,17 +15,13 @@ type Repository interface { Key() *crypto.Key - SetIndex(MasterIndex) error - Index() MasterIndex - SaveFullIndex(context.Context) error - SaveIndex(context.Context) error LoadIndex(context.Context) error + SetIndex(MasterIndex) error + LookupBlobSize(ID, BlobType) (uint, bool) Config() Config - LookupBlobSize(ID, BlobType) (uint, bool) - // List calls the function fn for each file of type t in the repository. // When an error is returned by fn, processing stops and List() returns the // error. @@ -65,6 +61,11 @@ type LoadJSONUnpackeder interface { LoadJSONUnpacked(ctx context.Context, t FileType, id ID, dest interface{}) error } +// SaverUnpacked allows saving a blob not stored in a pack file +type SaverUnpacked interface { + SaveUnpacked(context.Context, FileType, []byte) (ID, error) +} + type PackBlobs struct { PackID ID Blobs []Blob