From f78bd14e286dfa94b52c37f3b620955eecce9d51 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 13 Feb 2022 14:25:38 +0100 Subject: [PATCH] repository: Remove pack implementation details from MasterIndex --- cmd/restic/cmd_prune.go | 3 ++- cmd/restic/cmd_rebuild_index.go | 3 ++- internal/checker/checker.go | 2 +- internal/pack/pack.go | 26 ++++++++++++++++++++++++++ internal/repository/master_index.go | 22 ---------------------- internal/restic/repository.go | 1 - 6 files changed, 31 insertions(+), 26 deletions(-) diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index a03ae7474..76d7d7ea9 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -8,6 +8,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -245,7 +246,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB indexPack := make(map[restic.ID]packInfo) // save computed pack header size - for pid, hdrSize := range repo.Index().PackSize(ctx, true) { + for pid, hdrSize := range pack.Size(ctx, repo.Index(), true) { // initialize tpe with NumBlobTypes to indicate it's not set indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)} } diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_rebuild_index.go index 718d2c767..0b3274ec4 100644 --- a/cmd/restic/cmd_rebuild_index.go +++ b/cmd/restic/cmd_rebuild_index.go @@ -1,6 +1,7 @@ package main import ( + "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -97,7 +98,7 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor if err != nil { return err } - packSizeFromIndex = repo.Index().PackSize(ctx, false) + packSizeFromIndex = pack.Size(ctx, repo.Index(), false) } Verbosef("getting pack files to read...\n") diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 5262789d0..c2a900420 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -123,7 +123,7 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) { } // compute pack size using index entries - c.packs = c.masterIndex.PackSize(ctx, false) + c.packs = pack.Size(ctx, c.masterIndex, false) debug.Log("checking for duplicate packs") for packID := range c.packs { diff --git a/internal/pack/pack.go b/internal/pack/pack.go index 95f298acb..0c42f9ca2 100644 --- a/internal/pack/pack.go +++ b/internal/pack/pack.go @@ -1,6 +1,7 @@ package pack import ( + "context" "encoding/binary" "fmt" "io" @@ -317,3 +318,28 @@ func parseHeaderEntry(p []byte) (b restic.Blob, err error) { return b, nil } + +func CalculateHeaderSize(blobs []restic.Blob) int { + return HeaderSize + len(blobs)*int(EntrySize) +} + +// Size returns the size of all packs computed by index information. +// If onlyHdr is set to true, only the size of the header is returned +// Note that this function only gives correct sizes, if there are no +// duplicates in the index. +func Size(ctx context.Context, mi restic.MasterIndex, onlyHdr bool) map[restic.ID]int64 { + packSize := make(map[restic.ID]int64) + + for blob := range mi.Each(ctx) { + size, ok := packSize[blob.PackID] + if !ok { + size = HeaderSize + } + if !onlyHdr { + size += int64(blob.Length) + } + packSize[blob.PackID] = size + int64(EntrySize) + } + + return packSize +} diff --git a/internal/repository/master_index.go b/internal/repository/master_index.go index adc110df8..9056528a2 100644 --- a/internal/repository/master_index.go +++ b/internal/repository/master_index.go @@ -6,7 +6,6 @@ import ( "sync" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" @@ -131,27 +130,6 @@ func (mi *MasterIndex) Packs(packBlacklist restic.IDSet) restic.IDSet { return packs } -// PackSize returns the size of all packs computed by index information. -// If onlyHdr is set to true, only the size of the header is returned -// Note that this function only gives correct sizes, if there are no -// duplicates in the index. -func (mi *MasterIndex) PackSize(ctx context.Context, onlyHdr bool) map[restic.ID]int64 { - packSize := make(map[restic.ID]int64) - - for blob := range mi.Each(ctx) { - size, ok := packSize[blob.PackID] - if !ok { - size = pack.HeaderSize - } - if !onlyHdr { - size += int64(blob.Length) - } - packSize[blob.PackID] = size + int64(pack.EntrySize) - } - - return packSize -} - // Count returns the number of blobs of type t in the index. func (mi *MasterIndex) Count(t restic.BlobType) (n uint) { mi.idxMutex.RLock() diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 5e132157c..35f2f997b 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -70,7 +70,6 @@ type MasterIndex interface { Has(BlobHandle) bool Lookup(BlobHandle) []PackedBlob Count(BlobType) uint - PackSize(ctx context.Context, onlyHdr bool) map[ID]int64 // Each returns a channel that yields all blobs known to the index. When // the context is cancelled, the background goroutine terminates. This