mirror of
https://github.com/octoleo/restic.git
synced 2024-11-30 00:33:57 +00:00
Count packs directly in RebuildIndexFiles
This commit is contained in:
parent
69f9d269eb
commit
68b74e359e
@ -479,11 +479,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
|
|||||||
DeleteFiles(gopts, repo, removePacksFirst, restic.PackFile)
|
DeleteFiles(gopts, repo, removePacksFirst, restic.PackFile)
|
||||||
}
|
}
|
||||||
|
|
||||||
packsAddedByRepack := 0
|
|
||||||
if len(repackPacks) != 0 {
|
if len(repackPacks) != 0 {
|
||||||
// Remember the number of unique packs before repacking
|
|
||||||
packsBeforeRepacking := len(repo.Index().Packs())
|
|
||||||
|
|
||||||
Verbosef("repacking packs\n")
|
Verbosef("repacking packs\n")
|
||||||
bar := newProgressMax(!gopts.Quiet, uint64(len(repackPacks)), "packs repacked")
|
bar := newProgressMax(!gopts.Quiet, uint64(len(repackPacks)), "packs repacked")
|
||||||
_, err := repository.Repack(ctx, repo, repackPacks, keepBlobs, bar)
|
_, err := repository.Repack(ctx, repo, repackPacks, keepBlobs, bar)
|
||||||
@ -492,18 +488,12 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Since repacking will only add new packs, we can calculate the number
|
|
||||||
// of packs like this:
|
|
||||||
packsAddedByRepack = len(repo.Index().Packs()) - packsBeforeRepacking
|
|
||||||
|
|
||||||
// Also remove repacked packs
|
// Also remove repacked packs
|
||||||
removePacks.Merge(repackPacks)
|
removePacks.Merge(repackPacks)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(removePacks) != 0 {
|
if len(removePacks) != 0 {
|
||||||
totalpacks := int(stats.packs.used+stats.packs.partlyUsed+stats.packs.unused) -
|
err = rebuildIndexFiles(gopts, repo, removePacks, nil)
|
||||||
len(removePacks) + packsAddedByRepack
|
|
||||||
err = rebuildIndexFiles(gopts, repo, removePacks, nil, uint64(totalpacks))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -516,12 +506,13 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs, packcount uint64) error {
|
func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
|
||||||
Verbosef("rebuilding index\n")
|
Verbosef("rebuilding index\n")
|
||||||
|
|
||||||
|
idx := (repo.Index()).(*repository.MasterIndex)
|
||||||
|
packcount := uint64(len(idx.Packs(removePacks)))
|
||||||
bar := newProgressMax(!gopts.Quiet, packcount, "packs processed")
|
bar := newProgressMax(!gopts.Quiet, packcount, "packs processed")
|
||||||
obsoleteIndexes, err := (repo.Index()).(*repository.MasterIndex).
|
obsoleteIndexes, err := idx.Save(gopts.ctx, repo, removePacks, extraObsolete, bar)
|
||||||
Save(gopts.ctx, repo, removePacks, extraObsolete, bar)
|
|
||||||
bar.Done()
|
bar.Done()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -60,7 +60,6 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
|
|||||||
var obsoleteIndexes restic.IDs
|
var obsoleteIndexes restic.IDs
|
||||||
packSizeFromList := make(map[restic.ID]int64)
|
packSizeFromList := make(map[restic.ID]int64)
|
||||||
removePacks := restic.NewIDSet()
|
removePacks := restic.NewIDSet()
|
||||||
totalPacks := 0
|
|
||||||
|
|
||||||
if opts.ReadAllPacks {
|
if opts.ReadAllPacks {
|
||||||
// get old index files
|
// get old index files
|
||||||
@ -76,7 +75,6 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
|
|||||||
err = repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
|
err = repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
|
||||||
packSizeFromList[id] = size
|
packSizeFromList[id] = size
|
||||||
removePacks.Insert(id)
|
removePacks.Insert(id)
|
||||||
totalPacks++
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -99,7 +97,6 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
|
|||||||
packSizeFromList[id] = packSize
|
packSizeFromList[id] = packSize
|
||||||
removePacks.Insert(id)
|
removePacks.Insert(id)
|
||||||
}
|
}
|
||||||
totalPacks++
|
|
||||||
delete(packSizeFromIndex, id)
|
delete(packSizeFromIndex, id)
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
@ -123,11 +120,10 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
|
|||||||
|
|
||||||
for _, id := range invalidFiles {
|
for _, id := range invalidFiles {
|
||||||
Verboseff("skipped incomplete pack file: %v\n", id)
|
Verboseff("skipped incomplete pack file: %v\n", id)
|
||||||
totalPacks--
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err := rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes, uint64(totalPacks))
|
err := rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -100,13 +100,19 @@ func (mi *MasterIndex) Has(bh restic.BlobHandle) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Packs returns all packs that are covered by the index.
|
// Packs returns all packs that are covered by the index.
|
||||||
func (mi *MasterIndex) Packs() restic.IDSet {
|
// If packBlacklist ist given, those packs are only contained in the
|
||||||
|
// resulting IDSet if they are contained in a non-final index.
|
||||||
|
func (mi *MasterIndex) Packs(packBlacklist restic.IDSet) restic.IDSet {
|
||||||
mi.idxMutex.RLock()
|
mi.idxMutex.RLock()
|
||||||
defer mi.idxMutex.RUnlock()
|
defer mi.idxMutex.RUnlock()
|
||||||
|
|
||||||
packs := restic.NewIDSet()
|
packs := restic.NewIDSet()
|
||||||
for _, idx := range mi.idx {
|
for _, idx := range mi.idx {
|
||||||
packs.Merge(idx.Packs())
|
idxPacks := idx.Packs()
|
||||||
|
if idx.final {
|
||||||
|
idxPacks = idxPacks.Sub(packBlacklist)
|
||||||
|
}
|
||||||
|
packs.Merge(idxPacks)
|
||||||
}
|
}
|
||||||
|
|
||||||
return packs
|
return packs
|
||||||
|
@ -65,7 +65,6 @@ type MasterIndex interface {
|
|||||||
Has(BlobHandle) bool
|
Has(BlobHandle) bool
|
||||||
Lookup(BlobHandle) []PackedBlob
|
Lookup(BlobHandle) []PackedBlob
|
||||||
Count(BlobType) uint
|
Count(BlobType) uint
|
||||||
Packs() IDSet
|
|
||||||
PackSize(ctx context.Context, onlyHdr bool) map[ID]int64
|
PackSize(ctx context.Context, onlyHdr bool) map[ID]int64
|
||||||
|
|
||||||
// Each returns a channel that yields all blobs known to the index. When
|
// Each returns a channel that yields all blobs known to the index. When
|
||||||
|
Loading…
Reference in New Issue
Block a user