2
2
mirror of https://github.com/octoleo/restic.git synced 2024-10-31 19:02:32 +00:00

checker: Properly distinguish between data and tree blobs

If a data blob and a tree blob with the same ID (= same content) exist,
then the checker did not report a data or tree blob as unused when the
blob of the other type was still in use.
This commit is contained in:
Michael Eischer 2020-04-18 19:46:33 +02:00
parent 2d0c138c9b
commit ddf0b8cd0b
4 changed files with 29 additions and 21 deletions

View File

@ -259,7 +259,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
if opts.CheckUnused { if opts.CheckUnused {
for _, id := range chkr.UnusedBlobs() { for _, id := range chkr.UnusedBlobs() {
Verbosef("unused blob %v\n", id.Str()) Verbosef("unused blob %v\n", id)
errorsFound = true errorsFound = true
} }
} }

View File

@ -25,7 +25,7 @@ type Checker struct {
blobRefs struct { blobRefs struct {
sync.Mutex sync.Mutex
// see flags below // see flags below
M map[restic.ID]blobStatus M map[restic.BlobHandle]blobStatus
} }
masterIndex *repository.MasterIndex masterIndex *repository.MasterIndex
@ -36,9 +36,8 @@ type Checker struct {
type blobStatus uint8 type blobStatus uint8
const ( const (
blobExists blobStatus = 1 << iota blobStatusExists blobStatus = 1 << iota
blobReferenced blobStatusReferenced
treeProcessed
) )
// New returns a new checker which runs on repo. // New returns a new checker which runs on repo.
@ -49,7 +48,7 @@ func New(repo restic.Repository) *Checker {
repo: repo, repo: repo,
} }
c.blobRefs.M = make(map[restic.ID]blobStatus) c.blobRefs.M = make(map[restic.BlobHandle]blobStatus)
return c return c
} }
@ -163,7 +162,8 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) {
cnt := 0 cnt := 0
for blob := range res.Index.Each(ctx) { for blob := range res.Index.Each(ctx) {
c.packs.Insert(blob.PackID) c.packs.Insert(blob.PackID)
c.blobRefs.M[blob.ID] = blobExists h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
c.blobRefs.M[h] = blobStatusExists
cnt++ cnt++
if _, ok := packToIndex[blob.PackID]; !ok { if _, ok := packToIndex[blob.PackID]; !ok {
@ -500,9 +500,10 @@ func (c *Checker) filterTrees(ctx context.Context, backlog restic.IDs, loaderCha
// use a separate flag for processed trees to ensure that check still processes trees // use a separate flag for processed trees to ensure that check still processes trees
// even when a file references a tree blob // even when a file references a tree blob
c.blobRefs.Lock() c.blobRefs.Lock()
blobFlags := c.blobRefs.M[nextTreeID] h := restic.BlobHandle{ID: nextTreeID, Type: restic.TreeBlob}
status := c.blobRefs.M[h]
c.blobRefs.Unlock() c.blobRefs.Unlock()
if (blobFlags & treeProcessed) != 0 { if (status & blobStatusReferenced) != 0 {
continue continue
} }
@ -522,7 +523,8 @@ func (c *Checker) filterTrees(ctx context.Context, backlog restic.IDs, loaderCha
outstandingLoadTreeJobs++ outstandingLoadTreeJobs++
loadCh = nil loadCh = nil
c.blobRefs.Lock() c.blobRefs.Lock()
c.blobRefs.M[nextTreeID] |= treeProcessed | blobReferenced h := restic.BlobHandle{ID: nextTreeID, Type: restic.TreeBlob}
c.blobRefs.M[h] |= blobStatusReferenced
c.blobRefs.Unlock() c.blobRefs.Unlock()
case j, ok := <-inCh: case j, ok := <-inCh:
@ -655,11 +657,12 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) {
for _, blobID := range blobs { for _, blobID := range blobs {
c.blobRefs.Lock() c.blobRefs.Lock()
if (c.blobRefs.M[blobID] & blobExists) == 0 { h := restic.BlobHandle{ID: blobID, Type: restic.DataBlob}
if (c.blobRefs.M[h] & blobStatusExists) == 0 {
debug.Log("tree %v references blob %v which isn't contained in index", id, blobID) debug.Log("tree %v references blob %v which isn't contained in index", id, blobID)
errs = append(errs, Error{TreeID: id, BlobID: blobID, Err: errors.New("not found in index")}) errs = append(errs, Error{TreeID: id, BlobID: blobID, Err: errors.New("not found in index")})
} }
c.blobRefs.M[blobID] |= blobReferenced c.blobRefs.M[h] |= blobStatusReferenced
debug.Log("blob %v is referenced", blobID) debug.Log("blob %v is referenced", blobID)
c.blobRefs.Unlock() c.blobRefs.Unlock()
} }
@ -668,13 +671,13 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) {
} }
// UnusedBlobs returns all blobs that have never been referenced. // UnusedBlobs returns all blobs that have never been referenced.
func (c *Checker) UnusedBlobs() (blobs restic.IDs) { func (c *Checker) UnusedBlobs() (blobs restic.BlobHandles) {
c.blobRefs.Lock() c.blobRefs.Lock()
defer c.blobRefs.Unlock() defer c.blobRefs.Unlock()
debug.Log("checking %d blobs", len(c.blobRefs.M)) debug.Log("checking %d blobs", len(c.blobRefs.M))
for id, flags := range c.blobRefs.M { for id, flags := range c.blobRefs.M {
if (flags & blobReferenced) == 0 { if (flags & blobStatusReferenced) == 0 {
debug.Log("blob %v not referenced", id) debug.Log("blob %v not referenced", id)
blobs = append(blobs, id) blobs = append(blobs, id)
} }

View File

@ -157,13 +157,13 @@ func TestUnreferencedBlobs(t *testing.T) {
} }
test.OK(t, repo.Backend().Remove(context.TODO(), snapshotHandle)) test.OK(t, repo.Backend().Remove(context.TODO(), snapshotHandle))
unusedBlobsBySnapshot := restic.IDs{ unusedBlobsBySnapshot := restic.BlobHandles{
restic.TestParseID("58c748bbe2929fdf30c73262bd8313fe828f8925b05d1d4a87fe109082acb849"), restic.TestParseHandle("58c748bbe2929fdf30c73262bd8313fe828f8925b05d1d4a87fe109082acb849", restic.DataBlob),
restic.TestParseID("988a272ab9768182abfd1fe7d7a7b68967825f0b861d3b36156795832c772235"), restic.TestParseHandle("988a272ab9768182abfd1fe7d7a7b68967825f0b861d3b36156795832c772235", restic.DataBlob),
restic.TestParseID("c01952de4d91da1b1b80bc6e06eaa4ec21523f4853b69dc8231708b9b7ec62d8"), restic.TestParseHandle("c01952de4d91da1b1b80bc6e06eaa4ec21523f4853b69dc8231708b9b7ec62d8", restic.TreeBlob),
restic.TestParseID("bec3a53d7dc737f9a9bee68b107ec9e8ad722019f649b34d474b9982c3a3fec7"), restic.TestParseHandle("bec3a53d7dc737f9a9bee68b107ec9e8ad722019f649b34d474b9982c3a3fec7", restic.TreeBlob),
restic.TestParseID("2a6f01e5e92d8343c4c6b78b51c5a4dc9c39d42c04e26088c7614b13d8d0559d"), restic.TestParseHandle("2a6f01e5e92d8343c4c6b78b51c5a4dc9c39d42c04e26088c7614b13d8d0559d", restic.TreeBlob),
restic.TestParseID("18b51b327df9391732ba7aaf841a4885f350d8a557b2da8352c9acf8898e3f10"), restic.TestParseHandle("18b51b327df9391732ba7aaf841a4885f350d8a557b2da8352c9acf8898e3f10", restic.DataBlob),
} }
sort.Sort(unusedBlobsBySnapshot) sort.Sort(unusedBlobsBySnapshot)

View File

@ -200,3 +200,8 @@ func TestParseID(s string) ID {
return id return id
} }
// TestParseHandle parses s as a ID, panics if that fails and creates a BlobHandle with t.
func TestParseHandle(s string, t BlobType) BlobHandle {
return BlobHandle{ID: TestParseID(s), Type: t}
}