mirror of
https://github.com/octoleo/restic.git
synced 2024-10-31 19:02:32 +00:00
checker: Properly distinguish between data and tree blobs
If a data blob and a tree blob with the same ID (= same content) exist, then the checker did not report a data or tree blob as unused when the blob of the other type was still in use.
This commit is contained in:
parent
2d0c138c9b
commit
ddf0b8cd0b
@ -259,7 +259,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
|
|||||||
|
|
||||||
if opts.CheckUnused {
|
if opts.CheckUnused {
|
||||||
for _, id := range chkr.UnusedBlobs() {
|
for _, id := range chkr.UnusedBlobs() {
|
||||||
Verbosef("unused blob %v\n", id.Str())
|
Verbosef("unused blob %v\n", id)
|
||||||
errorsFound = true
|
errorsFound = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -25,7 +25,7 @@ type Checker struct {
|
|||||||
blobRefs struct {
|
blobRefs struct {
|
||||||
sync.Mutex
|
sync.Mutex
|
||||||
// see flags below
|
// see flags below
|
||||||
M map[restic.ID]blobStatus
|
M map[restic.BlobHandle]blobStatus
|
||||||
}
|
}
|
||||||
|
|
||||||
masterIndex *repository.MasterIndex
|
masterIndex *repository.MasterIndex
|
||||||
@ -36,9 +36,8 @@ type Checker struct {
|
|||||||
type blobStatus uint8
|
type blobStatus uint8
|
||||||
|
|
||||||
const (
|
const (
|
||||||
blobExists blobStatus = 1 << iota
|
blobStatusExists blobStatus = 1 << iota
|
||||||
blobReferenced
|
blobStatusReferenced
|
||||||
treeProcessed
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// New returns a new checker which runs on repo.
|
// New returns a new checker which runs on repo.
|
||||||
@ -49,7 +48,7 @@ func New(repo restic.Repository) *Checker {
|
|||||||
repo: repo,
|
repo: repo,
|
||||||
}
|
}
|
||||||
|
|
||||||
c.blobRefs.M = make(map[restic.ID]blobStatus)
|
c.blobRefs.M = make(map[restic.BlobHandle]blobStatus)
|
||||||
|
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
@ -163,7 +162,8 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) {
|
|||||||
cnt := 0
|
cnt := 0
|
||||||
for blob := range res.Index.Each(ctx) {
|
for blob := range res.Index.Each(ctx) {
|
||||||
c.packs.Insert(blob.PackID)
|
c.packs.Insert(blob.PackID)
|
||||||
c.blobRefs.M[blob.ID] = blobExists
|
h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
|
||||||
|
c.blobRefs.M[h] = blobStatusExists
|
||||||
cnt++
|
cnt++
|
||||||
|
|
||||||
if _, ok := packToIndex[blob.PackID]; !ok {
|
if _, ok := packToIndex[blob.PackID]; !ok {
|
||||||
@ -500,9 +500,10 @@ func (c *Checker) filterTrees(ctx context.Context, backlog restic.IDs, loaderCha
|
|||||||
// use a separate flag for processed trees to ensure that check still processes trees
|
// use a separate flag for processed trees to ensure that check still processes trees
|
||||||
// even when a file references a tree blob
|
// even when a file references a tree blob
|
||||||
c.blobRefs.Lock()
|
c.blobRefs.Lock()
|
||||||
blobFlags := c.blobRefs.M[nextTreeID]
|
h := restic.BlobHandle{ID: nextTreeID, Type: restic.TreeBlob}
|
||||||
|
status := c.blobRefs.M[h]
|
||||||
c.blobRefs.Unlock()
|
c.blobRefs.Unlock()
|
||||||
if (blobFlags & treeProcessed) != 0 {
|
if (status & blobStatusReferenced) != 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -522,7 +523,8 @@ func (c *Checker) filterTrees(ctx context.Context, backlog restic.IDs, loaderCha
|
|||||||
outstandingLoadTreeJobs++
|
outstandingLoadTreeJobs++
|
||||||
loadCh = nil
|
loadCh = nil
|
||||||
c.blobRefs.Lock()
|
c.blobRefs.Lock()
|
||||||
c.blobRefs.M[nextTreeID] |= treeProcessed | blobReferenced
|
h := restic.BlobHandle{ID: nextTreeID, Type: restic.TreeBlob}
|
||||||
|
c.blobRefs.M[h] |= blobStatusReferenced
|
||||||
c.blobRefs.Unlock()
|
c.blobRefs.Unlock()
|
||||||
|
|
||||||
case j, ok := <-inCh:
|
case j, ok := <-inCh:
|
||||||
@ -655,11 +657,12 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) {
|
|||||||
|
|
||||||
for _, blobID := range blobs {
|
for _, blobID := range blobs {
|
||||||
c.blobRefs.Lock()
|
c.blobRefs.Lock()
|
||||||
if (c.blobRefs.M[blobID] & blobExists) == 0 {
|
h := restic.BlobHandle{ID: blobID, Type: restic.DataBlob}
|
||||||
|
if (c.blobRefs.M[h] & blobStatusExists) == 0 {
|
||||||
debug.Log("tree %v references blob %v which isn't contained in index", id, blobID)
|
debug.Log("tree %v references blob %v which isn't contained in index", id, blobID)
|
||||||
errs = append(errs, Error{TreeID: id, BlobID: blobID, Err: errors.New("not found in index")})
|
errs = append(errs, Error{TreeID: id, BlobID: blobID, Err: errors.New("not found in index")})
|
||||||
}
|
}
|
||||||
c.blobRefs.M[blobID] |= blobReferenced
|
c.blobRefs.M[h] |= blobStatusReferenced
|
||||||
debug.Log("blob %v is referenced", blobID)
|
debug.Log("blob %v is referenced", blobID)
|
||||||
c.blobRefs.Unlock()
|
c.blobRefs.Unlock()
|
||||||
}
|
}
|
||||||
@ -668,13 +671,13 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// UnusedBlobs returns all blobs that have never been referenced.
|
// UnusedBlobs returns all blobs that have never been referenced.
|
||||||
func (c *Checker) UnusedBlobs() (blobs restic.IDs) {
|
func (c *Checker) UnusedBlobs() (blobs restic.BlobHandles) {
|
||||||
c.blobRefs.Lock()
|
c.blobRefs.Lock()
|
||||||
defer c.blobRefs.Unlock()
|
defer c.blobRefs.Unlock()
|
||||||
|
|
||||||
debug.Log("checking %d blobs", len(c.blobRefs.M))
|
debug.Log("checking %d blobs", len(c.blobRefs.M))
|
||||||
for id, flags := range c.blobRefs.M {
|
for id, flags := range c.blobRefs.M {
|
||||||
if (flags & blobReferenced) == 0 {
|
if (flags & blobStatusReferenced) == 0 {
|
||||||
debug.Log("blob %v not referenced", id)
|
debug.Log("blob %v not referenced", id)
|
||||||
blobs = append(blobs, id)
|
blobs = append(blobs, id)
|
||||||
}
|
}
|
||||||
|
@ -157,13 +157,13 @@ func TestUnreferencedBlobs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
test.OK(t, repo.Backend().Remove(context.TODO(), snapshotHandle))
|
test.OK(t, repo.Backend().Remove(context.TODO(), snapshotHandle))
|
||||||
|
|
||||||
unusedBlobsBySnapshot := restic.IDs{
|
unusedBlobsBySnapshot := restic.BlobHandles{
|
||||||
restic.TestParseID("58c748bbe2929fdf30c73262bd8313fe828f8925b05d1d4a87fe109082acb849"),
|
restic.TestParseHandle("58c748bbe2929fdf30c73262bd8313fe828f8925b05d1d4a87fe109082acb849", restic.DataBlob),
|
||||||
restic.TestParseID("988a272ab9768182abfd1fe7d7a7b68967825f0b861d3b36156795832c772235"),
|
restic.TestParseHandle("988a272ab9768182abfd1fe7d7a7b68967825f0b861d3b36156795832c772235", restic.DataBlob),
|
||||||
restic.TestParseID("c01952de4d91da1b1b80bc6e06eaa4ec21523f4853b69dc8231708b9b7ec62d8"),
|
restic.TestParseHandle("c01952de4d91da1b1b80bc6e06eaa4ec21523f4853b69dc8231708b9b7ec62d8", restic.TreeBlob),
|
||||||
restic.TestParseID("bec3a53d7dc737f9a9bee68b107ec9e8ad722019f649b34d474b9982c3a3fec7"),
|
restic.TestParseHandle("bec3a53d7dc737f9a9bee68b107ec9e8ad722019f649b34d474b9982c3a3fec7", restic.TreeBlob),
|
||||||
restic.TestParseID("2a6f01e5e92d8343c4c6b78b51c5a4dc9c39d42c04e26088c7614b13d8d0559d"),
|
restic.TestParseHandle("2a6f01e5e92d8343c4c6b78b51c5a4dc9c39d42c04e26088c7614b13d8d0559d", restic.TreeBlob),
|
||||||
restic.TestParseID("18b51b327df9391732ba7aaf841a4885f350d8a557b2da8352c9acf8898e3f10"),
|
restic.TestParseHandle("18b51b327df9391732ba7aaf841a4885f350d8a557b2da8352c9acf8898e3f10", restic.DataBlob),
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Sort(unusedBlobsBySnapshot)
|
sort.Sort(unusedBlobsBySnapshot)
|
||||||
|
@ -200,3 +200,8 @@ func TestParseID(s string) ID {
|
|||||||
|
|
||||||
return id
|
return id
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestParseHandle parses s as a ID, panics if that fails and creates a BlobHandle with t.
|
||||||
|
func TestParseHandle(s string, t BlobType) BlobHandle {
|
||||||
|
return BlobHandle{ID: TestParseID(s), Type: t}
|
||||||
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user