octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-26 06:46:34 +00:00
Code Issues Releases Wiki Activity

prune: Remove invalid files

Browse Source 
Closes #1029
This commit is contained in:
Alexander Neumann 2017-06-15 15:03:05 +02:00
parent eadf5dcb2d
commit 4ae59bef96
5 changed files with 32 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/cmds/restic/cmd_prune.go
View File

@ -106,11 +106,15 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
Verbosef("building new index for repo\n")
bar := newProgressMax(!gopts.Quiet, uint64(stats.packs), "packs")
idx, err := index.New(ctx, repo, restic.NewIDSet(), bar)
idx, invalidFiles, err := index.New(ctx, repo, restic.NewIDSet(), bar)
if err != nil {
return err
}
for _, id := range invalidFiles {
Warnf("incomplete pack file (will be removed): %v\n", id)
}
blobs := 0
for _, pack := range idx.Packs {
stats.bytes += pack.Size
@ -196,6 +200,12 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
// find packs that are unneeded
removePacks := restic.NewIDSet()
Verbosef("will remove %d invalid files\n", len(invalidFiles))
for _, id := range invalidFiles {
removePacks.Insert(id)
}
for packID, p := range idx.Packs {
hasActiveBlob := false

2
src/cmds/restic/cmd_rebuild_index.go
View File

@ -50,7 +50,7 @@ func rebuildIndex(ctx context.Context, repo restic.Repository, ignorePacks resti
}
bar := newProgressMax(!globalOptions.Quiet, packs, "packs")
idx, err := index.New(ctx, repo, ignorePacks, bar)
idx, _, err := index.New(ctx, repo, ignorePacks, bar)
if err != nil {
return err
}

20
src/restic/index/index.go
View File

@ -8,6 +8,7 @@ import (
"restic"
"restic/debug"
"restic/list"
"restic/pack"
"restic/worker"
"restic/errors"
@ -33,22 +34,29 @@ func newIndex() *Index {
}
}
// New creates a new index for repo from scratch.
func New(ctx context.Context, repo restic.Repository, ignorePacks restic.IDSet, p *restic.Progress) (*Index, error) {
// New creates a new index for repo from scratch. InvalidFiles contains all IDs
// of files that cannot be listed successfully.
func New(ctx context.Context, repo restic.Repository, ignorePacks restic.IDSet, p *restic.Progress) (idx *Index, invalidFiles restic.IDs, err error) {
p.Start()
defer p.Done()
ch := make(chan worker.Job)
go list.AllPacks(ctx, repo, ignorePacks, ch)
idx := newIndex()
idx = newIndex()
for job := range ch {
p.Report(restic.Stat{Blobs: 1})
packID := job.Data.(restic.ID)
if job.Error != nil {
fmt.Fprintf(os.Stderr, "unable to list pack %v: %v\n", packID.Str(), job.Error)
cause := errors.Cause(job.Error)
if _, ok := cause.(pack.InvalidFileError); ok {
invalidFiles = append(invalidFiles, packID)
continue
}
fmt.Fprintf(os.Stderr, "pack file cannot be listed %v: %v\n", packID.Str(), job.Error)
continue
}
@ -58,11 +66,11 @@ func New(ctx context.Context, repo restic.Repository, ignorePacks restic.IDSet,
err := idx.AddPack(packID, j.Size(), j.Entries())
if err != nil {
return nil, err
return nil, nil, err
}
}
return idx, nil
return idx, invalidFiles, nil
}
type packJSON struct {

10
src/restic/index/index_test.go
View File

@ -43,7 +43,7 @@ func TestIndexNew(t *testing.T) {
repo, cleanup := createFilledRepo(t, 3, 0)
defer cleanup()
idx, err := New(context.TODO(), repo, restic.NewIDSet(), nil)
idx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil)
if err != nil {
t.Fatalf("New() returned error %v", err)
}
@ -70,7 +70,7 @@ func TestIndexLoad(t *testing.T) {
validateIndex(t, repo, loadIdx)
newIdx, err := New(context.TODO(), repo, restic.NewIDSet(), nil)
newIdx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil)
if err != nil {
t.Fatalf("New() returned error %v", err)
}
@ -134,7 +134,7 @@ func BenchmarkIndexNew(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
idx, err := New(context.TODO(), repo, restic.NewIDSet(), nil)
idx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil)
if err != nil {
b.Fatalf("New() returned error %v", err)
@ -151,7 +151,7 @@ func BenchmarkIndexSave(b *testing.B) {
repo, cleanup := repository.TestRepository(b)
defer cleanup()
idx, err := New(context.TODO(), repo, restic.NewIDSet(), nil)
idx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil)
test.OK(b, err)
for i := 0; i < 8000; i++ {
@ -184,7 +184,7 @@ func TestIndexDuplicateBlobs(t *testing.T) {
repo, cleanup := createFilledRepo(t, 3, 0.01)
defer cleanup()
idx, err := New(context.TODO(), repo, restic.NewIDSet(), nil)
idx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil)
if err != nil {
t.Fatal(err)
}

2
src/restic/repository/repack_test.go
View File

@ -147,7 +147,7 @@ func saveIndex(t *testing.T, repo restic.Repository) {
}
func rebuildIndex(t *testing.T, repo restic.Repository) {
idx, err := index.New(context.TODO(), repo, restic.NewIDSet(), nil)
idx, _, err := index.New(context.TODO(), repo, restic.NewIDSet(), nil)
if err != nil {
t.Fatal(err)
}