From 4ae59bef966c1946298796696e01bb0d9f5af95e Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 15 Jun 2017 15:03:05 +0200 Subject: [PATCH 1/2] prune: Remove invalid files Closes #1029 --- src/cmds/restic/cmd_prune.go | 12 +++++++++++- src/cmds/restic/cmd_rebuild_index.go | 2 +- src/restic/index/index.go | 20 ++++++++++++++------ src/restic/index/index_test.go | 10 +++++----- src/restic/repository/repack_test.go | 2 +- 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/cmds/restic/cmd_prune.go b/src/cmds/restic/cmd_prune.go index d2a77be24..e841c7d38 100644 --- a/src/cmds/restic/cmd_prune.go +++ b/src/cmds/restic/cmd_prune.go @@ -106,11 +106,15 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error { Verbosef("building new index for repo\n") bar := newProgressMax(!gopts.Quiet, uint64(stats.packs), "packs") - idx, err := index.New(ctx, repo, restic.NewIDSet(), bar) + idx, invalidFiles, err := index.New(ctx, repo, restic.NewIDSet(), bar) if err != nil { return err } + for _, id := range invalidFiles { + Warnf("incomplete pack file (will be removed): %v\n", id) + } + blobs := 0 for _, pack := range idx.Packs { stats.bytes += pack.Size @@ -196,6 +200,12 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error { // find packs that are unneeded removePacks := restic.NewIDSet() + + Verbosef("will remove %d invalid files\n", len(invalidFiles)) + for _, id := range invalidFiles { + removePacks.Insert(id) + } + for packID, p := range idx.Packs { hasActiveBlob := false diff --git a/src/cmds/restic/cmd_rebuild_index.go b/src/cmds/restic/cmd_rebuild_index.go index 9f3cc888c..12604c704 100644 --- a/src/cmds/restic/cmd_rebuild_index.go +++ b/src/cmds/restic/cmd_rebuild_index.go @@ -50,7 +50,7 @@ func rebuildIndex(ctx context.Context, repo restic.Repository, ignorePacks resti } bar := newProgressMax(!globalOptions.Quiet, packs, "packs") - idx, err := index.New(ctx, repo, ignorePacks, bar) + idx, _, err := index.New(ctx, repo, ignorePacks, bar) if err != nil { return err } diff --git a/src/restic/index/index.go b/src/restic/index/index.go index be528efc0..81f0357cf 100644 --- a/src/restic/index/index.go +++ b/src/restic/index/index.go @@ -8,6 +8,7 @@ import ( "restic" "restic/debug" "restic/list" + "restic/pack" "restic/worker" "restic/errors" @@ -33,22 +34,29 @@ func newIndex() *Index { } } -// New creates a new index for repo from scratch. -func New(ctx context.Context, repo restic.Repository, ignorePacks restic.IDSet, p *restic.Progress) (*Index, error) { +// New creates a new index for repo from scratch. InvalidFiles contains all IDs +// of files that cannot be listed successfully. +func New(ctx context.Context, repo restic.Repository, ignorePacks restic.IDSet, p *restic.Progress) (idx *Index, invalidFiles restic.IDs, err error) { p.Start() defer p.Done() ch := make(chan worker.Job) go list.AllPacks(ctx, repo, ignorePacks, ch) - idx := newIndex() + idx = newIndex() for job := range ch { p.Report(restic.Stat{Blobs: 1}) packID := job.Data.(restic.ID) if job.Error != nil { - fmt.Fprintf(os.Stderr, "unable to list pack %v: %v\n", packID.Str(), job.Error) + cause := errors.Cause(job.Error) + if _, ok := cause.(pack.InvalidFileError); ok { + invalidFiles = append(invalidFiles, packID) + continue + } + + fmt.Fprintf(os.Stderr, "pack file cannot be listed %v: %v\n", packID.Str(), job.Error) continue } @@ -58,11 +66,11 @@ func New(ctx context.Context, repo restic.Repository, ignorePacks restic.IDSet, err := idx.AddPack(packID, j.Size(), j.Entries()) if err != nil { - return nil, err + return nil, nil, err } } - return idx, nil + return idx, invalidFiles, nil } type packJSON struct { diff --git a/src/restic/index/index_test.go b/src/restic/index/index_test.go index 81c3d6e4b..a469ee443 100644 --- a/src/restic/index/index_test.go +++ b/src/restic/index/index_test.go @@ -43,7 +43,7 @@ func TestIndexNew(t *testing.T) { repo, cleanup := createFilledRepo(t, 3, 0) defer cleanup() - idx, err := New(context.TODO(), repo, restic.NewIDSet(), nil) + idx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil) if err != nil { t.Fatalf("New() returned error %v", err) } @@ -70,7 +70,7 @@ func TestIndexLoad(t *testing.T) { validateIndex(t, repo, loadIdx) - newIdx, err := New(context.TODO(), repo, restic.NewIDSet(), nil) + newIdx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil) if err != nil { t.Fatalf("New() returned error %v", err) } @@ -134,7 +134,7 @@ func BenchmarkIndexNew(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - idx, err := New(context.TODO(), repo, restic.NewIDSet(), nil) + idx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil) if err != nil { b.Fatalf("New() returned error %v", err) @@ -151,7 +151,7 @@ func BenchmarkIndexSave(b *testing.B) { repo, cleanup := repository.TestRepository(b) defer cleanup() - idx, err := New(context.TODO(), repo, restic.NewIDSet(), nil) + idx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil) test.OK(b, err) for i := 0; i < 8000; i++ { @@ -184,7 +184,7 @@ func TestIndexDuplicateBlobs(t *testing.T) { repo, cleanup := createFilledRepo(t, 3, 0.01) defer cleanup() - idx, err := New(context.TODO(), repo, restic.NewIDSet(), nil) + idx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil) if err != nil { t.Fatal(err) } diff --git a/src/restic/repository/repack_test.go b/src/restic/repository/repack_test.go index fa64f8eff..0a4d1f617 100644 --- a/src/restic/repository/repack_test.go +++ b/src/restic/repository/repack_test.go @@ -147,7 +147,7 @@ func saveIndex(t *testing.T, repo restic.Repository) { } func rebuildIndex(t *testing.T, repo restic.Repository) { - idx, err := index.New(context.TODO(), repo, restic.NewIDSet(), nil) + idx, _, err := index.New(context.TODO(), repo, restic.NewIDSet(), nil) if err != nil { t.Fatal(err) } From 1baaa778ee3b43f0e696200782efc4ccac42539e Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Fri, 16 Jun 2017 12:27:44 +0200 Subject: [PATCH 2/2] Add entry to CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 569d26bd1..d000b8772 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,12 @@ Small changes https://github.com/restic/restic/issues/1023 https://github.com/restic/restic/pull/1025 + * The `prune` command has been improved and will now remove invalid pack + files, for example files that have not been uploaded completely because a + backup was interrupted. + https://github.com/restic/restic/issues/1029 + https://github.com/restic/restic/pull/1036 + Important Changes in 0.6.1 ==========================