diff --git a/changelog/unreleased/pull-3488 b/changelog/unreleased/pull-3488 new file mode 100644 index 000000000..5157448f5 --- /dev/null +++ b/changelog/unreleased/pull-3488 @@ -0,0 +1,7 @@ +Bugfix: rebuild-index failed if an index file was damaged + +The `rebuild-index` command failed with an error if an index file was damaged +or truncated. This has been fixed. A (slow) workaround is to use +`rebuild-index --read-all-packs` or to manually delete the damaged index. + +https://github.com/restic/restic/pull/3488 diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_rebuild_index.go index 707309e64..718d2c767 100644 --- a/cmd/restic/cmd_rebuild_index.go +++ b/cmd/restic/cmd_rebuild_index.go @@ -73,7 +73,27 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor } } else { Verbosef("loading indexes...\n") - err := repo.LoadIndex(gopts.ctx) + mi := repository.NewMasterIndex() + err := repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error { + if err != nil { + Warnf("removing invalid index %v: %v\n", id, err) + obsoleteIndexes = append(obsoleteIndexes, id) + return nil + } + + mi.Insert(idx) + return nil + }) + if err != nil { + return err + } + + err = mi.MergeFinalIndexes() + if err != nil { + return err + } + + err = repo.SetIndex(mi) if err != nil { return err } diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index ecd70b075..09269c575 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -15,6 +15,7 @@ import ( "regexp" "runtime" "strings" + "sync" "syscall" "testing" "time" @@ -1416,7 +1417,7 @@ func TestFindJSON(t *testing.T) { rtest.Assert(t, matches[0].Hits == 3, "expected hits to show 3 matches (%v)", datafile) } -func TestRebuildIndex(t *testing.T) { +func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) { env, cleanup := withTestEnvironment(t) defer cleanup() @@ -1436,8 +1437,10 @@ func TestRebuildIndex(t *testing.T) { t.Fatalf("did not find hint for rebuild-index command") } + env.gopts.backendTestHook = backendTestHook testRunRebuildIndex(t, env.gopts) + env.gopts.backendTestHook = nil out, err = testRunCheckOutput(env.gopts) if len(out) != 0 { t.Fatalf("expected no output from the checker, got: %v", out) @@ -1448,9 +1451,57 @@ func TestRebuildIndex(t *testing.T) { } } +func TestRebuildIndex(t *testing.T) { + testRebuildIndex(t, nil) +} + func TestRebuildIndexAlwaysFull(t *testing.T) { + indexFull := repository.IndexFull + defer func() { + repository.IndexFull = indexFull + }() repository.IndexFull = func(*repository.Index) bool { return true } - TestRebuildIndex(t) + testRebuildIndex(t, nil) +} + +// indexErrorBackend modifies the first index after reading. +type indexErrorBackend struct { + restic.Backend + lock sync.Mutex + hasErred bool +} + +func (b *indexErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { + return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { + // protect hasErred + b.lock.Lock() + defer b.lock.Unlock() + if !b.hasErred && h.Type == restic.IndexFile { + b.hasErred = true + return consumer(errorReadCloser{rd}) + } + return consumer(rd) + }) +} + +type errorReadCloser struct { + io.Reader +} + +func (erd errorReadCloser) Read(p []byte) (int, error) { + n, err := erd.Reader.Read(p) + if n > 0 { + p[0] ^= 1 + } + return n, err +} + +func TestRebuildIndexDamage(t *testing.T) { + testRebuildIndex(t, func(r restic.Backend) (restic.Backend, error) { + return &indexErrorBackend{ + Backend: r, + }, nil + }) } type appendOnlyBackend struct {