Merge pull request #476 from restic/fix-475

Ignore invalid index files

src/cmds/restic/cmd_dump.go

@@ -105,7 +105,7 @@ func (cmd CmdDump) DumpIndexes() error {
 	for id := range cmd.repo.List(backend.Index, done) {
 		fmt.Printf("index_id: %v\n", id)
 
-		idx, err := repository.LoadIndex(cmd.repo, id.String())
+		idx, err := repository.LoadIndex(cmd.repo, id)
 		if err != nil {
 			return err
 		}

src/cmds/restic/cmd_rebuild_index.go

@@ -71,7 +71,7 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
 		cmd.global.Printf("  loading index %v\n", i)
 
 		debug.Log("RebuildIndex.RebuildIndex", "load index %v", indexID.Str())
-		idx, err := repository.LoadIndex(cmd.repo, indexID.String())
+		idx, err := repository.LoadIndex(cmd.repo, indexID)
 		if err != nil {
 			return err
 		}

src/restic/backend/local/local.go

@@ -235,31 +235,84 @@ func (b *Local) Remove(t backend.Type, name string) error {
 	return os.Remove(fn)
 }
 
+func isFile(fi os.FileInfo) bool {
+	return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0
+}
+
+func readdir(d string) (fileInfos []os.FileInfo, err error) {
+	f, e := os.Open(d)
+	if e != nil {
+		return nil, e
+	}
+
+	defer func() {
+		e := f.Close()
+		if err == nil {
+			err = e
+		}
+	}()
+
+	return f.Readdir(-1)
+}
+
+// listDir returns a list of all files in d.
+func listDir(d string) (filenames []string, err error) {
+	fileInfos, err := readdir(d)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, fi := range fileInfos {
+		if isFile(fi) {
+			filenames = append(filenames, fi.Name())
+		}
+	}
+
+	return filenames, nil
+}
+
+// listDirs returns a list of all files in directories within d.
+func listDirs(dir string) (filenames []string, err error) {
+	fileInfos, err := readdir(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, fi := range fileInfos {
+		if !fi.IsDir() {
+			continue
+		}
+
+		files, err := listDir(filepath.Join(dir, fi.Name()))
+		if err != nil {
+			continue
+		}
+
+		filenames = append(filenames, files...)
+	}
+
+	return filenames, nil
+}
+
 // List returns a channel that yields all names of blobs of type t. A
 // goroutine is started for this. If the channel done is closed, sending
 // stops.
 func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string {
-	var pattern string
+	lister := listDir
 	if t == backend.Data {
-		pattern = filepath.Join(dirname(b.p, t, ""), "*", "*")
-	} else {
-		pattern = filepath.Join(dirname(b.p, t, ""), "*")
+		lister = listDirs
 	}
 
 	ch := make(chan string)
-	matches, err := filepath.Glob(pattern)
+	items, err := lister(filepath.Join(dirname(b.p, t, "")))
 	if err != nil {
 		close(ch)
 		return ch
 	}
 
-	for i := range matches {
-		matches[i] = filepath.Base(matches[i])
-	}
-
 	go func() {
 		defer close(ch)
-		for _, m := range matches {
+		for _, m := range items {
 			if m == "" {
 				continue
 			}

src/restic/checker/checker.go

@@ -83,12 +83,12 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
 
 	worker := func(id backend.ID, done <-chan struct{}) error {
 		debug.Log("LoadIndex", "worker got index %v", id)
-		idx, err := repository.LoadIndexWithDecoder(c.repo, id.String(), repository.DecodeIndex)
+		idx, err := repository.LoadIndexWithDecoder(c.repo, id, repository.DecodeIndex)
 		if err == repository.ErrOldIndexFormat {
 			debug.Log("LoadIndex", "index %v has old format", id.Str())
 			hints = append(hints, ErrOldIndexFormat{id})
 
-			idx, err = repository.LoadIndexWithDecoder(c.repo, id.String(), repository.DecodeOldIndex)
+			idx, err = repository.LoadIndexWithDecoder(c.repo, id, repository.DecodeOldIndex)
 		}
 
 		if err != nil {

src/restic/repository/index.go

@@ -557,15 +557,10 @@ func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
 }
 
 // LoadIndexWithDecoder loads the index and decodes it with fn.
-func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Index, error)) (idx *Index, err error) {
+func LoadIndexWithDecoder(repo *Repository, id backend.ID, fn func(io.Reader) (*Index, error)) (idx *Index, err error) {
 	debug.Log("LoadIndexWithDecoder", "Loading index %v", id[:8])
 
-	idxID, err := backend.ParseID(id)
-	if err != nil {
-		return nil, err
-	}
-
-	buf, err := repo.LoadAndDecrypt(backend.Index, idxID)
+	buf, err := repo.LoadAndDecrypt(backend.Index, id)
 	if err != nil {
 		return nil, err
 	}
@@ -576,7 +571,7 @@ func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Inde
 		return nil, err
 	}
 
-	idx.id = idxID
+	idx.id = id
 
 	return idx, nil
 }
@@ -588,7 +583,7 @@ func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Inde
 func ConvertIndex(repo *Repository, id backend.ID) (backend.ID, error) {
 	debug.Log("ConvertIndex", "checking index %v", id.Str())
 
-	idx, err := LoadIndexWithDecoder(repo, id.String(), DecodeOldIndex)
+	idx, err := LoadIndexWithDecoder(repo, id, DecodeOldIndex)
 	if err != nil {
 		debug.Log("ConvertIndex", "LoadIndexWithDecoder(%v) returned error: %v", id.Str(), err)
 		return id, err

src/restic/repository/parallel.go

@@ -4,6 +4,7 @@ import (
 	"sync"
 
 	"restic/backend"
+	"restic/debug"
 )
 
 func closeIfOpen(ch chan struct{}) {
@@ -75,12 +76,14 @@ func FilesInParallel(repo backend.Lister, t backend.Type, n uint, f ParallelWork
 }
 
 // ParallelWorkFuncParseID converts a function that takes a backend.ID to a
-// function that takes a string.
+// function that takes a string. Filenames that do not parse as a backend.ID
+// are ignored.
 func ParallelWorkFuncParseID(f ParallelIDWorkFunc) ParallelWorkFunc {
 	return func(s string, done <-chan struct{}) error {
 		id, err := backend.ParseID(s)
 		if err != nil {
-			return err
+			debug.Log("repository.ParallelWorkFuncParseID", "invalid ID %q: %v", id, err)
+			return nil
 		}
 
 		return f(id, done)

src/restic/repository/repository.go

@@ -366,7 +366,7 @@ func (r *Repository) LoadIndex() error {
 	errCh := make(chan error, 1)
 	indexes := make(chan *Index)
 
-	worker := func(id string, done <-chan struct{}) error {
+	worker := func(id backend.ID, done <-chan struct{}) error {
 		idx, err := LoadIndex(r, id)
 		if err != nil {
 			return err
@@ -382,7 +382,8 @@ func (r *Repository) LoadIndex() error {
 
 	go func() {
 		defer close(indexes)
-		errCh <- FilesInParallel(r.be, backend.Index, loadIndexParallelism, worker)
+		errCh <- FilesInParallel(r.be, backend.Index, loadIndexParallelism,
+			ParallelWorkFuncParseID(worker))
 	}()
 
 	for idx := range indexes {
@@ -397,14 +398,14 @@ func (r *Repository) LoadIndex() error {
 }
 
 // LoadIndex loads the index id from backend and returns it.
-func LoadIndex(repo *Repository, id string) (*Index, error) {
+func LoadIndex(repo *Repository, id backend.ID) (*Index, error) {
 	idx, err := LoadIndexWithDecoder(repo, id, DecodeIndex)
 	if err == nil {
 		return idx, nil
 	}
 
 	if err == ErrOldIndexFormat {
-		fmt.Fprintf(os.Stderr, "index %v has old format\n", id[:10])
+		fmt.Fprintf(os.Stderr, "index %v has old format\n", id.Str())
 		return LoadIndexWithDecoder(repo, id, DecodeOldIndex)
 	}
 

src/restic/repository/repository_test.go

@@ -292,7 +292,7 @@ func TestRepositoryIncrementalIndex(t *testing.T) {
 	packEntries := make(map[backend.ID]map[backend.ID]struct{})
 
 	for id := range repo.List(backend.Index, nil) {
-		idx, err := repository.LoadIndex(repo, id.String())
+		idx, err := repository.LoadIndex(repo, id)
 		OK(t, err)
 
 		for pb := range idx.Each(nil) {