Merge pull request #194 from restic/fix-orphaned-check

Fix fsck orphaned check, add debug command 'dump', cleanup

archiver.go

@@ -254,7 +254,7 @@ func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *Progress, done <-chan st
 				// check if all content is still available in the repository
 				contentMissing := false
 				for _, blob := range oldNode.blobs {
-					if ok, err := arch.repo.Test(backend.Data, blob.Storage.String()); !ok || err != nil {
+					if ok, err := arch.repo.Backend().Test(backend.Data, blob.Storage.String()); !ok || err != nil {
 						debug.Log("Archiver.fileWorker", "   %v not using old data, %v (%v) is missing", e.Path(), blob.ID.Str(), blob.Storage.Str())
 						contentMissing = true
 						break

backend/generic.go

@@ -56,18 +56,6 @@ func Find(be Lister, t Type, prefix string) (string, error) {
 	return "", ErrNoIDPrefixFound
 }
 
-// FindSnapshot takes a string and tries to find a snapshot whose ID matches
-// the string as closely as possible.
-func FindSnapshot(be Lister, s string) (string, error) {
-	// find snapshot id with prefix
-	name, err := Find(be, Snapshot, s)
-	if err != nil {
-		return "", err
-	}
-
-	return name, nil
-}
-
 // PrefixLength returns the number of bytes required so that all prefixes of
 // all names of type t are unique.
 func PrefixLength(be Lister, t Type) (int, error) {

cache.go

@@ -115,7 +115,7 @@ func (c *Cache) Clear(repo *repository.Repository) error {
 	for _, entry := range list {
 		debug.Log("Cache.Clear", "found entry %v", entry)
 
-		if ok, err := repo.Test(backend.Snapshot, entry.ID.String()); !ok || err != nil {
+		if ok, err := repo.Backend().Test(backend.Snapshot, entry.ID.String()); !ok || err != nil {
 			debug.Log("Cache.Clear", "snapshot %v doesn't exist any more, removing %v", entry.ID, entry)
 
 			err = c.purge(backend.Snapshot, entry.Subtype, entry.ID)

cmd/restic/cmd_backup.go

@@ -185,28 +185,20 @@ func (cmd CmdBackup) Execute(args []string) error {
 		return err
 	}
 
-	var (
+	var parentSnapshotID backend.ID
-		parentSnapshot   string
-		parentSnapshotID backend.ID
-	)
 
 	// Force using a parent
 	if !cmd.Force && cmd.Parent != "" {
-		parentSnapshot, err = s.FindSnapshot(cmd.Parent)
+		parentSnapshotID, err = restic.FindSnapshot(s, cmd.Parent)
 		if err != nil {
 			return fmt.Errorf("invalid id %q: %v", cmd.Parent, err)
 		}
 
-		parentSnapshotID, err = backend.ParseID(parentSnapshot)
-		if err != nil {
-			return fmt.Errorf("invalid parent snapshot id %v", parentSnapshot)
-		}
-
 		fmt.Printf("found parent snapshot %v\n", parentSnapshotID)
 	}
 
 	// Find last snapshot to set it as parent, if not already set
-	if !cmd.Force && parentSnapshot == "" {
+	if !cmd.Force && parentSnapshotID == nil {
 		samePaths := func(expected, actual []string) bool {
 			if len(expected) != len(actual) {
 				return false
@@ -221,24 +213,19 @@ func (cmd CmdBackup) Execute(args []string) error {
 
 		var latest time.Time
 
-		for snapshotIDString := range s.List(backend.Snapshot, make(chan struct{})) {
+		for id := range s.List(backend.Snapshot, make(chan struct{})) {
-			snapshotID, err := backend.ParseID(snapshotIDString)
+			snapshot, err := restic.LoadSnapshot(s, id)
-			if err != nil {
-				return fmt.Errorf("Error with the listing of snapshots inputting invalid backend ids: %v", err)
-			}
-
-			snapshot, err := restic.LoadSnapshot(s, snapshotID)
 			if err != nil {
 				return fmt.Errorf("Error listing snapshot: %v", err)
 			}
 			if snapshot.Time.After(latest) && samePaths(snapshot.Paths, target) {
 				latest = snapshot.Time
-				parentSnapshotID = snapshotID
+				parentSnapshotID = id
-				parentSnapshot = snapshotIDString
 			}
 
 		}
-		if parentSnapshot != "" {
+
+		if parentSnapshotID != nil {
 			fmt.Printf("using parent snapshot %v\n", parentSnapshotID)
 		}
 	}

cmd/restic/cmd_cat.go

@@ -53,12 +53,7 @@ func (cmd CmdCat) Execute(args []string) error {
 			}
 
 			// find snapshot id with prefix
-			name, err := s.FindSnapshot(args[1])
+			id, err = restic.FindSnapshot(s, args[1])
-			if err != nil {
-				return err
-			}
-
-			id, err = backend.ParseID(name)
 			if err != nil {
 				return err
 			}

cmd/restic/cmd_dump.go

@@ -0,0 +1,149 @@
+// +build debug
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/juju/errors"
+	"github.com/restic/restic"
+	"github.com/restic/restic/backend"
+	"github.com/restic/restic/pack"
+	"github.com/restic/restic/repository"
+)
+
+type CmdDump struct{}
+
+func init() {
+	_, err := parser.AddCommand("dump",
+		"dump data structures",
+		"The dump command dumps data structures from a repository as JSON documents",
+		&CmdDump{})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func dumpIndex(r *repository.Repository, wr io.Writer) error {
+	fmt.Fprintln(wr, "foo")
+	return nil
+}
+
+func (cmd CmdDump) Usage() string {
+	return "[index|snapshots|trees|all]"
+}
+
+func prettyPrintJSON(wr io.Writer, item interface{}) error {
+	buf, err := json.MarshalIndent(item, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	_, err = wr.Write(append(buf, '\n'))
+	return err
+}
+
+func printSnapshots(repo *repository.Repository, wr io.Writer) error {
+	done := make(chan struct{})
+	defer close(done)
+
+	for id := range repo.List(backend.Snapshot, done) {
+		snapshot, err := restic.LoadSnapshot(repo, id)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "LoadSnapshot(%v): %v", id.Str(), err)
+			continue
+		}
+
+		fmt.Fprintf(wr, "snapshot_id: %v\n", id)
+
+		err = prettyPrintJSON(wr, snapshot)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func printTrees(repo *repository.Repository, wr io.Writer) error {
+	done := make(chan struct{})
+	defer close(done)
+
+	trees := []backend.ID{}
+
+	for blob := range repo.Index().Each(done) {
+		if blob.Type != pack.Tree {
+			continue
+		}
+
+		trees = append(trees, blob.ID)
+	}
+
+	for _, id := range trees {
+		tree, err := restic.LoadTree(repo, id)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "LoadTree(%v): %v", id.Str(), err)
+			continue
+		}
+
+		fmt.Fprintf(wr, "tree_id: %v\n", id)
+
+		prettyPrintJSON(wr, tree)
+	}
+
+	return nil
+}
+
+func (cmd CmdDump) Execute(args []string) error {
+	if len(args) != 1 {
+		return fmt.Errorf("type not specified, Usage: %s", cmd.Usage())
+	}
+
+	repo, err := OpenRepo()
+	if err != nil {
+		return err
+	}
+
+	err = repo.LoadIndex()
+	if err != nil {
+		return err
+	}
+
+	tpe := args[0]
+
+	switch tpe {
+	case "index":
+		return repo.Index().Dump(os.Stdout)
+	case "snapshots":
+		return printSnapshots(repo, os.Stdout)
+	case "trees":
+		return printTrees(repo, os.Stdout)
+	case "all":
+		fmt.Printf("snapshots:\n")
+		err := printSnapshots(repo, os.Stdout)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("\ntrees:\n")
+
+		err = printTrees(repo, os.Stdout)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("\nindex:\n")
+
+		err = repo.Index().Dump(os.Stdout)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	default:
+		return errors.Errorf("no such type %q", tpe)
+	}
+}

cmd/restic/cmd_find.go

@@ -105,13 +105,8 @@ func (c CmdFind) findInTree(repo *repository.Repository, id backend.ID, path str
 	return results, nil
 }
 
-func (c CmdFind) findInSnapshot(repo *repository.Repository, name string) error {
+func (c CmdFind) findInSnapshot(repo *repository.Repository, id backend.ID) error {
-	debug.Log("restic.find", "searching in snapshot %s\n  for entries within [%s %s]", name, c.oldest, c.newest)
+	debug.Log("restic.find", "searching in snapshot %s\n  for entries within [%s %s]", id.Str(), c.oldest, c.newest)
-
-	id, err := backend.ParseID(name)
-	if err != nil {
-		return err
-	}
 
 	sn, err := restic.LoadSnapshot(repo, id)
 	if err != nil {
@@ -169,7 +164,7 @@ func (c CmdFind) Execute(args []string) error {
 	c.pattern = args[0]
 
 	if c.Snapshot != "" {
-		snapshotID, err := backend.FindSnapshot(s, c.Snapshot)
+		snapshotID, err := restic.FindSnapshot(s, c.Snapshot)
 		if err != nil {
 			return fmt.Errorf("invalid id %q: %v", args[1], err)
 		}

cmd/restic/cmd_fsck.go

@@ -57,8 +57,8 @@ func fsckFile(opts CmdFsck, repo *repository.Repository, IDs []backend.ID) (uint
 				return 0, err
 			}
 		} else {
-			// test if data blob is there
+			// test if pack for data blob is there
-			ok, err := repo.Test(backend.Data, packID.String())
+			ok, err := repo.Backend().Test(backend.Data, packID.String())
 			if err != nil {
 				return 0, err
 			}
@@ -70,6 +70,7 @@ func fsckFile(opts CmdFsck, repo *repository.Repository, IDs []backend.ID) (uint
 
 		// if orphan check is active, record storage id
 		if opts.o_data != nil {
+			debug.Log("restic.fsck", "  recording blob %v as used\n", id)
 			opts.o_data.Insert(id)
 		}
 	}
@@ -198,16 +199,11 @@ func (cmd CmdFsck) Execute(args []string) error {
 	}
 
 	if cmd.Snapshot != "" {
-		name, err := s.FindSnapshot(cmd.Snapshot)
+		id, err := restic.FindSnapshot(s, cmd.Snapshot)
 		if err != nil {
 			return fmt.Errorf("invalid id %q: %v", cmd.Snapshot, err)
 		}
 
-		id, err := backend.ParseID(name)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "invalid snapshot id %v\n", name)
-		}
-
 		err = fsckSnapshot(cmd, s, id)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "check for snapshot %v failed\n", id)
@@ -225,13 +221,7 @@ func (cmd CmdFsck) Execute(args []string) error {
 	defer close(done)
 
 	var firstErr error
-	for name := range s.List(backend.Snapshot, done) {
+	for id := range s.List(backend.Snapshot, done) {
-		id, err := backend.ParseID(name)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "invalid snapshot id %v\n", name)
-			continue
-		}
-
 		err = fsckSnapshot(cmd, s, id)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "check for snapshot %v failed\n", id)
@@ -246,14 +236,16 @@ func (cmd CmdFsck) Execute(args []string) error {
 	debug.Log("restic.fsck", "starting orphaned check\n")
 
 	cnt := make(map[pack.BlobType]*backend.IDSet)
-	cnt[pack.Data] = backend.NewIDSet()
+	cnt[pack.Data] = cmd.o_data
-	cnt[pack.Tree] = backend.NewIDSet()
+	cnt[pack.Tree] = cmd.o_trees
 
 	for blob := range s.Index().Each(done) {
-		fmt.Println(blob.ID)
+		debug.Log("restic.fsck", "checking %v blob %v\n", blob.Type, blob.ID)
 
 		err = cnt[blob.Type].Find(blob.ID)
 		if err != nil {
+			debug.Log("restic.fsck", "  blob %v is orphaned\n", blob.ID)
+
 			if !cmd.RemoveOrphaned {
 				fmt.Printf("orphaned %v blob %v\n", blob.Type, blob.ID)
 				continue

cmd/restic/cmd_key.go

@@ -34,20 +34,20 @@ func listKeys(s *repository.Repository) error {
 	done := make(chan struct{})
 	defer close(done)
 
-	for name := range s.List(backend.Key, done) {
+	for id := range s.List(backend.Key, done) {
-		k, err := repository.LoadKey(s, name)
+		k, err := repository.LoadKey(s, id.String())
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "LoadKey() failed: %v\n", err)
 			continue
 		}
 
 		var current string
-		if name == s.KeyName() {
+		if id.String() == s.KeyName() {
 			current = "*"
 		} else {
 			current = " "
 		}
-		tab.Rows = append(tab.Rows, []interface{}{current, name[:plen],
+		tab.Rows = append(tab.Rows, []interface{}{current, id.String()[:plen],
 			k.Username, k.Hostname, k.Created.Format(TimeFormat)})
 	}
 
@@ -79,7 +79,7 @@ func deleteKey(repo *repository.Repository, name string) error {
 		return errors.New("refusing to remove key currently used to access repository")
 	}
 
-	err := repo.Remove(backend.Key, name)
+	err := repo.Backend().Remove(backend.Key, name)
 	if err != nil {
 		return err
 	}
@@ -103,7 +103,7 @@ func changePassword(s *repository.Repository) error {
 	}
 
 	// remove old key
-	err = s.Remove(backend.Key, s.KeyName())
+	err = s.Backend().Remove(backend.Key, s.KeyName())
 	if err != nil {
 		return err
 	}
@@ -133,7 +133,7 @@ func (cmd CmdKey) Execute(args []string) error {
 	case "add":
 		return addKey(s)
 	case "rm":
-		id, err := backend.Find(s, backend.Key, args[1])
+		id, err := backend.Find(s.Backend(), backend.Key, args[1])
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_ls.go

@@ -77,12 +77,7 @@ func (cmd CmdLs) Execute(args []string) error {
 		return err
 	}
 
-	name, err := backend.FindSnapshot(s, args[0])
+	id, err := restic.FindSnapshot(s, args[0])
-	if err != nil {
-		return err
-	}
-
-	id, err := backend.ParseID(name)
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_restore.go

@@ -6,7 +6,6 @@ import (
 	"path/filepath"
 
 	"github.com/restic/restic"
-	"github.com/restic/restic/backend"
 )
 
 type CmdRestore struct{}
@@ -40,12 +39,7 @@ func (cmd CmdRestore) Execute(args []string) error {
 		return err
 	}
 
-	name, err := backend.FindSnapshot(s, args[0])
+	id, err := restic.FindSnapshot(s, args[0])
-	if err != nil {
-		errx(1, "invalid id %q: %v", args[0], err)
-	}
-
-	id, err := backend.ParseID(name)
 	if err != nil {
 		errx(1, "invalid id %q: %v", args[0], err)
 	}

cmd/restic/cmd_snapshots.go

@@ -105,13 +105,7 @@ func (cmd CmdSnapshots) Execute(args []string) error {
 	defer close(done)
 
 	list := []*restic.Snapshot{}
-	for name := range s.List(backend.Snapshot, done) {
+	for id := range s.List(backend.Snapshot, done) {
-		id, err := backend.ParseID(name)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "error parsing id: %v", name)
-			continue
-		}
-
 		sn, err := restic.LoadSnapshot(s, id)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err)

repository/index.go

@@ -173,28 +173,24 @@ type blobJSON struct {
 	Length uint          `json:"length"`
 }
 
-// Encode writes the JSON serialization of the index to the writer w. This
+// generatePackList returns a list of packs containing only the index entries
-// serialization only contains new blobs added via idx.Store(), not old ones
+// that selsectFn returned true for. If selectFn is nil, the list contains all
-// introduced via DecodeIndex().
+// blobs in the index.
-func (idx *Index) Encode(w io.Writer) error {
+func (idx *Index) generatePackList(selectFn func(indexEntry) bool) ([]*packJSON, error) {
-	debug.Log("Index.Encode", "encoding index")
-	idx.m.Lock()
-	defer idx.m.Unlock()
-
 	list := []*packJSON{}
 	packs := make(map[string]*packJSON)
 
 	for id, blob := range idx.pack {
-		if blob.old {
+		if selectFn != nil && !selectFn(blob) {
 			continue
 		}
 
-		debug.Log("Index.Encode", "handle blob %q", id[:8])
+		debug.Log("Index.generatePackList", "handle blob %q", id[:8])
 
 		if blob.packID == nil {
-			debug.Log("Index.Encode", "blob %q has no packID! (type %v, offset %v, length %v)",
+			debug.Log("Index.generatePackList", "blob %q has no packID! (type %v, offset %v, length %v)",
 				id[:8], blob.tpe, blob.offset, blob.length)
-			return fmt.Errorf("unable to serialize index: pack for blob %v hasn't been written yet", id)
+			return nil, fmt.Errorf("unable to serialize index: pack for blob %v hasn't been written yet", id)
 		}
 
 		// see if pack is already in map
@@ -217,12 +213,63 @@ func (idx *Index) Encode(w io.Writer) error {
 		})
 	}
 
+	debug.Log("Index.generatePackList", "done")
+
+	return list, nil
+}
+
+// encode writes the JSON serialization of the index filtered by selectFn to enc.
+func (idx *Index) encode(w io.Writer, selectFn func(indexEntry) bool) error {
+	list, err := idx.generatePackList(func(entry indexEntry) bool {
+		return !entry.old
+	})
+	if err != nil {
+		return err
+	}
+
 	debug.Log("Index.Encode", "done")
 
 	enc := json.NewEncoder(w)
 	return enc.Encode(list)
 }
 
+// Encode writes the JSON serialization of the index to the writer w. This
+// serialization only contains new blobs added via idx.Store(), not old ones
+// introduced via DecodeIndex().
+func (idx *Index) Encode(w io.Writer) error {
+	debug.Log("Index.Encode", "encoding index")
+	idx.m.Lock()
+	defer idx.m.Unlock()
+
+	return idx.encode(w, func(e indexEntry) bool { return !e.old })
+}
+
+// Dump writes the pretty-printed JSON representation of the index to w.
+func (idx *Index) Dump(w io.Writer) error {
+	debug.Log("Index.Dump", "dumping index")
+	idx.m.Lock()
+	defer idx.m.Unlock()
+
+	list, err := idx.generatePackList(nil)
+	if err != nil {
+		return err
+	}
+
+	buf, err := json.MarshalIndent(list, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	_, err = w.Write(append(buf, '\n'))
+	if err != nil {
+		return err
+	}
+
+	debug.Log("Index.Dump", "done")
+
+	return nil
+}
+
 // DecodeIndex loads and unserializes an index from rd.
 func DecodeIndex(rd io.Reader) (*Index, error) {
 	debug.Log("Index.DecodeIndex", "Start decoding index")

repository/key.go

@@ -98,7 +98,7 @@ func SearchKey(s *Repository, password string) (*Key, error) {
 	// try all keys in repo
 	done := make(chan struct{})
 	defer close(done)
-	for name := range s.List(backend.Key, done) {
+	for name := range s.Backend().List(backend.Key, done) {
 		key, err := OpenKey(s, name, password)
 		if err != nil {
 			continue

repository/repository.go

@@ -52,12 +52,6 @@ func (s *Repository) Find(t backend.Type, prefix string) (string, error) {
 	return backend.Find(s.be, t, prefix)
 }
 
-// FindSnapshot takes a string and tries to find a snapshot whose ID matches
-// the string as closely as possible.
-func (s *Repository) FindSnapshot(name string) (string, error) {
-	return backend.FindSnapshot(s.be, name)
-}
-
 // PrefixLength returns the number of bytes required so that all prefixes of
 // all IDs of type t are unique.
 func (s *Repository) PrefixLength(t backend.Type) (int, error) {
@@ -586,7 +580,7 @@ func (s *Repository) SearchKey(password string) error {
 // Init creates a new master key with the supplied password and initializes the
 // repository config.
 func (s *Repository) Init(password string) error {
-	has, err := s.Test(backend.Config, "")
+	has, err := s.be.Test(backend.Config, "")
 	if err != nil {
 		return err
 	}
@@ -637,26 +631,49 @@ func (s *Repository) Count(t backend.Type) (n uint) {
 	return
 }
 
-// Proxy methods to backend
+func (s *Repository) list(t backend.Type, done <-chan struct{}, out chan<- backend.ID) {
+	defer close(out)
+	in := s.be.List(t, done)
 
-func (s *Repository) Get(t backend.Type, name string) (io.ReadCloser, error) {
+	var (
-	return s.be.Get(t, name)
+		// disable sending on the outCh until we received a job
+		outCh chan<- backend.ID
+		// enable receiving from in
+		inCh = in
+		id   backend.ID
+		err  error
+	)
+
+	for {
+		select {
+		case <-done:
+			return
+		case strID, ok := <-inCh:
+			if !ok {
+				// input channel closed, we're done
+				return
+			}
+			id, err = backend.ParseID(strID)
+			if err != nil {
+				// ignore invalid IDs
+				continue
+			}
+
+			inCh = nil
+			outCh = out
+		case outCh <- id:
+			outCh = nil
+			inCh = in
+		}
+	}
 }
 
-func (s *Repository) List(t backend.Type, done <-chan struct{}) <-chan string {
+func (s *Repository) List(t backend.Type, done <-chan struct{}) <-chan backend.ID {
-	return s.be.List(t, done)
+	outCh := make(chan backend.ID)
-}
 
-func (s *Repository) Test(t backend.Type, name string) (bool, error) {
+	go s.list(t, done, outCh)
-	return s.be.Test(t, name)
-}
 
-func (s *Repository) Remove(t backend.Type, name string) error {
+	return outCh
-	return s.be.Remove(t, name)
-}
-
-func (s *Repository) Close() error {
-	return s.be.Close()
 }
 
 func (s *Repository) Delete() error {
@@ -667,6 +684,6 @@ func (s *Repository) Delete() error {
 	return errors.New("Delete() called for backend that does not implement this method")
 }
 
-func (s *Repository) Location() string {
+func (s *Repository) Close() error {
-	return s.be.Location()
+	return s.be.Close()
 }

snapshot.go

@@ -89,3 +89,15 @@ func (sn *Snapshot) fillUserInfo() error {
 
 	return nil
 }
+
+// FindSnapshot takes a string and tries to find a snapshot whose ID matches
+// the string as closely as possible.
+func FindSnapshot(repo *repository.Repository, s string) (backend.ID, error) {
+	// find snapshot id with prefix
+	name, err := backend.Find(repo.Backend(), backend.Snapshot, s)
+	if err != nil {
+		return nil, err
+	}
+
+	return backend.ParseID(name)
+}