Add local cache

archiver.go

@@ -61,6 +61,11 @@ func NewArchiver(s Server) (*Archiver, error) {
 // Preload loads all tree objects from repository and adds all blobs that are
 // still available to the map for deduplication.
 func (arch *Archiver) Preload(p *Progress) error {
+	cache, err := NewCache()
+	if err != nil {
+		return err
+	}
+
 	p.Start()
 	defer p.Done()
 
@@ -69,10 +74,28 @@ func (arch *Archiver) Preload(p *Progress) error {
 	// load all trees, in parallel
 	worker := func(wg *sync.WaitGroup, c <-chan backend.ID) {
 		for id := range c {
-			tree, err := LoadTree(arch.s, id)
-			// ignore error and advance to next tree
-			if err != nil {
-				return
+			var tree *Tree
+
+			// load from cache
+			var t Tree
+			rd, err := cache.Load(backend.Tree, id)
+			if err == nil {
+				debug.Log("Archiver.Preload", "tree %v cached", id.Str())
+				tree = &t
+				dec := json.NewDecoder(rd)
+				err = dec.Decode(&t)
+
+				if err != nil {
+					continue
+				}
+			} else {
+				debug.Log("Archiver.Preload", "tree %v not cached: %v", id.Str(), err)
+
+				tree, err = LoadTree(arch.s, id)
+				// ignore error and advance to next tree
+				if err != nil {
+					continue
+				}
 			}
 
 			debug.Log("Archiver.Preload", "load tree %v with %d blobs", id, tree.Map.Len())
@@ -93,7 +116,7 @@ func (arch *Archiver) Preload(p *Progress) error {
 
 	// list ids
 	trees := 0
-	err := arch.s.EachID(backend.Tree, func(id backend.ID) {
+	err = arch.s.EachID(backend.Tree, func(id backend.ID) {
 		trees++
 
 		if trees%1000 == 0 {

archiver_test.go

@@ -202,10 +202,6 @@ func TestArchivePreload(t *testing.T) {
 	archiveWithPreload(t)
 }
 
-func BenchmarkArchivePreload(b *testing.B) {
-	archiveWithPreload(b)
-}
-
 func BenchmarkPreload(t *testing.B) {
 	if *benchArchiveDirectory == "" {
 		t.Skip("benchdir not set, skipping TestArchiverPreload")
@@ -233,3 +229,33 @@ func BenchmarkPreload(t *testing.B) {
 		ok(t, arch2.Preload(nil))
 	}
 }
+
+func BenchmarkLoadTree(t *testing.B) {
+	if *benchArchiveDirectory == "" {
+		t.Skip("benchdir not set, skipping TestArchiverPreload")
+	}
+
+	be := setupBackend(t)
+	defer teardownBackend(t, be)
+	key := setupKey(t, be, "geheim")
+	server := restic.NewServerWithKey(be, key)
+
+	// archive a few files
+	arch, err := restic.NewArchiver(server)
+	ok(t, err)
+	sn, _, err := arch.Snapshot(nil, *benchArchiveDirectory, nil)
+	ok(t, err)
+	t.Logf("archived snapshot %v", sn.ID())
+
+	// start benchmark
+	t.ResetTimer()
+
+	list, err := server.List(backend.Tree)
+	ok(t, err)
+	list = list[:10]
+
+	for i := 0; i < t.N; i++ {
+		_, err := restic.LoadTree(server, list[0])
+		ok(t, err)
+	}
+}

cache.go

@@ -0,0 +1,92 @@
+package restic
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/restic/restic/backend"
+)
+
+type Cache struct {
+	base string
+}
+
+func NewCache() (*Cache, error) {
+	dir, err := GetCacheDir()
+	if err != nil {
+		return nil, err
+	}
+
+	return &Cache{base: dir}, nil
+}
+
+func (c *Cache) Has(t backend.Type, id backend.ID) (bool, error) {
+	// try to open file
+	filename, err := c.filename(t, id)
+	if err != nil {
+		return false, err
+	}
+
+	fd, err := os.Open(filename)
+	defer fd.Close()
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+
+		return false, err
+	}
+
+	return true, nil
+}
+
+func (c *Cache) Store(t backend.Type, id backend.ID, rd io.Reader) error {
+	filename, err := c.filename(t, id)
+	if err != nil {
+		return err
+	}
+
+	dirname := filepath.Dir(filename)
+	err = os.MkdirAll(dirname, 0700)
+	if err != nil {
+		return err
+	}
+
+	file, err := os.Create(filename)
+	defer file.Close()
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(file, rd)
+	return err
+}
+
+func (c *Cache) Load(t backend.Type, id backend.ID) (io.ReadCloser, error) {
+	// try to open file
+	filename, err := c.filename(t, id)
+	if err != nil {
+		return nil, err
+	}
+
+	return os.Open(filename)
+}
+
+// Construct file name for given Type.
+func (c *Cache) filename(t backend.Type, id backend.ID) (string, error) {
+	cachedir, err := GetCacheDir()
+	if err != nil {
+		return "", err
+	}
+
+	switch t {
+	case backend.Snapshot:
+		return filepath.Join(cachedir, "snapshots", id.String()), nil
+	case backend.Tree:
+		return filepath.Join(cachedir, "trees", id.String()), nil
+	}
+
+	return "", fmt.Errorf("cache not supported for type %v", t)
+}

cache_linux.go

@@ -0,0 +1,49 @@
+package restic
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/restic/restic/debug"
+)
+
+// GetCacheDir returns the cache directory according to XDG basedir spec, see
+// http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
+func GetCacheDir() (string, error) {
+	xdgcache := os.Getenv("XDG_CACHE_HOME")
+	home := os.Getenv("HOME")
+
+	if xdgcache == "" && home == "" {
+		return "", errors.New("unable to locate cache directory (XDG_CACHE_HOME and HOME unset)")
+	}
+
+	cachedir := ""
+	if xdgcache != "" {
+		cachedir = filepath.Join(xdgcache, "restic")
+	} else if home != "" {
+		cachedir = filepath.Join(home, ".cache", "restic")
+	}
+
+	fi, err := os.Stat(cachedir)
+	if os.IsNotExist(err) {
+		err = os.MkdirAll(cachedir, 0700)
+		if err != nil {
+			return "", err
+		}
+
+		fi, err = os.Stat(cachedir)
+		debug.Log("getCacheDir", "create cache dir %v", cachedir)
+	}
+
+	if err != nil {
+		return "", err
+	}
+
+	if !fi.IsDir() {
+		return "", fmt.Errorf("cache dir %v is not a directory", cachedir)
+	}
+
+	return cachedir, nil
+}

cmd/restic/cmd_backup.go

@@ -112,8 +112,6 @@ func newLoadBlobsProgress(s restic.Server) (*restic.Progress, error) {
 			}
 		}
 
-		// fmt.Printf("sec: %v, trees: %v / %v\n", sec, s.Trees, trees)
-
 		fmt.Printf("\x1b[2K\r[%s] %3.2f%%  %d trees/s  %d / %d trees, %d blobs  ETA %s",
 			format_duration(d),
 			float64(s.Trees)/float64(trees)*100,

cmd/restic/cmd_cache.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/restic/restic"
+	"github.com/restic/restic/backend"
+)
+
+type CmdCache struct{}
+
+func init() {
+	_, err := parser.AddCommand("cache",
+		"manage cache",
+		"The cache command creates and manages the local cache",
+		&CmdCache{})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdCache) Usage() string {
+	return "[update|clear]"
+}
+
+func (cmd CmdCache) Execute(args []string) error {
+	// if len(args) == 0 || len(args) > 2 {
+	// 	return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage())
+	// }
+
+	s, err := OpenRepo()
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("update cache, load trees\n")
+
+	list, err := s.List(backend.Tree)
+	if err != nil {
+		return err
+	}
+
+	cache, err := restic.NewCache()
+	if err != nil {
+		return err
+	}
+
+	treeCh := make(chan backend.ID)
+	worker := func(wg *sync.WaitGroup, ch chan backend.ID) {
+		for treeID := range ch {
+			cached, err := cache.Has(backend.Tree, treeID)
+			if err != nil {
+				fmt.Printf("tree %v cache error: %v\n", treeID.Str(), err)
+				continue
+			}
+
+			if cached {
+				fmt.Printf("tree %v already cached\n", treeID.Str())
+				continue
+			}
+
+			rd, err := s.GetReader(backend.Tree, treeID)
+			if err != nil {
+				fmt.Printf("  load error: %v\n", err)
+				continue
+			}
+
+			decRd, err := s.Key().DecryptFrom(rd)
+			if err != nil {
+				fmt.Printf("  store error: %v\n", err)
+				continue
+			}
+
+			err = cache.Store(backend.Tree, treeID, decRd)
+			if err != nil {
+				fmt.Printf("  store error: %v\n", err)
+				continue
+			}
+
+			err = decRd.Close()
+			if err != nil {
+				fmt.Printf("  close error: %v\n", err)
+				continue
+			}
+
+			err = rd.Close()
+			if err != nil {
+				fmt.Printf("  close error: %v\n", err)
+				continue
+			}
+
+			fmt.Printf("tree %v stored\n", treeID.Str())
+		}
+		wg.Done()
+	}
+
+	var wg sync.WaitGroup
+	// start workers
+	for i := 0; i < 500; i++ {
+		wg.Add(1)
+		go worker(&wg, treeCh)
+	}
+
+	for _, treeID := range list {
+		treeCh <- treeID
+	}
+
+	close(treeCh)
+
+	wg.Wait()
+
+	return nil
+}

cmd/restic/main.go

@@ -165,6 +165,7 @@ func init() {
 
 func main() {
 	// defer profile.Start(profile.MemProfileRate(100000), profile.ProfilePath(".")).Stop()
+	// defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop()
 	opts.Repo = os.Getenv("RESTIC_REPOSITORY")
 
 	debug.Log("restic", "main %#v", os.Args)