From b17840c6ee9d1d3c80d1cbd2dacf5c26fb53ca51 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 14 Mar 2015 12:30:47 +0100 Subject: [PATCH] Refactor cache and 'cache' command --- cache.go | 105 ++++++++++++++++++++++++++++++++++++++++ cmd/restic/cmd_cache.go | 83 ++----------------------------- 2 files changed, 110 insertions(+), 78 deletions(-) diff --git a/cache.go b/cache.go index 7a670e183..32762bc84 100644 --- a/cache.go +++ b/cache.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "strings" "github.com/restic/restic/backend" "github.com/restic/restic/debug" @@ -43,12 +44,15 @@ func (c *Cache) Has(t backend.Type, subtype string, id backend.ID) (bool, error) defer fd.Close() if err != nil { if os.IsNotExist(err) { + debug.Log("Cache.Has", "test for file %v: not cached", filename) return false, nil } + debug.Log("Cache.Has", "test for file %v: error %v", filename, err) return false, err } + debug.Log("Cache.Has", "test for file %v: is cached", filename) return true, nil } @@ -66,9 +70,11 @@ func (c *Cache) Store(t backend.Type, subtype string, id backend.ID) (io.WriteCl file, err := os.Create(filename) if err != nil { + debug.Log("Cache.Store", "error creating file %v: %v", filename, err) return nil, err } + debug.Log("Cache.Store", "created file %v", filename) return file, nil } @@ -82,6 +88,105 @@ func (c *Cache) Load(t backend.Type, subtype string, id backend.ID) (io.ReadClos return os.Open(filename) } +func (c *Cache) Purge(t backend.Type, subtype string, id backend.ID) error { + filename, err := c.filename(t, subtype, id) + if err != nil { + return err + } + + err = os.Remove(filename) + debug.Log("Cache.Purge", "Remove file %v: %v", filename, err) + + if err != nil && os.IsNotExist(err) { + return nil + } + + return err +} + +func (c *Cache) Clear(s backend.Backend) error { + list, err := c.List(backend.Snapshot) + if err != nil { + return err + } + + for _, entry := range list { + debug.Log("Cache.Clear", "found entry %v", entry) + + if ok, err := s.Test(backend.Snapshot, entry.ID); !ok || err != nil { + debug.Log("Cache.Clear", "snapshot %v doesn't exist any more, removing %v", entry.ID, entry) + + err = c.Purge(backend.Snapshot, entry.Subtype, entry.ID) + if err != nil { + return err + } + } + } + + return nil +} + +type CacheEntry struct { + ID backend.ID + Subtype string +} + +func (c CacheEntry) String() string { + if c.Subtype != "" { + return c.ID.Str() + "." + c.Subtype + } + return c.ID.Str() +} + +func (c *Cache) List(t backend.Type) ([]CacheEntry, error) { + var dir string + + switch t { + case backend.Snapshot: + dir = filepath.Join(c.base, "snapshots") + case backend.Tree: + dir = filepath.Join(c.base, "trees") + default: + return nil, fmt.Errorf("cache not supported for type %v", t) + } + + fd, err := os.Open(dir) + if err != nil { + if os.IsNotExist(err) { + return []CacheEntry{}, nil + } + return nil, err + } + defer fd.Close() + + fis, err := fd.Readdir(-1) + if err != nil { + return nil, err + } + + entries := make([]CacheEntry, 0, len(fis)) + + for _, fi := range fis { + parts := strings.SplitN(fi.Name(), ".", 2) + + id, err := backend.ParseID(parts[0]) + // ignore invalid cache entries for now + if err != nil { + continue + } + + e := CacheEntry{ID: id} + + if len(parts) == 2 { + e.Subtype = parts[1] + } + + entries = append(entries, e) + } + + return entries, nil +} + // Construct file name for given Type. func (c *Cache) filename(t backend.Type, subtype string, id backend.ID) (string, error) { filename := id.String() diff --git a/cmd/restic/cmd_cache.go b/cmd/restic/cmd_cache.go index 2b145e349..c6fafd786 100644 --- a/cmd/restic/cmd_cache.go +++ b/cmd/restic/cmd_cache.go @@ -2,11 +2,8 @@ package main import ( "fmt" - "io" - "sync" "github.com/restic/restic" - "github.com/restic/restic/backend" ) type CmdCache struct{} @@ -35,87 +32,17 @@ func (cmd CmdCache) Execute(args []string) error { return err } - fmt.Printf("update cache, load trees\n") - - list, err := s.List(backend.Tree) - if err != nil { - return err - } - cache, err := restic.NewCache(s) if err != nil { return err } - treeCh := make(chan backend.ID) - worker := func(wg *sync.WaitGroup, ch chan backend.ID) { - for treeID := range ch { - cached, err := cache.Has(backend.Tree, "", treeID) - if err != nil { - fmt.Printf("tree %v cache error: %v\n", treeID.Str(), err) - continue - } - - if cached { - fmt.Printf("tree %v already cached\n", treeID.Str()) - continue - } - - rd, err := s.GetReader(backend.Tree, treeID) - if err != nil { - fmt.Printf(" load error: %v\n", err) - continue - } - - decRd, err := s.Key().DecryptFrom(rd) - if err != nil { - fmt.Printf(" store error: %v\n", err) - continue - } - - wr, err := cache.Store(backend.Tree, "", treeID) - if err != nil { - fmt.Printf(" store error: %v\n", err) - continue - } - - _, err = io.Copy(wr, decRd) - if err != nil { - fmt.Printf(" Copy error: %v\n", err) - continue - } - - err = decRd.Close() - if err != nil { - fmt.Printf(" close error: %v\n", err) - continue - } - - err = rd.Close() - if err != nil { - fmt.Printf(" close error: %v\n", err) - continue - } - - fmt.Printf("tree %v stored\n", treeID.Str()) - } - wg.Done() + fmt.Printf("clear cache for old snapshots\n") + err = cache.Clear(s) + if err != nil { + return err } - - var wg sync.WaitGroup - // start workers - for i := 0; i < 500; i++ { - wg.Add(1) - go worker(&wg, treeCh) - } - - for _, treeID := range list { - treeCh <- treeID - } - - close(treeCh) - - wg.Wait() + fmt.Printf("done\n") return nil }