Merge pull request #1436 from restic/remove-old-cache

Remove old cache directories

CHANGELOG.md

@@ -35,12 +35,14 @@ Important Changes in 0.8.0
    that can be used to disable the cache. By deafult, the cache a standard
    cache folder for the OS, which can be overridden with `--cache-dir`.  The
    cache will automatically populate, indexes and snapshots are saved as they
-   are loaded.
+   are loaded. Cache directories for repos that haven't been used recently can
+   automatically be removed by restic with the `--cleanup-cache` option.
    https://github.com/restic/restic/pull/1040
    https://github.com/restic/restic/issues/29
    https://github.com/restic/restic/issues/738
    https://github.com/restic/restic/issues/282
    https://github.com/restic/restic/pull/1287
+   https://github.com/restic/restic/pull/1436
 
  * A related change was to by default create pack files in the repo that
    contain either data or metadata, not both mixed together. This allows easy

cmd/restic/global.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"syscall"
@@ -23,6 +24,7 @@ import (
 	"github.com/restic/restic/internal/backend/swift"
 	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/limiter"
 	"github.com/restic/restic/internal/options"
 	"github.com/restic/restic/internal/repository"
@@ -45,6 +47,7 @@ type GlobalOptions struct {
 	CacheDir     string
 	NoCache      bool
 	CACerts      []string
+	CleanupCache bool
 
 	LimitUploadKb   int
 	LimitDownloadKb int
@@ -81,6 +84,7 @@ func init() {
 	f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache directory")
 	f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache")
 	f.StringSliceVar(&globalOptions.CACerts, "cacert", nil, "path to load root certificates from (default: use system certificates)")
+	f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories")
 	f.IntVar(&globalOptions.LimitUploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)")
 	f.IntVar(&globalOptions.LimitDownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)")
 	f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
@@ -353,13 +357,39 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 		return s, nil
 	}
 
-	cache, err := cache.New(s.Config().ID, opts.CacheDir)
+	c, err := cache.New(s.Config().ID, opts.CacheDir)
 	if err != nil {
 		Warnf("unable to open cache: %v\n", err)
-	} else {
+		return s, nil
-		s.UseCache(cache)
 	}
 
+	oldCacheDirs, err := cache.Old(c.Base)
+	if err != nil {
+		Warnf("unable to find old cache directories: %v", err)
+	}
+
+	// nothing more to do if no old cache dirs could be found
+	if len(oldCacheDirs) == 0 {
+		return s, nil
+	}
+
+	// cleanup old cache dirs if instructed to do so
+	if opts.CleanupCache {
+		Printf("removing %d old cache dirs from %v\n", len(oldCacheDirs), c.Base)
+
+		for _, item := range oldCacheDirs {
+			dir := filepath.Join(c.Base, item)
+			err = fs.RemoveAll(dir)
+			if err != nil {
+				Warnf("unable to remove %v: %v\n", dir, err)
+			}
+		}
+	} else {
+		Verbosef("found %d old cache directories in %v, pass --cleanup-cache to remove them\n",
+			len(oldCacheDirs), c.Base)
+	}
+
+	s.UseCache(c)
 	return s, nil
 }
 

doc/cache.rst

@@ -19,8 +19,18 @@ a lower version number is found the cache is recreated with the current
 version. If a higher version number is found the cache is ignored and left as
 is.
 
-Snapshots and Indexes
+Snapshots, Data and Indexes
----------------------
+---------------------------
 
 Snapshot, Data and Index files are cached in the sub-directories ``snapshots``,
 ``data`` and  ``index``, as read from the repository.
+
+Expiry
+------
+
+Whenever a cache directory for a repo is used, that directory's modification
+timestamp is updated to the current time. By looking at the modification
+timestamps of the repo cache directories it is easy to decide which directories
+are old and haven't been used in a long time. Those are probably stale and can
+be removed.
+

doc/manual_rest.rst

@@ -280,3 +280,10 @@ entirely. In this case, all data is loaded from the repo.
 
 The cache is ephemeral: When a file cannot be read from the cache, it is loaded
 from the repository.
+
+Within the cache directory, there's a sub directory for each repository the
+cache was used with. Restic updates the timestamps of a repo directory each
+time it is used, so by looking at the timestamps of the sub directories of the
+cache directory it can decide which sub directories are old and probably not
+needed any more. You can either remove these directories manually, or run a
+restic command with the ``--cleanup-cache`` flag.

internal/cache/cache.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/restic/restic/internal/debug"
@@ -84,10 +85,12 @@ func writeCachedirTag(dir string) error {
 // performReadahead returns true.
 func New(id string, basedir string) (c *Cache, err error) {
 	if basedir == "" {
-		basedir, err = defaultCacheDir()
+		basedir, err = DefaultDir()
-		if err != nil {
+	}
-			return nil, err
+
-		}
+	err = mkdirCacheDir(basedir)
+	if err != nil {
+		return nil, err
 	}
 
 	// create base dir and tag it as a cache directory
@@ -112,6 +115,12 @@ func New(id string, basedir string) (c *Cache, err error) {
 		return nil, err
 	}
 
+	// update the timestamp so that we can detect old cache dirs
+	err = updateTimestamp(cachedir)
+	if err != nil {
+		return nil, err
+	}
+
 	if v < cacheVersion {
 		err = ioutil.WriteFile(filepath.Join(cachedir, "version"), []byte(fmt.Sprintf("%d", cacheVersion)), 0644)
 		if err != nil {
@@ -137,6 +146,53 @@ func New(id string, basedir string) (c *Cache, err error) {
 	return c, nil
 }
 
+// updateTimestamp sets the modification timestamp (mtime and atime) for the
+// directory d to the current time.
+func updateTimestamp(d string) error {
+	t := time.Now()
+	return fs.Chtimes(d, t, t)
+}
+
+const maxCacheAge = 30 * 24 * time.Hour
+
+// Old returns a list of cache directories with a modification time of more
+// than 30 days ago.
+func Old(basedir string) ([]string, error) {
+	var oldCacheDirs []string
+
+	f, err := fs.Open(basedir)
+	if err != nil {
+		return nil, err
+	}
+
+	entries, err := f.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	oldest := time.Now().Add(-maxCacheAge)
+	for _, fi := range entries {
+		if !fi.IsDir() {
+			continue
+		}
+
+		if !fi.ModTime().Before(oldest) {
+			continue
+		}
+
+		oldCacheDirs = append(oldCacheDirs, fi.Name())
+	}
+
+	err = f.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	debug.Log("%d old cache dirs found", len(oldCacheDirs))
+
+	return oldCacheDirs, nil
+}
+
 // errNoSuchFile is returned when a file is not cached.
 type errNoSuchFile struct {
 	Type string

internal/cache/dir.go

@@ -49,29 +49,25 @@ func darwinCacheDir() (string, error) {
 	return filepath.Join(home, "Library", "Caches", "restic"), nil
 }
 
-// defaultCacheDir determines and creates the default cache directory for this
+// DefaultDir returns the default cache directory for the current OS.
-// system.
+func DefaultDir() (cachedir string, err error) {
-func defaultCacheDir() (string, error) {
-	var cachedir string
-	var err error
 	switch runtime.GOOS {
 	case "darwin":
 		cachedir, err = darwinCacheDir()
 	case "windows":
 		cachedir, err = windowsCacheDir()
-	default:
-		// Default to XDG for Linux and any other OSes.
-		cachedir, err = xdgCacheDir()
-	}
-	if err != nil {
-		return "", err
 	}
 
+	// Default to XDG for Linux and any other OSes.
+	return xdgCacheDir()
+}
+
+func mkdirCacheDir(cachedir string) error {
 	fi, err := fs.Stat(cachedir)
 	if os.IsNotExist(errors.Cause(err)) {
 		err = fs.MkdirAll(cachedir, 0700)
 		if err != nil {
-			return "", errors.Wrap(err, "MkdirAll")
+			return errors.Wrap(err, "MkdirAll")
 		}
 
 		fi, err = fs.Stat(cachedir)
@@ -79,12 +75,12 @@ func defaultCacheDir() (string, error) {
 	}
 
 	if err != nil {
-		return "", errors.Wrap(err, "Stat")
+		return errors.Wrap(err, "Stat")
 	}
 
 	if !fi.IsDir() {
-		return "", errors.Errorf("cache dir %v is not a directory", cachedir)
+		return errors.Errorf("cache dir %v is not a directory", cachedir)
 	}
 
-	return cachedir, nil
+	return nil
 }

internal/fs/file.go

@@ -4,6 +4,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"time"
 )
 
 // File is an open file on a file system.
@@ -120,3 +121,12 @@ func RemoveIfExists(filename string) error {
 	}
 	return err
 }
+
+// Chtimes changes the access and modification times of the named file,
+// similar to the Unix utime() or utimes() functions.
+//
+// The underlying filesystem may truncate or round the values to a less
+// precise time unit. If there is an error, it will be of type *PathError.
+func Chtimes(name string, atime time.Time, mtime time.Time) error {
+	return os.Chtimes(fixpath(name), atime, mtime)
+}