From fa893ee477e38ca69250b871cffcef274d8881e4 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 20 Nov 2017 21:32:25 +0100 Subject: [PATCH 1/5] cache: Add detection code for old cache dirs --- cmd/restic/global.go | 14 ++++++++--- internal/cache/cache.go | 53 +++++++++++++++++++++++++++++++++++++++++ internal/fs/file.go | 10 ++++++++ 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 70d031289..e1c745104 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -353,13 +353,21 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { return s, nil } - cache, err := cache.New(s.Config().ID, opts.CacheDir) + c, err := cache.New(s.Config().ID, opts.CacheDir) if err != nil { Warnf("unable to open cache: %v\n", err) - } else { - s.UseCache(cache) + return s, nil } + oldCacheDirs, err := cache.Old(c.Base) + if err != nil { + Warnf("unable to find old cache directories: %v", err) + } else { + Verbosef("found %d old cache directories in %v remove them with 'restic cache --cleanup'\n", + len(oldCacheDirs), c.Base) + } + + s.UseCache(c) return s, nil } diff --git a/internal/cache/cache.go b/internal/cache/cache.go index f83142e19..f4dd59bfd 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strconv" + "time" "github.com/pkg/errors" "github.com/restic/restic/internal/debug" @@ -107,6 +108,11 @@ func New(id string, basedir string) (c *Cache, err error) { return nil, errors.New("cache version is newer") } + err = updateTimestamp(cachedir) + if err != nil { + return nil, err + } + // create the repo cache dir if it does not exist yet if err = fs.MkdirAll(cachedir, dirMode); err != nil { return nil, err @@ -137,6 +143,53 @@ func New(id string, basedir string) (c *Cache, err error) { return c, nil } +// updateTimestamp sets the modification timestamp (mtime and atime) for the +// directory d to the current time. +func updateTimestamp(d string) error { + t := time.Now() + return fs.Chtimes(d, t, t) +} + +const maxCacheAge = 30 * 24 * time.Hour + +// Old returns a list of cache directories with a modification time of more +// than 30 days ago. +func Old(basedir string) ([]string, error) { + var oldCacheDirs []string + + f, err := fs.Open(basedir) + if err != nil { + return nil, err + } + + entries, err := f.Readdir(-1) + if err != nil { + return nil, err + } + + oldest := time.Now().Add(-maxCacheAge) + for _, fi := range entries { + if !fi.IsDir() { + continue + } + + if !fi.ModTime().Before(oldest) { + continue + } + + oldCacheDirs = append(oldCacheDirs, fi.Name()) + } + + err = f.Close() + if err != nil { + return nil, err + } + + debug.Log("%d old cache dirs found", len(oldCacheDirs)) + + return oldCacheDirs, nil +} + // errNoSuchFile is returned when a file is not cached. type errNoSuchFile struct { Type string diff --git a/internal/fs/file.go b/internal/fs/file.go index a90d1b2e7..d055107b4 100644 --- a/internal/fs/file.go +++ b/internal/fs/file.go @@ -4,6 +4,7 @@ import ( "io" "os" "path/filepath" + "time" ) // File is an open file on a file system. @@ -120,3 +121,12 @@ func RemoveIfExists(filename string) error { } return err } + +// Chtimes changes the access and modification times of the named file, +// similar to the Unix utime() or utimes() functions. +// +// The underlying filesystem may truncate or round the values to a less +// precise time unit. If there is an error, it will be of type *PathError. +func Chtimes(name string, atime time.Time, mtime time.Time) error { + return os.Chtimes(fixpath(name), atime, mtime) +} From f4bab789b8c5a0c707238c34c0a5c3a30778a368 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 20 Nov 2017 21:58:38 +0100 Subject: [PATCH 2/5] cache: Simplify cache dir creation --- internal/cache/cache.go | 19 +++++++++++-------- internal/cache/dir.go | 26 +++++++++++--------------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/internal/cache/cache.go b/internal/cache/cache.go index f4dd59bfd..db8f1ae02 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -85,10 +85,12 @@ func writeCachedirTag(dir string) error { // performReadahead returns true. func New(id string, basedir string) (c *Cache, err error) { if basedir == "" { - basedir, err = defaultCacheDir() - if err != nil { - return nil, err - } + basedir, err = DefaultDir() + } + + err = mkdirCacheDir(basedir) + if err != nil { + return nil, err } // create base dir and tag it as a cache directory @@ -108,13 +110,14 @@ func New(id string, basedir string) (c *Cache, err error) { return nil, errors.New("cache version is newer") } - err = updateTimestamp(cachedir) - if err != nil { + // create the repo cache dir if it does not exist yet + if err = fs.MkdirAll(cachedir, dirMode); err != nil { return nil, err } - // create the repo cache dir if it does not exist yet - if err = fs.MkdirAll(cachedir, dirMode); err != nil { + // update the timestamp so that we can detect old cache dirs + err = updateTimestamp(cachedir) + if err != nil { return nil, err } diff --git a/internal/cache/dir.go b/internal/cache/dir.go index d26db519f..55ec8ea9a 100644 --- a/internal/cache/dir.go +++ b/internal/cache/dir.go @@ -49,29 +49,25 @@ func darwinCacheDir() (string, error) { return filepath.Join(home, "Library", "Caches", "restic"), nil } -// defaultCacheDir determines and creates the default cache directory for this -// system. -func defaultCacheDir() (string, error) { - var cachedir string - var err error +// DefaultDir returns the default cache directory for the current OS. +func DefaultDir() (cachedir string, err error) { switch runtime.GOOS { case "darwin": cachedir, err = darwinCacheDir() case "windows": cachedir, err = windowsCacheDir() - default: - // Default to XDG for Linux and any other OSes. - cachedir, err = xdgCacheDir() - } - if err != nil { - return "", err } + // Default to XDG for Linux and any other OSes. + return xdgCacheDir() +} + +func mkdirCacheDir(cachedir string) error { fi, err := fs.Stat(cachedir) if os.IsNotExist(errors.Cause(err)) { err = fs.MkdirAll(cachedir, 0700) if err != nil { - return "", errors.Wrap(err, "MkdirAll") + return errors.Wrap(err, "MkdirAll") } fi, err = fs.Stat(cachedir) @@ -79,12 +75,12 @@ func defaultCacheDir() (string, error) { } if err != nil { - return "", errors.Wrap(err, "Stat") + return errors.Wrap(err, "Stat") } if !fi.IsDir() { - return "", errors.Errorf("cache dir %v is not a directory", cachedir) + return errors.Errorf("cache dir %v is not a directory", cachedir) } - return cachedir, nil + return nil } From 6341c7d72c8f773d5f14321150207c8b48a68f09 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 20 Nov 2017 22:08:53 +0100 Subject: [PATCH 3/5] cache: Add option to remove old cache dirs --- cmd/restic/global.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index e1c745104..df36df9b8 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -6,6 +6,7 @@ import ( "io" "io/ioutil" "os" + "path/filepath" "runtime" "strings" "syscall" @@ -23,6 +24,7 @@ import ( "github.com/restic/restic/internal/backend/swift" "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/limiter" "github.com/restic/restic/internal/options" "github.com/restic/restic/internal/repository" @@ -45,6 +47,7 @@ type GlobalOptions struct { CacheDir string NoCache bool CACerts []string + CleanupCache bool LimitUploadKb int LimitDownloadKb int @@ -81,6 +84,7 @@ func init() { f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache directory") f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache") f.StringSliceVar(&globalOptions.CACerts, "cacert", nil, "path to load root certificates from (default: use system certificates)") + f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.IntVar(&globalOptions.LimitUploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.LimitDownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)") f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)") @@ -362,8 +366,26 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { oldCacheDirs, err := cache.Old(c.Base) if err != nil { Warnf("unable to find old cache directories: %v", err) + } + + // nothing more to do if no old cache dirs could be found + if len(oldCacheDirs) == 0 { + return s, nil + } + + // cleanup old cache dirs if instructed to do so + if opts.CleanupCache { + Printf("removing %d old cache dirs from %v\n", len(oldCacheDirs), c.Base) + + for _, item := range oldCacheDirs { + dir := filepath.Join(c.Base, item) + err = fs.RemoveAll(dir) + if err != nil { + Warnf("unable to remove %v: %v\n", dir, err) + } + } } else { - Verbosef("found %d old cache directories in %v remove them with 'restic cache --cleanup'\n", + Verbosef("found %d old cache directories in %v, pass --cleanup-cache to remove them\n", len(oldCacheDirs), c.Base) } From 37ea764000c1c2102139b57f43afc339c74e9a2c Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 25 Nov 2017 11:28:59 +0100 Subject: [PATCH 4/5] cache: Add more documentation --- doc/cache.rst | 14 ++++++++++++-- doc/manual_rest.rst | 7 +++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/doc/cache.rst b/doc/cache.rst index b9dbf2797..a39a1e76c 100644 --- a/doc/cache.rst +++ b/doc/cache.rst @@ -19,8 +19,18 @@ a lower version number is found the cache is recreated with the current version. If a higher version number is found the cache is ignored and left as is. -Snapshots and Indexes ---------------------- +Snapshots, Data and Indexes +--------------------------- Snapshot, Data and Index files are cached in the sub-directories ``snapshots``, ``data`` and ``index``, as read from the repository. + +Expiry +------ + +Whenever a cache directory for a repo is used, that directory's modification +timestamp is updated to the current time. By looking at the modification +timestamps of the repo cache directories it is easy to decide which directories +are old and haven't been used in a long time. Those are probably stale and can +be removed. + diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index a3da3453e..be2e26805 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -280,3 +280,10 @@ entirely. In this case, all data is loaded from the repo. The cache is ephemeral: When a file cannot be read from the cache, it is loaded from the repository. + +Within the cache directory, there's a sub directory for each repository the +cache was used with. Restic updates the timestamps of a repo directory each +time it is used, so by looking at the timestamps of the sub directories of the +cache directory it can decide which sub directories are old and probably not +needed any more. You can either remove these directories manually, or run a +restic command with the ``--cleanup-cache`` flag. From fe557b022ad46fff73ec5695e06d47f036f11b7e Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 25 Nov 2017 12:12:08 +0100 Subject: [PATCH 5/5] Add entry to CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c95d4ea41..1bc72e9d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,14 @@ Important Changes in 0.X.Y that can be used to disable the cache. By deafult, the cache a standard cache folder for the OS, which can be overridden with `--cache-dir`. The cache will automatically populate, indexes and snapshots are saved as they - are loaded. + are loaded. Cache directories for repos that haven't been used recently can + automatically be removed by restic with the `--cleanup-cache` option. https://github.com/restic/restic/pull/1040 https://github.com/restic/restic/issues/29 https://github.com/restic/restic/issues/738 https://github.com/restic/restic/issues/282 https://github.com/restic/restic/pull/1287 + https://github.com/restic/restic/pull/1436 * A related change was to by default create pack files in the repo that contain either data or metadata, not both mixed together. This allows easy