mirror of
https://github.com/octoleo/restic.git
synced 2024-12-23 03:18:55 +00:00
commit
5a999cb77f
@ -387,6 +387,16 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// exclude restic cache
|
||||
if repo.Cache != nil {
|
||||
f, err := rejectResticCache(repo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rejectFuncs = append(rejectFuncs, f)
|
||||
}
|
||||
|
||||
err = repo.LoadIndex(context.TODO())
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -19,6 +19,9 @@ var cmdCheck = &cobra.Command{
|
||||
Long: `
|
||||
The "check" command tests the repository for errors and reports any errors it
|
||||
finds. It can also be used to read all data and therefore simulate a restore.
|
||||
|
||||
By default, the "check" command will always load all data directly from the
|
||||
repository and not use a local cache.
|
||||
`,
|
||||
DisableAutoGenTag: true,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
@ -30,6 +33,7 @@ finds. It can also be used to read all data and therefore simulate a restore.
|
||||
type CheckOptions struct {
|
||||
ReadData bool
|
||||
CheckUnused bool
|
||||
WithCache bool
|
||||
}
|
||||
|
||||
var checkOptions CheckOptions
|
||||
@ -40,6 +44,7 @@ func init() {
|
||||
f := cmdCheck.Flags()
|
||||
f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
|
||||
f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
|
||||
f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
|
||||
}
|
||||
|
||||
func newReadProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
|
||||
@ -77,6 +82,11 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
|
||||
return errors.Fatal("check has no arguments")
|
||||
}
|
||||
|
||||
if !opts.WithCache {
|
||||
// do not use a cache for the checker
|
||||
gopts.NoCache = true
|
||||
}
|
||||
|
||||
repo, err := OpenRepository(gopts)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -85,6 +85,25 @@ func runPrune(gopts GlobalOptions) error {
|
||||
return pruneRepository(gopts, repo)
|
||||
}
|
||||
|
||||
func mixedBlobs(list []restic.Blob) bool {
|
||||
var tree, data bool
|
||||
|
||||
for _, pb := range list {
|
||||
switch pb.Type {
|
||||
case restic.TreeBlob:
|
||||
tree = true
|
||||
case restic.DataBlob:
|
||||
data = true
|
||||
}
|
||||
|
||||
if tree && data {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
|
||||
ctx := gopts.ctx
|
||||
|
||||
@ -191,6 +210,11 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
|
||||
// find packs that need a rewrite
|
||||
rewritePacks := restic.NewIDSet()
|
||||
for _, pack := range idx.Packs {
|
||||
if mixedBlobs(pack.Entries) {
|
||||
rewritePacks.Insert(pack.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, blob := range pack.Entries {
|
||||
h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
|
||||
if !usedBlobs.Has(h) {
|
||||
|
@ -12,6 +12,7 @@ import (
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/filter"
|
||||
"github.com/restic/restic/internal/fs"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
)
|
||||
|
||||
// RejectFunc is a function that takes a filename and os.FileInfo of a
|
||||
@ -177,3 +178,27 @@ func rejectByDevice(samples []string) (RejectFunc, error) {
|
||||
panic(fmt.Sprintf("item %v, device id %v not found, allowedDevs: %v", item, id, allowed))
|
||||
}, nil
|
||||
}
|
||||
|
||||
// rejectResticCache returns a RejectFunc that rejects the restic cache
|
||||
// directory (if set).
|
||||
func rejectResticCache(repo *repository.Repository) (RejectFunc, error) {
|
||||
if repo.Cache == nil {
|
||||
return func(string, os.FileInfo) bool {
|
||||
return false
|
||||
}, nil
|
||||
}
|
||||
cacheBase := repo.Cache.BaseDir()
|
||||
|
||||
if cacheBase == "" {
|
||||
return nil, errors.New("cacheBase is empty string")
|
||||
}
|
||||
|
||||
return func(item string, _ os.FileInfo) bool {
|
||||
if fs.HasPathPrefix(cacheBase, item) {
|
||||
debug.Log("rejecting restic cache directory %v", item)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}, nil
|
||||
}
|
||||
|
@ -19,6 +19,7 @@ import (
|
||||
"github.com/restic/restic/internal/backend/s3"
|
||||
"github.com/restic/restic/internal/backend/sftp"
|
||||
"github.com/restic/restic/internal/backend/swift"
|
||||
"github.com/restic/restic/internal/cache"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/options"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
@ -38,6 +39,8 @@ type GlobalOptions struct {
|
||||
Quiet bool
|
||||
NoLock bool
|
||||
JSON bool
|
||||
CacheDir string
|
||||
NoCache bool
|
||||
|
||||
ctx context.Context
|
||||
password string
|
||||
@ -68,7 +71,8 @@ func init() {
|
||||
f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report")
|
||||
f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repo, this allows some operations on read-only repos")
|
||||
f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it")
|
||||
|
||||
f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache directory")
|
||||
f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache")
|
||||
f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
|
||||
|
||||
restoreTerminal()
|
||||
@ -322,6 +326,17 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if opts.NoCache {
|
||||
return s, nil
|
||||
}
|
||||
|
||||
cache, err := cache.New(s.Config().ID, opts.CacheDir)
|
||||
if err != nil {
|
||||
Warnf("unable to open cache: %v\n", err)
|
||||
} else {
|
||||
s.UseCache(cache)
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
|
@ -199,6 +199,7 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
|
||||
env.gopts = GlobalOptions{
|
||||
Repo: env.repo,
|
||||
Quiet: true,
|
||||
CacheDir: env.cache,
|
||||
ctx: context.Background(),
|
||||
password: TestPassword,
|
||||
stdout: os.Stdout,
|
||||
|
26
doc/cache.rst
Normal file
26
doc/cache.rst
Normal file
@ -0,0 +1,26 @@
|
||||
Local Cache
|
||||
===========
|
||||
|
||||
In order to speed up certain operations, restic manages a local cache of data.
|
||||
This document describes the data structures for the local cache with version 1.
|
||||
|
||||
Versions
|
||||
--------
|
||||
|
||||
The cache directory is selected according to the `XDG base dir specification
|
||||
<http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html>`__.
|
||||
Each repository has its own cache sub-directory, consting of the repository ID
|
||||
which is chosen at ``init``. All cache directories for different repos are
|
||||
independent of each other.
|
||||
|
||||
The cache dir for a repo contains a file named ``version``, which contains a
|
||||
single ASCII integer line that stands for the current version of the cache. If
|
||||
a lower version number is found the cache is recreated with the current
|
||||
version. If a higher version number is found the cache is ignored and left as
|
||||
is.
|
||||
|
||||
Snapshots and Indexes
|
||||
---------------------
|
||||
|
||||
Snapshot, Data and Index files are cached in the sub-directories ``snapshots``,
|
||||
``data`` and ``index``, as read from the repository.
|
@ -40,10 +40,18 @@ $ sudo restic autocomplete
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -78,10 +78,18 @@ given as the arguments.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -25,10 +25,18 @@ The "cat" command is used to print internal objects to stdout.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -18,6 +18,10 @@ restic\-check \- Check the repository for errors
|
||||
The "check" command tests the repository for errors and reports any errors it
|
||||
finds. It can also be used to read all data and therefore simulate a restore.
|
||||
|
||||
.PP
|
||||
By default, the "check" command will always load all data directly from the
|
||||
repository and not use a local cache.
|
||||
|
||||
|
||||
.SH OPTIONS
|
||||
.PP
|
||||
@ -32,12 +36,24 @@ finds. It can also be used to read all data and therefore simulate a restore.
|
||||
\fB\-\-read\-data\fP[=false]
|
||||
read all data blobs
|
||||
|
||||
.PP
|
||||
\fB\-\-with\-cache\fP[=false]
|
||||
use the cache
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -26,10 +26,18 @@ is used for debugging purposes only.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -58,10 +58,18 @@ repo.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -88,10 +88,18 @@ data after 'forget' was run successfully, see the 'prune' command.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -25,10 +25,18 @@ The "init" command initializes a new repository.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -25,10 +25,18 @@ The "key" command manages keys (passwords) for accessing the repository.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -25,10 +25,18 @@ The "list" command allows listing objects in the repository based on type.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -44,10 +44,18 @@ The special snapshot\-ID "latest" can be used to list files and directories of t
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -31,10 +31,18 @@ set and no command is specified, all manpages are written to the directory.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -30,10 +30,18 @@ name is explicitly given, a list of migrations that can be applied is printed.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -50,10 +50,18 @@ read\-only mount.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -26,10 +26,18 @@ referenced and therefore not needed any more.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -26,10 +26,18 @@ repository.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -54,10 +54,18 @@ repository.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -41,10 +41,18 @@ The "snapshots" command lists all snapshots stored in the repository.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -56,10 +56,18 @@ When no snapshot\-ID is given, all snapshots matching the host, tag and path fil
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -29,10 +29,18 @@ The "unlock" command removes stale locks that have been created by other restic
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -26,10 +26,18 @@ and the version of this software.
|
||||
|
||||
|
||||
.SH OPTIONS INHERITED FROM PARENT COMMANDS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -20,6 +20,10 @@ directories in an encrypted repository stored on different backends.
|
||||
|
||||
|
||||
.SH OPTIONS
|
||||
.PP
|
||||
\fB\-\-cache\-dir\fP=""
|
||||
set the cache directory
|
||||
|
||||
.PP
|
||||
\fB\-h\fP, \fB\-\-help\fP[=false]
|
||||
help for restic
|
||||
@ -28,6 +32,10 @@ directories in an encrypted repository stored on different backends.
|
||||
\fB\-\-json\fP[=false]
|
||||
set output mode to JSON for commands that support it
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-cache\fP[=false]
|
||||
do not use a local cache
|
||||
|
||||
.PP
|
||||
\fB\-\-no\-lock\fP[=false]
|
||||
do not lock the repo, this allows some operations on read\-only repos
|
||||
|
@ -1240,3 +1240,19 @@ instead of the default, set the environment variable like this:
|
||||
|
||||
$ export TMPDIR=/var/tmp/restic-tmp
|
||||
$ restic -r /tmp/backup backup ~/work
|
||||
|
||||
Caching
|
||||
-------
|
||||
|
||||
Restic keeps a cache with some files from the repository on the local machine.
|
||||
This allows faster operations, since meta data does not need to be loaded from
|
||||
a remote repository. The cache is automatically created, usually in the
|
||||
directory ``.cache/restic`` in the user's home directory. The environment
|
||||
variable ``XDG_CACHE_DIR`` or the command line parameter ``--cache-dir`` can
|
||||
each be used to specify where the cache is located. The parameter
|
||||
``--no-cache`` disables the cache entirely. In this case, all data is loaded
|
||||
from the repo.
|
||||
|
||||
The cache is ephemeral: When a file cannot be read from the cache, it is loaded
|
||||
from the repository.
|
||||
|
||||
|
@ -7,3 +7,7 @@ References
|
||||
------------------------
|
||||
|
||||
.. include:: rest_backend.rst
|
||||
|
||||
------------------------
|
||||
|
||||
.. include:: cache.rst
|
||||
|
170
internal/cache/backend.go
vendored
Normal file
170
internal/cache/backend.go
vendored
Normal file
@ -0,0 +1,170 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// Backend wraps a restic.Backend and adds a cache.
|
||||
type Backend struct {
|
||||
restic.Backend
|
||||
*Cache
|
||||
}
|
||||
|
||||
// ensure cachedBackend implements restic.Backend
|
||||
var _ restic.Backend = &Backend{}
|
||||
|
||||
// Remove deletes a file from the backend and the cache if it has been cached.
|
||||
func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
|
||||
debug.Log("cache Remove(%v)", h)
|
||||
err := b.Backend.Remove(ctx, h)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return b.Cache.Remove(h)
|
||||
}
|
||||
|
||||
type teeReader struct {
|
||||
rd io.Reader
|
||||
wr io.Writer
|
||||
err error
|
||||
}
|
||||
|
||||
func (t *teeReader) Read(p []byte) (n int, err error) {
|
||||
n, err = t.rd.Read(p)
|
||||
if t.err == nil && n > 0 {
|
||||
_, t.err = t.wr.Write(p[:n])
|
||||
}
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
var autoCacheTypes = map[restic.FileType]struct{}{
|
||||
restic.IndexFile: struct{}{},
|
||||
restic.SnapshotFile: struct{}{},
|
||||
}
|
||||
|
||||
// Save stores a new file is the backend and the cache.
|
||||
func (b *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
|
||||
if _, ok := autoCacheTypes[h.Type]; !ok {
|
||||
return b.Backend.Save(ctx, h, rd)
|
||||
}
|
||||
|
||||
debug.Log("Save(%v): auto-store in the cache", h)
|
||||
wr, err := b.Cache.SaveWriter(h)
|
||||
if err != nil {
|
||||
debug.Log("unable to save %v to cache: %v", h, err)
|
||||
return b.Backend.Save(ctx, h, rd)
|
||||
}
|
||||
|
||||
tr := &teeReader{rd: rd, wr: wr}
|
||||
err = b.Backend.Save(ctx, h, tr)
|
||||
if err != nil {
|
||||
wr.Close()
|
||||
b.Cache.Remove(h)
|
||||
return err
|
||||
}
|
||||
|
||||
err = wr.Close()
|
||||
if err != nil {
|
||||
debug.Log("cache writer returned error: %v", err)
|
||||
_ = b.Cache.Remove(h)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var autoCacheFiles = map[restic.FileType]bool{
|
||||
restic.IndexFile: true,
|
||||
restic.SnapshotFile: true,
|
||||
}
|
||||
|
||||
func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
|
||||
rd, err := b.Backend.Load(ctx, h, 0, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = b.Cache.Save(h, rd); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = rd.Close(); err != nil {
|
||||
// try to remove from the cache, ignore errors
|
||||
_ = b.Cache.Remove(h)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Load loads a file from the cache or the backend.
|
||||
func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
|
||||
if b.Cache.Has(h) {
|
||||
debug.Log("Load(%v, %v, %v) from cache", h, length, offset)
|
||||
rd, err := b.Cache.Load(h, length, offset)
|
||||
if err == nil {
|
||||
return rd, nil
|
||||
}
|
||||
debug.Log("error loading %v from cache: %v", h, err)
|
||||
}
|
||||
|
||||
// partial file requested
|
||||
if offset != 0 || length != 0 {
|
||||
if b.Cache.PerformReadahead(h) {
|
||||
debug.Log("performing readahead for %v", h)
|
||||
err := b.cacheFile(ctx, h)
|
||||
if err == nil {
|
||||
return b.Cache.Load(h, length, offset)
|
||||
}
|
||||
|
||||
debug.Log("error caching %v: %v", h, err)
|
||||
}
|
||||
|
||||
debug.Log("Load(%v, %v, %v): partial file requested, delegating to backend", h, length, offset)
|
||||
return b.Backend.Load(ctx, h, length, offset)
|
||||
}
|
||||
|
||||
// if we don't automatically cache this file type, fall back to the backend
|
||||
if _, ok := autoCacheFiles[h.Type]; !ok {
|
||||
debug.Log("Load(%v, %v, %v): delegating to backend", h, length, offset)
|
||||
return b.Backend.Load(ctx, h, length, offset)
|
||||
}
|
||||
|
||||
debug.Log("auto-store %v in the cache", h)
|
||||
err := b.cacheFile(ctx, h)
|
||||
|
||||
if err == nil {
|
||||
// load the cached version
|
||||
return b.Cache.Load(h, 0, 0)
|
||||
}
|
||||
|
||||
debug.Log("error caching %v: %v, falling back to backend", h, err)
|
||||
return b.Backend.Load(ctx, h, length, offset)
|
||||
}
|
||||
|
||||
// Stat tests whether the backend has a file. If it does not exist but still
|
||||
// exists in the cache, it is removed from the cache.
|
||||
func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
|
||||
debug.Log("cache Stat(%v)", h)
|
||||
|
||||
fi, err := b.Backend.Stat(ctx, h)
|
||||
if err != nil {
|
||||
if b.Backend.IsNotExist(err) {
|
||||
// try to remove from the cache, ignore errors
|
||||
_ = b.Cache.Remove(h)
|
||||
}
|
||||
|
||||
return fi, err
|
||||
}
|
||||
|
||||
return fi, err
|
||||
}
|
||||
|
||||
// IsNotExist returns true if the error is caused by a non-existing file.
|
||||
func (b *Backend) IsNotExist(err error) bool {
|
||||
return b.Backend.IsNotExist(err)
|
||||
}
|
114
internal/cache/backend_test.go
vendored
Normal file
114
internal/cache/backend_test.go
vendored
Normal file
@ -0,0 +1,114 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"math/rand"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/backend/mem"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func loadAndCompare(t testing.TB, be restic.Backend, h restic.Handle, data []byte) {
|
||||
buf, err := backend.LoadAll(context.TODO(), be, h)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(buf) != len(data) {
|
||||
t.Fatalf("wrong number of bytes read, want %v, got %v", len(data), len(buf))
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf, data) {
|
||||
t.Fatalf("wrong data returned, want:\n %02x\ngot:\n %02x", data[:16], buf[:16])
|
||||
}
|
||||
}
|
||||
|
||||
func save(t testing.TB, be restic.Backend, h restic.Handle, data []byte) {
|
||||
err := be.Save(context.TODO(), h, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func remove(t testing.TB, be restic.Backend, h restic.Handle) {
|
||||
err := be.Remove(context.TODO(), h)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func randomData(n int) (restic.Handle, []byte) {
|
||||
data := test.Random(rand.Int(), n)
|
||||
id := restic.Hash(data)
|
||||
copy(id[:], data)
|
||||
h := restic.Handle{
|
||||
Type: restic.IndexFile,
|
||||
Name: id.String(),
|
||||
}
|
||||
return h, data
|
||||
}
|
||||
|
||||
func TestBackend(t *testing.T) {
|
||||
be := mem.New()
|
||||
|
||||
c, cleanup := TestNewCache(t)
|
||||
defer cleanup()
|
||||
|
||||
wbe := c.Wrap(be)
|
||||
|
||||
h, data := randomData(5234142)
|
||||
|
||||
// save directly in backend
|
||||
save(t, be, h, data)
|
||||
if c.Has(h) {
|
||||
t.Errorf("cache has file too early")
|
||||
}
|
||||
|
||||
// load data via cache
|
||||
loadAndCompare(t, wbe, h, data)
|
||||
if !c.Has(h) {
|
||||
t.Errorf("cache dosen't have file after load")
|
||||
}
|
||||
|
||||
// remove via cache
|
||||
remove(t, wbe, h)
|
||||
if c.Has(h) {
|
||||
t.Errorf("cache has file after remove")
|
||||
}
|
||||
|
||||
// save via cache
|
||||
save(t, wbe, h, data)
|
||||
if !c.Has(h) {
|
||||
t.Errorf("cache dosen't have file after load")
|
||||
}
|
||||
|
||||
// load data directly from backend
|
||||
loadAndCompare(t, be, h, data)
|
||||
|
||||
// load data via cache
|
||||
loadAndCompare(t, be, h, data)
|
||||
|
||||
// remove directly
|
||||
remove(t, be, h)
|
||||
if !c.Has(h) {
|
||||
t.Errorf("file not in cache any more")
|
||||
}
|
||||
|
||||
// run stat
|
||||
_, err := wbe.Stat(context.TODO(), h)
|
||||
if err == nil {
|
||||
t.Errorf("expected error for removed file not found, got nil")
|
||||
}
|
||||
|
||||
if !wbe.IsNotExist(err) {
|
||||
t.Errorf("Stat() returned error that does not match IsNotExist(): %v", err)
|
||||
}
|
||||
|
||||
if c.Has(h) {
|
||||
t.Errorf("removed file still in cache after stat")
|
||||
}
|
||||
}
|
167
internal/cache/cache.go
vendored
Normal file
167
internal/cache/cache.go
vendored
Normal file
@ -0,0 +1,167 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/fs"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// Cache manages a local cache.
|
||||
type Cache struct {
|
||||
Path string
|
||||
Base string
|
||||
PerformReadahead func(restic.Handle) bool
|
||||
}
|
||||
|
||||
const dirMode = 0700
|
||||
const fileMode = 0600
|
||||
|
||||
func readVersion(dir string) (v uint, err error) {
|
||||
buf, err := ioutil.ReadFile(filepath.Join(dir, "version"))
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return 0, errors.Wrap(err, "ReadFile")
|
||||
}
|
||||
|
||||
ver, err := strconv.ParseUint(string(buf), 10, 32)
|
||||
if err != nil {
|
||||
return 0, errors.Wrap(err, "ParseUint")
|
||||
}
|
||||
|
||||
return uint(ver), nil
|
||||
}
|
||||
|
||||
const cacheVersion = 1
|
||||
|
||||
// ensure Cache implements restic.Cache
|
||||
var _ restic.Cache = &Cache{}
|
||||
|
||||
var cacheLayoutPaths = map[restic.FileType]string{
|
||||
restic.DataFile: "data",
|
||||
restic.SnapshotFile: "snapshots",
|
||||
restic.IndexFile: "index",
|
||||
}
|
||||
|
||||
const cachedirTagSignature = "Signature: 8a477f597d28d172789f06886806bc55\n"
|
||||
|
||||
func writeCachedirTag(dir string) error {
|
||||
if err := fs.MkdirAll(dir, dirMode); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f, err := fs.OpenFile(filepath.Join(dir, "CACHEDIR.TAG"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
if os.IsExist(errors.Cause(err)) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return errors.Wrap(err, "OpenFile")
|
||||
}
|
||||
|
||||
debug.Log("Create CACHEDIR.TAG at %v", dir)
|
||||
if _, err := f.Write([]byte(cachedirTagSignature)); err != nil {
|
||||
f.Close()
|
||||
return errors.Wrap(err, "Write")
|
||||
}
|
||||
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
// New returns a new cache for the repo ID at basedir. If basedir is the empty
|
||||
// string, the default cache location (according to the XDG standard) is used.
|
||||
//
|
||||
// For partial files, the complete file is loaded and stored in the cache when
|
||||
// performReadahead returns true.
|
||||
func New(id string, basedir string) (c *Cache, err error) {
|
||||
if basedir == "" {
|
||||
basedir, err = getXDGCacheDir()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// create base dir and tag it as a cache directory
|
||||
if err = writeCachedirTag(basedir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cachedir := filepath.Join(basedir, id)
|
||||
debug.Log("using cache dir %v", cachedir)
|
||||
|
||||
v, err := readVersion(cachedir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if v > cacheVersion {
|
||||
return nil, errors.New("cache version is newer")
|
||||
}
|
||||
|
||||
// create the repo cache dir if it does not exist yet
|
||||
if err = fs.MkdirAll(cachedir, dirMode); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if v < cacheVersion {
|
||||
err = ioutil.WriteFile(filepath.Join(cachedir, "version"), []byte(fmt.Sprintf("%d", cacheVersion)), 0644)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "WriteFile")
|
||||
}
|
||||
}
|
||||
|
||||
for _, p := range cacheLayoutPaths {
|
||||
if err = fs.MkdirAll(filepath.Join(cachedir, p), dirMode); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
c = &Cache{
|
||||
Path: cachedir,
|
||||
Base: basedir,
|
||||
PerformReadahead: func(restic.Handle) bool {
|
||||
// do not perform readahead by default
|
||||
return false
|
||||
},
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// errNoSuchFile is returned when a file is not cached.
|
||||
type errNoSuchFile struct {
|
||||
Type string
|
||||
Name string
|
||||
}
|
||||
|
||||
func (e errNoSuchFile) Error() string {
|
||||
return fmt.Sprintf("file %v (%v) is not cached", e.Name, e.Type)
|
||||
}
|
||||
|
||||
// IsNotExist returns true if the error was caused by a non-existing file.
|
||||
func (c *Cache) IsNotExist(err error) bool {
|
||||
_, ok := errors.Cause(err).(errNoSuchFile)
|
||||
return ok
|
||||
}
|
||||
|
||||
// Wrap returns a backend with a cache.
|
||||
func (c *Cache) Wrap(be restic.Backend) restic.Backend {
|
||||
return &Backend{
|
||||
Backend: be,
|
||||
Cache: c,
|
||||
}
|
||||
}
|
||||
|
||||
// BaseDir returns the base directory.
|
||||
func (c *Cache) BaseDir() string {
|
||||
return c.Base
|
||||
}
|
49
internal/cache/dir.go
vendored
Normal file
49
internal/cache/dir.go
vendored
Normal file
@ -0,0 +1,49 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/fs"
|
||||
)
|
||||
|
||||
// getXDGCacheDir returns the cache directory according to XDG basedir spec, see
|
||||
// http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
|
||||
func getXDGCacheDir() (string, error) {
|
||||
xdgcache := os.Getenv("XDG_CACHE_HOME")
|
||||
home := os.Getenv("HOME")
|
||||
|
||||
if xdgcache == "" && home == "" {
|
||||
return "", errors.New("unable to locate cache directory (XDG_CACHE_HOME and HOME unset)")
|
||||
}
|
||||
|
||||
cachedir := ""
|
||||
if xdgcache != "" {
|
||||
cachedir = filepath.Join(xdgcache, "restic")
|
||||
} else if home != "" {
|
||||
cachedir = filepath.Join(home, ".cache", "restic")
|
||||
}
|
||||
|
||||
fi, err := fs.Stat(cachedir)
|
||||
if os.IsNotExist(errors.Cause(err)) {
|
||||
err = fs.MkdirAll(cachedir, 0700)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "MkdirAll")
|
||||
}
|
||||
|
||||
fi, err = fs.Stat(cachedir)
|
||||
debug.Log("create cache dir %v", cachedir)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "Stat")
|
||||
}
|
||||
|
||||
if !fi.IsDir() {
|
||||
return "", errors.Errorf("cache dir %v is not a directory", cachedir)
|
||||
}
|
||||
|
||||
return cachedir, nil
|
||||
}
|
207
internal/cache/file.go
vendored
Normal file
207
internal/cache/file.go
vendored
Normal file
@ -0,0 +1,207 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/restic/restic/internal/crypto"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/fs"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
func (c *Cache) filename(h restic.Handle) string {
|
||||
if len(h.Name) < 2 {
|
||||
panic("Name is empty or too short")
|
||||
}
|
||||
subdir := h.Name[:2]
|
||||
return filepath.Join(c.Path, cacheLayoutPaths[h.Type], subdir, h.Name)
|
||||
}
|
||||
|
||||
func (c *Cache) canBeCached(t restic.FileType) bool {
|
||||
if c == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if _, ok := cacheLayoutPaths[t]; !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
type readCloser struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// Load returns a reader that yields the contents of the file with the
|
||||
// given handle. rd must be closed after use. If an error is returned, the
|
||||
// ReadCloser is nil.
|
||||
func (c *Cache) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
|
||||
debug.Log("Load from cache: %v", h)
|
||||
if !c.canBeCached(h.Type) {
|
||||
return nil, errors.New("cannot be cached")
|
||||
}
|
||||
|
||||
f, err := fs.Open(c.filename(h))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Open")
|
||||
}
|
||||
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return nil, errors.Wrap(err, "Stat")
|
||||
}
|
||||
|
||||
if fi.Size() <= crypto.Extension {
|
||||
_ = f.Close()
|
||||
_ = c.Remove(h)
|
||||
return nil, errors.New("cached file is truncated, removing")
|
||||
}
|
||||
|
||||
if offset > 0 {
|
||||
if _, err = f.Seek(offset, io.SeekStart); err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
rd := readCloser{Reader: f, Closer: f}
|
||||
if length > 0 {
|
||||
rd.Reader = io.LimitReader(f, int64(length))
|
||||
}
|
||||
|
||||
return rd, nil
|
||||
}
|
||||
|
||||
// SaveWriter returns a writer for the cache object h. It must be closed after writing is finished.
|
||||
func (c *Cache) SaveWriter(h restic.Handle) (io.WriteCloser, error) {
|
||||
debug.Log("Save to cache: %v", h)
|
||||
if !c.canBeCached(h.Type) {
|
||||
return nil, errors.New("cannot be cached")
|
||||
}
|
||||
|
||||
p := c.filename(h)
|
||||
err := fs.MkdirAll(filepath.Dir(p), 0700)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "MkdirAll")
|
||||
}
|
||||
|
||||
f, err := fs.OpenFile(p, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0400)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Create")
|
||||
}
|
||||
|
||||
return f, err
|
||||
}
|
||||
|
||||
// Save saves a file in the cache.
|
||||
func (c *Cache) Save(h restic.Handle, rd io.Reader) error {
|
||||
debug.Log("Save to cache: %v", h)
|
||||
if rd == nil {
|
||||
return errors.New("Save() called with nil reader")
|
||||
}
|
||||
|
||||
f, err := c.SaveWriter(h)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err = io.Copy(f, rd); err != nil {
|
||||
_ = f.Close()
|
||||
_ = c.Remove(h)
|
||||
return errors.Wrap(err, "Copy")
|
||||
}
|
||||
|
||||
if err = f.Close(); err != nil {
|
||||
return errors.Wrap(err, "Close")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove deletes a file. When the file is not cache, no error is returned.
|
||||
func (c *Cache) Remove(h restic.Handle) error {
|
||||
if !c.Has(h) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return fs.Remove(c.filename(h))
|
||||
}
|
||||
|
||||
// Clear removes all files of type t from the cache that are not contained in
|
||||
// the set valid.
|
||||
func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
|
||||
debug.Log("Clearing cache for %v: %v valid files", t, len(valid))
|
||||
if !c.canBeCached(t) {
|
||||
return nil
|
||||
}
|
||||
|
||||
list, err := c.list(t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for id := range list {
|
||||
if valid.Has(id) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err = fs.Remove(c.filename(restic.Handle{Type: t, Name: id.String()})); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isFile(fi os.FileInfo) bool {
|
||||
return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0
|
||||
}
|
||||
|
||||
// list returns a list of all files of type T in the cache.
|
||||
func (c *Cache) list(t restic.FileType) (restic.IDSet, error) {
|
||||
if !c.canBeCached(t) {
|
||||
return nil, errors.New("cannot be cached")
|
||||
}
|
||||
|
||||
list := restic.NewIDSet()
|
||||
dir := filepath.Join(c.Path, cacheLayoutPaths[t])
|
||||
err := filepath.Walk(dir, func(name string, fi os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Walk")
|
||||
}
|
||||
|
||||
if !isFile(fi) {
|
||||
return nil
|
||||
}
|
||||
|
||||
id, err := restic.ParseID(filepath.Base(name))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
list.Insert(id)
|
||||
return nil
|
||||
})
|
||||
|
||||
return list, err
|
||||
}
|
||||
|
||||
// Has returns true if the file is cached.
|
||||
func (c *Cache) Has(h restic.Handle) bool {
|
||||
if !c.canBeCached(h.Type) {
|
||||
return false
|
||||
}
|
||||
|
||||
_, err := fs.Stat(c.filename(h))
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
259
internal/cache/file_test.go
vendored
Normal file
259
internal/cache/file_test.go
vendored
Normal file
@ -0,0 +1,259 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func generateRandomFiles(t testing.TB, tpe restic.FileType, c *Cache) restic.IDSet {
|
||||
ids := restic.NewIDSet()
|
||||
for i := 0; i < rand.Intn(15)+10; i++ {
|
||||
buf := test.Random(rand.Int(), 1<<19)
|
||||
id := restic.Hash(buf)
|
||||
h := restic.Handle{Type: tpe, Name: id.String()}
|
||||
|
||||
if c.Has(h) {
|
||||
t.Errorf("index %v present before save", id)
|
||||
}
|
||||
|
||||
err := c.Save(h, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids.Insert(id)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// randomID returns a random ID from s.
|
||||
func randomID(s restic.IDSet) restic.ID {
|
||||
for id := range s {
|
||||
return id
|
||||
}
|
||||
panic("set is empty")
|
||||
}
|
||||
|
||||
func load(t testing.TB, c *Cache, h restic.Handle) []byte {
|
||||
rd, err := c.Load(h, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if rd == nil {
|
||||
t.Fatalf("Load() returned nil reader")
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadAll(rd)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err = rd.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return buf
|
||||
}
|
||||
|
||||
func listFiles(t testing.TB, c *Cache, tpe restic.FileType) restic.IDSet {
|
||||
list, err := c.list(tpe)
|
||||
if err != nil {
|
||||
t.Errorf("listing failed: %v", err)
|
||||
}
|
||||
|
||||
return list
|
||||
}
|
||||
|
||||
func clearFiles(t testing.TB, c *Cache, tpe restic.FileType, valid restic.IDSet) {
|
||||
if err := c.Clear(tpe, valid); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFiles(t *testing.T) {
|
||||
seed := time.Now().Unix()
|
||||
t.Logf("seed is %v", seed)
|
||||
rand.Seed(seed)
|
||||
|
||||
c, cleanup := TestNewCache(t)
|
||||
defer cleanup()
|
||||
|
||||
var tests = []restic.FileType{
|
||||
restic.SnapshotFile,
|
||||
restic.DataFile,
|
||||
restic.IndexFile,
|
||||
}
|
||||
|
||||
for _, tpe := range tests {
|
||||
t.Run(fmt.Sprintf("%v", tpe), func(t *testing.T) {
|
||||
ids := generateRandomFiles(t, tpe, c)
|
||||
id := randomID(ids)
|
||||
|
||||
h := restic.Handle{Type: tpe, Name: id.String()}
|
||||
id2 := restic.Hash(load(t, c, h))
|
||||
|
||||
if !id.Equal(id2) {
|
||||
t.Errorf("wrong data returned, want %v, got %v", id.Str(), id2.Str())
|
||||
}
|
||||
|
||||
if !c.Has(h) {
|
||||
t.Errorf("cache thinks index %v isn't present", id.Str())
|
||||
}
|
||||
|
||||
list := listFiles(t, c, tpe)
|
||||
if !ids.Equals(list) {
|
||||
t.Errorf("wrong list of index IDs returned, want:\n %v\ngot:\n %v", ids, list)
|
||||
}
|
||||
|
||||
clearFiles(t, c, tpe, restic.NewIDSet(id))
|
||||
list2 := listFiles(t, c, tpe)
|
||||
ids.Delete(id)
|
||||
want := restic.NewIDSet(id)
|
||||
if !list2.Equals(want) {
|
||||
t.Errorf("ClearIndexes removed indexes, want:\n %v\ngot:\n %v", list2, want)
|
||||
}
|
||||
|
||||
clearFiles(t, c, tpe, restic.NewIDSet())
|
||||
want = restic.NewIDSet()
|
||||
list3 := listFiles(t, c, tpe)
|
||||
if !list3.Equals(want) {
|
||||
t.Errorf("ClearIndexes returned a wrong list, want:\n %v\ngot:\n %v", want, list3)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSaveWriter(t *testing.T) {
|
||||
seed := time.Now().Unix()
|
||||
t.Logf("seed is %v", seed)
|
||||
rand.Seed(seed)
|
||||
|
||||
c, cleanup := TestNewCache(t)
|
||||
defer cleanup()
|
||||
|
||||
// save about 5 MiB of data in the cache
|
||||
data := test.Random(rand.Int(), 5234142)
|
||||
id := restic.ID{}
|
||||
copy(id[:], data)
|
||||
h := restic.Handle{
|
||||
Type: restic.DataFile,
|
||||
Name: id.String(),
|
||||
}
|
||||
|
||||
wr, err := c.SaveWriter(h)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
n, err := io.Copy(wr, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if n != int64(len(data)) {
|
||||
t.Fatalf("wrong number of bytes written, want %v, got %v", len(data), n)
|
||||
}
|
||||
|
||||
if err = wr.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rd, err := c.Load(h, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadAll(rd)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(buf) != len(data) {
|
||||
t.Fatalf("wrong number of bytes read, want %v, got %v", len(data), len(buf))
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf, data) {
|
||||
t.Fatalf("wrong data returned, want:\n %02x\ngot:\n %02x", data[:16], buf[:16])
|
||||
}
|
||||
|
||||
if err = rd.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileLoad(t *testing.T) {
|
||||
seed := time.Now().Unix()
|
||||
t.Logf("seed is %v", seed)
|
||||
rand.Seed(seed)
|
||||
|
||||
c, cleanup := TestNewCache(t)
|
||||
defer cleanup()
|
||||
|
||||
// save about 5 MiB of data in the cache
|
||||
data := test.Random(rand.Int(), 5234142)
|
||||
id := restic.ID{}
|
||||
copy(id[:], data)
|
||||
h := restic.Handle{
|
||||
Type: restic.DataFile,
|
||||
Name: id.String(),
|
||||
}
|
||||
if err := c.Save(h, bytes.NewReader(data)); err != nil {
|
||||
t.Fatalf("Save() returned error: %v", err)
|
||||
}
|
||||
|
||||
var tests = []struct {
|
||||
offset int64
|
||||
length int
|
||||
}{
|
||||
{0, 0},
|
||||
{5, 0},
|
||||
{32*1024 + 5, 0},
|
||||
{0, 123},
|
||||
{0, 64*1024 + 234},
|
||||
{100, 5234142},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(fmt.Sprintf("%v/%v", test.length, test.offset), func(t *testing.T) {
|
||||
rd, err := c.Load(h, test.length, test.offset)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadAll(rd)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err = rd.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
o := int(test.offset)
|
||||
l := test.length
|
||||
if test.length == 0 {
|
||||
l = len(data) - o
|
||||
}
|
||||
|
||||
if l > len(data)-o {
|
||||
l = len(data) - o
|
||||
}
|
||||
|
||||
if len(buf) != l {
|
||||
t.Fatalf("wrong number of bytes returned: want %d, got %d", l, len(buf))
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf, data[o:o+l]) {
|
||||
t.Fatalf("wrong data returned, want:\n %02x\ngot:\n %02x", data[o:o+16], buf[:16])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
20
internal/cache/testing.go
vendored
Normal file
20
internal/cache/testing.go
vendored
Normal file
@ -0,0 +1,20 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
// TestNewCache returns a cache in a temporary directory which is removed when
|
||||
// cleanup is called.
|
||||
func TestNewCache(t testing.TB) (*Cache, func()) {
|
||||
dir, cleanup := test.TempDir(t)
|
||||
t.Logf("created new cache at %v", dir)
|
||||
cache, err := New(restic.NewRandomID().String(), dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return cache, cleanup
|
||||
}
|
@ -15,8 +15,9 @@ import (
|
||||
|
||||
// Index holds a lookup table for id -> pack.
|
||||
type Index struct {
|
||||
m sync.Mutex
|
||||
pack map[restic.BlobHandle][]indexEntry
|
||||
m sync.Mutex
|
||||
pack map[restic.BlobHandle][]indexEntry
|
||||
treePacks restic.IDs
|
||||
|
||||
final bool // set to true for all indexes read from the backend ("finalized")
|
||||
id restic.ID // set to the ID of the index when it's finalized
|
||||
@ -437,6 +438,11 @@ func (idx *Index) Dump(w io.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// TreePacks returns a list of packs that contain only tree blobs.
|
||||
func (idx *Index) TreePacks() restic.IDs {
|
||||
return idx.treePacks
|
||||
}
|
||||
|
||||
// isErrOldIndex returns true if the error may be caused by an old index
|
||||
// format.
|
||||
func isErrOldIndex(err error) bool {
|
||||
@ -469,6 +475,8 @@ func DecodeIndex(buf []byte) (idx *Index, err error) {
|
||||
|
||||
idx = NewIndex()
|
||||
for _, pack := range idxJSON.Packs {
|
||||
var data, tree bool
|
||||
|
||||
for _, blob := range pack.Blobs {
|
||||
idx.store(restic.PackedBlob{
|
||||
Blob: restic.Blob{
|
||||
@ -479,6 +487,17 @@ func DecodeIndex(buf []byte) (idx *Index, err error) {
|
||||
},
|
||||
PackID: pack.ID,
|
||||
})
|
||||
|
||||
switch blob.Type {
|
||||
case restic.DataBlob:
|
||||
data = true
|
||||
case restic.TreeBlob:
|
||||
tree = true
|
||||
}
|
||||
}
|
||||
|
||||
if !data && tree {
|
||||
idx.treePacks = append(idx.treePacks, pack.ID)
|
||||
}
|
||||
}
|
||||
idx.supersedes = idxJSON.Supersedes
|
||||
@ -501,6 +520,8 @@ func DecodeOldIndex(buf []byte) (idx *Index, err error) {
|
||||
|
||||
idx = NewIndex()
|
||||
for _, pack := range list {
|
||||
var data, tree bool
|
||||
|
||||
for _, blob := range pack.Blobs {
|
||||
idx.store(restic.PackedBlob{
|
||||
Blob: restic.Blob{
|
||||
@ -511,6 +532,17 @@ func DecodeOldIndex(buf []byte) (idx *Index, err error) {
|
||||
},
|
||||
PackID: pack.ID,
|
||||
})
|
||||
|
||||
switch blob.Type {
|
||||
case restic.DataBlob:
|
||||
data = true
|
||||
case restic.TreeBlob:
|
||||
tree = true
|
||||
}
|
||||
}
|
||||
|
||||
if !data && tree {
|
||||
idx.treePacks = append(idx.treePacks, pack.ID)
|
||||
}
|
||||
}
|
||||
idx.final = true
|
||||
|
@ -89,8 +89,8 @@ func (r *packerManager) insertPacker(p *Packer) {
|
||||
}
|
||||
|
||||
// savePacker stores p in the backend.
|
||||
func (r *Repository) savePacker(p *Packer) error {
|
||||
debug.Log("save packer with %d blobs (%d bytes)\n", p.Packer.Count(), p.Packer.Size())
|
||||
func (r *Repository) savePacker(t restic.BlobType, p *Packer) error {
|
||||
debug.Log("save packer for %v with %d blobs (%d bytes)\n", t, p.Packer.Count(), p.Packer.Size())
|
||||
_, err := p.Packer.Finalize()
|
||||
if err != nil {
|
||||
return err
|
||||
@ -112,6 +112,20 @@ func (r *Repository) savePacker(p *Packer) error {
|
||||
|
||||
debug.Log("saved as %v", h)
|
||||
|
||||
if t == restic.TreeBlob && r.Cache != nil {
|
||||
debug.Log("saving tree pack file in cache")
|
||||
|
||||
_, err = p.tmpfile.Seek(0, 0)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Seek")
|
||||
}
|
||||
|
||||
err := r.Cache.Save(h, p.tmpfile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = p.tmpfile.Close()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "close tempfile")
|
||||
|
@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/restic/restic/internal/cache"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
|
||||
@ -23,6 +24,7 @@ type Repository struct {
|
||||
key *crypto.Key
|
||||
keyName string
|
||||
idx *MasterIndex
|
||||
restic.Cache
|
||||
|
||||
treePM *packerManager
|
||||
dataPM *packerManager
|
||||
@ -45,6 +47,16 @@ func (r *Repository) Config() restic.Config {
|
||||
return r.cfg
|
||||
}
|
||||
|
||||
// UseCache replaces the backend with the wrapped cache.
|
||||
func (r *Repository) UseCache(c restic.Cache) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
debug.Log("using cache")
|
||||
r.Cache = c
|
||||
r.be = c.Wrap(r.be)
|
||||
}
|
||||
|
||||
// PrefixLength returns the number of bytes required so that all prefixes of
|
||||
// all IDs of type t are unique.
|
||||
func (r *Repository) PrefixLength(t restic.FileType) (int, error) {
|
||||
@ -53,11 +65,11 @@ func (r *Repository) PrefixLength(t restic.FileType) (int, error) {
|
||||
|
||||
// LoadAndDecrypt loads and decrypts data identified by t and id from the
|
||||
// backend.
|
||||
func (r *Repository) LoadAndDecrypt(ctx context.Context, t restic.FileType, id restic.ID) ([]byte, error) {
|
||||
func (r *Repository) LoadAndDecrypt(ctx context.Context, t restic.FileType, id restic.ID) (buf []byte, err error) {
|
||||
debug.Log("load %v with id %v", t, id.Str())
|
||||
|
||||
h := restic.Handle{Type: t, Name: id.String()}
|
||||
buf, err := backend.LoadAll(ctx, r.be, h)
|
||||
buf, err = backend.LoadAll(ctx, r.be, h)
|
||||
if err != nil {
|
||||
debug.Log("error loading %v: %v", h, err)
|
||||
return nil, err
|
||||
@ -76,6 +88,26 @@ func (r *Repository) LoadAndDecrypt(ctx context.Context, t restic.FileType, id r
|
||||
return buf[:n], nil
|
||||
}
|
||||
|
||||
// sortCachedPacks moves all cached pack files to the front of blobs.
|
||||
func (r *Repository) sortCachedPacks(blobs []restic.PackedBlob) []restic.PackedBlob {
|
||||
if r.Cache == nil {
|
||||
return blobs
|
||||
}
|
||||
|
||||
cached := make([]restic.PackedBlob, 0, len(blobs)/2)
|
||||
noncached := make([]restic.PackedBlob, 0, len(blobs)/2)
|
||||
|
||||
for _, blob := range blobs {
|
||||
if r.Cache.Has(restic.Handle{Type: restic.DataFile, Name: blob.PackID.String()}) {
|
||||
cached = append(cached, blob)
|
||||
continue
|
||||
}
|
||||
noncached = append(noncached, blob)
|
||||
}
|
||||
|
||||
return append(cached, noncached...)
|
||||
}
|
||||
|
||||
// loadBlob tries to load and decrypt content identified by t and id from a
|
||||
// pack from the backend, the result is stored in plaintextBuf, which must be
|
||||
// large enough to hold the complete blob.
|
||||
@ -89,9 +121,12 @@ func (r *Repository) loadBlob(ctx context.Context, id restic.ID, t restic.BlobTy
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// try cached pack files first
|
||||
blobs = r.sortCachedPacks(blobs)
|
||||
|
||||
var lastError error
|
||||
for _, blob := range blobs {
|
||||
debug.Log("id %v found: %v", id.Str(), blob)
|
||||
debug.Log("blob %v/%v found: %v", t, id.Str(), blob)
|
||||
|
||||
if blob.Type != t {
|
||||
debug.Log("blob %v has wrong block type, want %v", blob, t)
|
||||
@ -212,7 +247,7 @@ func (r *Repository) SaveAndEncrypt(ctx context.Context, t restic.BlobType, data
|
||||
}
|
||||
|
||||
// else write the pack to the backend
|
||||
return *id, r.savePacker(packer)
|
||||
return *id, r.savePacker(t, packer)
|
||||
}
|
||||
|
||||
// SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the
|
||||
@ -251,20 +286,27 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by
|
||||
|
||||
// Flush saves all remaining packs.
|
||||
func (r *Repository) Flush() error {
|
||||
for _, pm := range []*packerManager{r.dataPM, r.treePM} {
|
||||
pm.pm.Lock()
|
||||
pms := []struct {
|
||||
t restic.BlobType
|
||||
pm *packerManager
|
||||
}{
|
||||
{restic.DataBlob, r.dataPM},
|
||||
{restic.TreeBlob, r.treePM},
|
||||
}
|
||||
|
||||
debug.Log("manually flushing %d packs", len(pm.packers))
|
||||
for _, p := range pm.packers {
|
||||
err := r.savePacker(p)
|
||||
for _, p := range pms {
|
||||
p.pm.pm.Lock()
|
||||
|
||||
debug.Log("manually flushing %d packs", len(p.pm.packers))
|
||||
for _, packer := range p.pm.packers {
|
||||
err := r.savePacker(p.t, packer)
|
||||
if err != nil {
|
||||
pm.pm.Unlock()
|
||||
p.pm.pm.Unlock()
|
||||
return err
|
||||
}
|
||||
}
|
||||
pm.packers = pm.packers[:0]
|
||||
|
||||
pm.pm.Unlock()
|
||||
p.pm.packers = p.pm.packers[:0]
|
||||
p.pm.pm.Unlock()
|
||||
}
|
||||
|
||||
return nil
|
||||
@ -353,10 +395,58 @@ func (r *Repository) LoadIndex(ctx context.Context) error {
|
||||
ParallelWorkFuncParseID(worker))
|
||||
}()
|
||||
|
||||
validIndex := restic.NewIDSet()
|
||||
for idx := range indexes {
|
||||
id, err := idx.ID()
|
||||
if err == nil {
|
||||
validIndex.Insert(id)
|
||||
}
|
||||
r.idx.Insert(idx)
|
||||
}
|
||||
|
||||
if r.Cache != nil {
|
||||
// clear old index files
|
||||
err := r.Cache.Clear(restic.IndexFile, validIndex)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error clearing index files in cache: %v\n", err)
|
||||
}
|
||||
|
||||
packs := restic.NewIDSet()
|
||||
for _, idx := range r.idx.All() {
|
||||
for id := range idx.Packs() {
|
||||
packs.Insert(id)
|
||||
}
|
||||
}
|
||||
|
||||
// clear old data files
|
||||
err = r.Cache.Clear(restic.DataFile, packs)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error clearing data files in cache: %v\n", err)
|
||||
}
|
||||
|
||||
treePacks := restic.NewIDSet()
|
||||
for _, idx := range r.idx.All() {
|
||||
for _, id := range idx.TreePacks() {
|
||||
treePacks.Insert(id)
|
||||
}
|
||||
}
|
||||
|
||||
// use readahead
|
||||
cache := r.Cache.(*cache.Cache)
|
||||
cache.PerformReadahead = func(h restic.Handle) bool {
|
||||
if h.Type != restic.DataFile {
|
||||
return false
|
||||
}
|
||||
|
||||
id, err := restic.ParseID(h.Name)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return treePacks.Has(id)
|
||||
}
|
||||
}
|
||||
|
||||
if err := <-errCh; err != nil {
|
||||
return err
|
||||
}
|
||||
|
37
internal/restic/cache.go
Normal file
37
internal/restic/cache.go
Normal file
@ -0,0 +1,37 @@
|
||||
package restic
|
||||
|
||||
import "io"
|
||||
|
||||
// Cache manages a local cache.
|
||||
type Cache interface {
|
||||
// BaseDir returns the base directory of the cache.
|
||||
BaseDir() string
|
||||
|
||||
// Wrap returns a backend with a cache.
|
||||
Wrap(Backend) Backend
|
||||
|
||||
// IsNotExist returns true if the error was caused by a non-existing file.
|
||||
IsNotExist(err error) bool
|
||||
|
||||
// Load returns a reader that yields the contents of the file with the
|
||||
// given id if it is cached. rd must be closed after use. If an error is
|
||||
// returned, the ReadCloser is nil. The files are still encrypted
|
||||
Load(h Handle, length int, offset int64) (io.ReadCloser, error)
|
||||
|
||||
// SaveIndex saves an index in the cache.
|
||||
Save(Handle, io.Reader) error
|
||||
|
||||
// SaveWriter returns a writer for the to be cached object h. It must be
|
||||
// closed after writing is finished.
|
||||
SaveWriter(Handle) (io.WriteCloser, error)
|
||||
|
||||
// Remove deletes a single file from the cache. If it isn't cached, this
|
||||
// functions must return no error.
|
||||
Remove(Handle) error
|
||||
|
||||
// Clear removes all files of type t from the cache that are not contained in the set.
|
||||
Clear(FileType, IDSet) error
|
||||
|
||||
// Has returns true if the file is cached.
|
||||
Has(Handle) bool
|
||||
}
|
@ -5,6 +5,7 @@ import (
|
||||
"io"
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
)
|
||||
|
||||
type backendReaderAt struct {
|
||||
@ -37,5 +38,5 @@ func ReadAt(ctx context.Context, be Backend, h Handle, offset int64, p []byte) (
|
||||
|
||||
debug.Log("ReadAt(%v) ReadFull returned %v bytes", h, n)
|
||||
|
||||
return n, err
|
||||
return n, errors.Wrapf(err, "ReadFull(%v)", h)
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user