From dbda89254204e53efe3d7ce4abf283d5dcca56d0 Mon Sep 17 00:00:00 2001 From: Fabian Wickborn Date: Sat, 19 Aug 2017 22:44:18 +0200 Subject: [PATCH] Add option to exclude directories with a tagfile The option is named --exclude-if-present and accepts a parameter filename[:content]. Directories are excluded and their contents is not backed up if they contain a file with the specified name and, optionally, that starts with the specified content. The tagfile itself is never excluded. There is also a shortcut --exclude-caches that works in the same way as the likewise-named option of tar(1): Directories are recognized as cache if they contain a file named "CACHEDIR.TAG. Closes #317. --- CHANGELOG.md | 8 +++ cmd/restic/cmd_backup.go | 125 ++++++++++++++++++++++++++++++++++--- cmd/restic/exclude_test.go | 58 +++++++++++++++++ doc/man/restic-backup.1 | 8 +++ 4 files changed, 189 insertions(+), 10 deletions(-) create mode 100644 cmd/restic/exclude_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 3865f4b98..f13ae3d58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,14 @@ Small changes run. This is now corrected. https://github.com/restic/restic/pull/1191 + * A new option `--exclude-caches` was added that allows excluding cache + directories (that are tagged as such). This is a special case of a more + generic option `--exclude-if-present` which excludes a directory if a file + with a specific name (and contents) is present. + https://github.com/restic/restic/issues/317 + https://github.com/restic/restic/pull/1170 + + Important Changes in 0.7.1 ========================== diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 292220cda..09a4ba2f5 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "bytes" "context" "fmt" "io" @@ -53,16 +54,18 @@ given as the arguments. // BackupOptions bundles all options for the backup command. type BackupOptions struct { - Parent string - Force bool - Excludes []string - ExcludeFiles []string - ExcludeOtherFS bool - Stdin bool - StdinFilename string - Tags []string - Hostname string - FilesFrom string + Parent string + Force bool + Excludes []string + ExcludeFiles []string + ExcludeOtherFS bool + ExcludeIfPresent string + ExcludeCaches bool + Stdin bool + StdinFilename string + Tags []string + Hostname string + FilesFrom string } var backupOptions BackupOptions @@ -76,6 +79,8 @@ func init() { f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)") f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems") + f.StringVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", "", "takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided") + f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file`) f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin") f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin") f.StringArrayVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)") @@ -416,6 +421,18 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error { opts.Excludes = append(opts.Excludes, readExcludePatternsFromFiles(opts.ExcludeFiles)...) } + if opts.ExcludeCaches { + if opts.ExcludeIfPresent != "" { + return fmt.Errorf("cannot have --exclude-caches defined at the same time as --exclude-if-present") + } + opts.ExcludeIfPresent = "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55" + } + + excludeByFile, err := excludeByFile(opts.ExcludeIfPresent) + if err != nil { + return err + } + selectFilter := func(item string, fi os.FileInfo) bool { matched, _, err := filter.List(opts.Excludes, item) if err != nil { @@ -427,6 +444,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error { return false } + if excludeByFile(item) { + debug.Log("path %q excluded by tagfile", item) + return false + } + if !opts.ExcludeOtherFS || fi == nil { return true } @@ -522,3 +544,86 @@ func readExcludePatternsFromFiles(excludeFiles []string) []string { } return excludes } + +// FilenameCheck is a function that takes a filename and returns a boolean +// depending on arbitrary check. +type FilenameCheck func(filename string) bool + +// excludeByFile returns a FilenameCheck which itself returns whether a path +// should be excluded. The FilenameCheck considers a file to be excluded when +// it resides in a directory with an exclusion file, that is specified by +// excludeFileSpec in the form "filename[:content]". The returned error is +// non-nil if the filename component of excludeFileSpec is empty. +func excludeByFile(excludeFileSpec string) (FilenameCheck, error) { + if excludeFileSpec == "" { + return func(string) bool { return false }, nil + } + colon := strings.Index(excludeFileSpec, ":") + if colon == 0 { + return nil, fmt.Errorf("no name for exclusion tagfile provided") + } + tf, tc := "", "" + if colon > 0 { + tf = excludeFileSpec[:colon] + tc = excludeFileSpec[colon+1:] + } else { + tf = excludeFileSpec + } + debug.Log("using %q as exclusion tagfile", tf) + fn := func(filename string) bool { + return isExcludedByFile(filename, tf, tc) + } + return fn, nil +} + +// isExcludedByFile interprets filename as a path and returns true if that file +// is in a excluded directory. A directory is identified as excluded if it contains a +// tagfile which bears the name specified in tagFilename and starts with header. +func isExcludedByFile(filename, tagFilename, header string) bool { + if tagFilename == "" { + return false + } + dir, base := filepath.Split(filename) + if base == tagFilename { + return false // do not exclude the tagfile itself + } + tf := filepath.Join(dir, tagFilename) + _, err := fs.Lstat(tf) + if os.IsNotExist(err) { + return false + } + if err != nil { + Warnf("could not access exclusion tagfile: %v", err) + return false + } + // when no signature is given, the mere presence of tf is enough reason + // to exclude filename + if len(header) == 0 { + return true + } + // From this stage, errors mean tagFilename exists but it is malformed. + // Warnings will be generated so that the user is informed that the + // indented ignore-action is not performed. + f, err := os.Open(tf) + if err != nil { + Warnf("could not open exclusion tagfile: %v", err) + return false + } + defer f.Close() + buf := make([]byte, len(header)) + _, err = io.ReadFull(f, buf) + // EOF is handled with a dedicated message, otherwise the warning were too cryptic + if err == io.EOF { + Warnf("invalid (too short) signature in exclusion tagfile %q\n", tf) + return false + } + if err != nil { + Warnf("could not read signature from exclusion tagfile %q: %v\n", tf, err) + return false + } + if bytes.Compare(buf, []byte(header)) != 0 { + Warnf("invalid signature in exclusion tagfile %q\n", tf) + return false + } + return true +} diff --git a/cmd/restic/exclude_test.go b/cmd/restic/exclude_test.go new file mode 100644 index 000000000..cdbd4ef55 --- /dev/null +++ b/cmd/restic/exclude_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func TestIsExcludedByFile(t *testing.T) { + const ( + tagFilename = "CACHEDIR.TAG" + header = "Signature: 8a477f597d28d172789f06886806bc55" + ) + tests := []struct { + name string + tagFile string + content string + want bool + }{ + {"NoTagfile", "", "", false}, + {"EmptyTagfile", tagFilename, "", true}, + {"UnnamedTagFile", "", header, false}, + {"WrongTagFile", "notatagfile", header, false}, + {"IncorrectSig", tagFilename, header[1:], false}, + {"ValidSig", tagFilename, header, true}, + {"ValidPlusStuff", tagFilename, header + "foo", true}, + {"ValidPlusNewlineAndStuff", tagFilename, header + "\nbar", true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tempDir, err := ioutil.TempDir("", "restic-test-") + if err != nil { + t.Fatalf("could not create temp dir: %v", err) + } + defer os.RemoveAll(tempDir) + foo := filepath.Join(tempDir, "foo") + err = ioutil.WriteFile(foo, []byte("foo"), 0666) + if err != nil { + t.Fatalf("could not write file: %v", err) + } + if tc.tagFile != "" { + tagFile := filepath.Join(tempDir, tc.tagFile) + err = ioutil.WriteFile(tagFile, []byte(tc.content), 0666) + if err != nil { + t.Fatalf("could not write tagfile: %v", err) + } + } + h := header + if tc.content == "" { + h = "" + } + if got := isExcludedByFile(foo, tagFilename, h); tc.want != got { + t.Fatalf("expected %v, got %v", tc.want, got) + } + }) + } +} diff --git a/doc/man/restic-backup.1 b/doc/man/restic-backup.1 index d791688d8..93e76ca19 100644 --- a/doc/man/restic-backup.1 +++ b/doc/man/restic-backup.1 @@ -24,10 +24,18 @@ given as the arguments. \fB\-e\fP, \fB\-\-exclude\fP=[] exclude a \fB\fCpattern\fR (can be specified multiple times) +.PP +\fB\-\-exclude\-caches\fP[=false] + excludes cache directories that are marked with a CACHEDIR.TAG file + .PP \fB\-\-exclude\-file\fP=[] read exclude patterns from a \fB\fCfile\fR (can be specified multiple times) +.PP +\fB\-\-exclude\-if\-present\fP="" + takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided + .PP \fB\-\-files\-from\fP="" read the files to backup from file (can be combined with file args)