diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 562108a33..107e8bbe0 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -20,10 +20,12 @@ import ( "github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/textfile" + "github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui/backup" "github.com/restic/restic/internal/ui/termstatus" ) @@ -66,7 +68,7 @@ Exit status is 12 if the password is incorrect. // BackupOptions bundles all options for the backup command. type BackupOptions struct { - excludePatternOptions + filter.ExcludePatternOptions Parent string GroupBy restic.SnapshotGroupByOptions @@ -108,7 +110,7 @@ func init() { f.VarP(&backupOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')") f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`) - initExcludePatternOptions(f, &backupOptions.excludePatternOptions) + backupOptions.ExcludePatternOptions.Add(f) f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes") f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)") @@ -297,7 +299,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error { // collectRejectByNameFuncs returns a list of all functions which may reject data // from being saved in a snapshot based on path only -func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []RejectByNameFunc, err error) { +func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []archiver.RejectByNameFunc, err error) { // exclude restic cache if repo.Cache != nil { f, err := rejectResticCache(repo) @@ -308,23 +310,12 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) ( fs = append(fs, f) } - fsPatterns, err := opts.excludePatternOptions.CollectPatterns() + fsPatterns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf) if err != nil { return nil, err } - fs = append(fs, fsPatterns...) - - if opts.ExcludeCaches { - opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55") - } - - for _, spec := range opts.ExcludeIfPresent { - f, err := rejectIfPresent(spec) - if err != nil { - return nil, err - } - - fs = append(fs, f) + for _, pat := range fsPatterns { + fs = append(fs, archiver.RejectByNameFunc(pat)) } return fs, nil @@ -332,25 +323,43 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) ( // collectRejectFuncs returns a list of all functions which may reject data // from being saved in a snapshot based on path and file info -func collectRejectFuncs(opts BackupOptions, targets []string) (fs []RejectFunc, err error) { +func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS) (funcs []archiver.RejectFunc, err error) { // allowed devices - if opts.ExcludeOtherFS && !opts.Stdin { - f, err := rejectByDevice(targets) + if opts.ExcludeOtherFS && !opts.Stdin && !opts.StdinCommand { + f, err := archiver.RejectByDevice(targets, fs) if err != nil { return nil, err } - fs = append(fs, f) + funcs = append(funcs, f) } - if len(opts.ExcludeLargerThan) != 0 && !opts.Stdin { - f, err := rejectBySize(opts.ExcludeLargerThan) + if len(opts.ExcludeLargerThan) != 0 && !opts.Stdin && !opts.StdinCommand { + maxSize, err := ui.ParseBytes(opts.ExcludeLargerThan) if err != nil { return nil, err } - fs = append(fs, f) + + f, err := archiver.RejectBySize(maxSize) + if err != nil { + return nil, err + } + funcs = append(funcs, f) } - return fs, nil + if opts.ExcludeCaches { + opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55") + } + + for _, spec := range opts.ExcludeIfPresent { + f, err := archiver.RejectIfPresent(spec, Warnf) + if err != nil { + return nil, err + } + + funcs = append(funcs, f) + } + + return funcs, nil } // collectTargets returns a list of target files/dirs from several sources. @@ -505,12 +514,6 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter return err } - // rejectFuncs collect functions that can reject items from the backup based on path and file info - rejectFuncs, err := collectRejectFuncs(opts, targets) - if err != nil { - return err - } - var parentSnapshot *restic.Snapshot if !opts.Stdin { parentSnapshot, err = findParentSnapshot(ctx, repo, opts, targets, timeStamp) @@ -532,30 +535,11 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter } bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term) - err = repo.LoadIndex(ctx, bar) if err != nil { return err } - selectByNameFilter := func(item string) bool { - for _, reject := range rejectByNameFuncs { - if reject(item) { - return false - } - } - return true - } - - selectFilter := func(item string, fi os.FileInfo) bool { - for _, reject := range rejectFuncs { - if reject(item, fi) { - return false - } - } - return true - } - var targetFS fs.FS = fs.Local{} if runtime.GOOS == "windows" && opts.UseFsSnapshot { if err = fs.HasSufficientPrivilegesForVSS(); err != nil { @@ -598,6 +582,15 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter targets = []string{filename} } + // rejectFuncs collect functions that can reject items from the backup based on path and file info + rejectFuncs, err := collectRejectFuncs(opts, targets, targetFS) + if err != nil { + return err + } + + selectByNameFilter := archiver.CombineRejectByNames(rejectByNameFuncs) + selectFilter := archiver.CombineRejects(rejectFuncs) + wg, wgCtx := errgroup.WithContext(ctx) cancelCtx, cancel := context.WithCancel(wgCtx) defer cancel() diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index c58b0b80d..82dd408a8 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -7,6 +7,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restorer" "github.com/restic/restic/internal/ui" @@ -49,8 +50,8 @@ Exit status is 12 if the password is incorrect. // RestoreOptions collects all options for the restore command. type RestoreOptions struct { - excludePatternOptions - includePatternOptions + filter.ExcludePatternOptions + filter.IncludePatternOptions Target string restic.SnapshotFilter DryRun bool @@ -68,8 +69,8 @@ func init() { flags := cmdRestore.Flags() flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to") - initExcludePatternOptions(flags, &restoreOptions.excludePatternOptions) - initIncludePatternOptions(flags, &restoreOptions.includePatternOptions) + restoreOptions.ExcludePatternOptions.Add(flags) + restoreOptions.IncludePatternOptions.Add(flags) initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter) flags.BoolVar(&restoreOptions.DryRun, "dry-run", false, "do not write any data, just show what would be done") @@ -82,12 +83,12 @@ func init() { func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error { - excludePatternFns, err := opts.excludePatternOptions.CollectPatterns() + excludePatternFns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf) if err != nil { return err } - includePatternFns, err := opts.includePatternOptions.CollectPatterns() + includePatternFns, err := opts.IncludePatternOptions.CollectPatterns(Warnf) if err != nil { return err } diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 7788016b7..a9f664110 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -10,6 +10,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/walker" @@ -88,7 +89,7 @@ type RewriteOptions struct { Metadata snapshotMetadataArgs restic.SnapshotFilter - excludePatternOptions + filter.ExcludePatternOptions } var rewriteOptions RewriteOptions @@ -103,7 +104,7 @@ func init() { f.StringVar(&rewriteOptions.Metadata.Time, "new-time", "", "replace time of the backup") initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true) - initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions) + rewriteOptions.ExcludePatternOptions.Add(f) } type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) @@ -113,7 +114,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str()) } - rejectByNameFuncs, err := opts.excludePatternOptions.CollectPatterns() + rejectByNameFuncs, err := opts.ExcludePatternOptions.CollectPatterns(Warnf) if err != nil { return false, err } @@ -263,7 +264,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r } func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error { - if opts.excludePatternOptions.Empty() && opts.Metadata.empty() { + if opts.ExcludePatternOptions.Empty() && opts.Metadata.empty() { return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided") } diff --git a/cmd/restic/cmd_rewrite_integration_test.go b/cmd/restic/cmd_rewrite_integration_test.go index 781266184..6471d49ba 100644 --- a/cmd/restic/cmd_rewrite_integration_test.go +++ b/cmd/restic/cmd_rewrite_integration_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "testing" + "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" "github.com/restic/restic/internal/ui" @@ -12,7 +13,7 @@ import ( func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool, metadata snapshotMetadataArgs) { opts := RewriteOptions{ - excludePatternOptions: excludePatternOptions{ + ExcludePatternOptions: filter.ExcludePatternOptions{ Excludes: excludes, }, Forget: forget, diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index 4657e4915..99d1128a9 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -1,347 +1,16 @@ package main import ( - "bufio" - "bytes" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "sync" - + "github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/textfile" - "github.com/restic/restic/internal/ui" - "github.com/spf13/pflag" ) -type rejectionCache struct { - m map[string]bool - mtx sync.Mutex -} - -// Lock locks the mutex in rc. -func (rc *rejectionCache) Lock() { - if rc != nil { - rc.mtx.Lock() - } -} - -// Unlock unlocks the mutex in rc. -func (rc *rejectionCache) Unlock() { - if rc != nil { - rc.mtx.Unlock() - } -} - -// Get returns the last stored value for dir and a second boolean that -// indicates whether that value was actually written to the cache. It is the -// callers responsibility to call rc.Lock and rc.Unlock before using this -// method, otherwise data races may occur. -func (rc *rejectionCache) Get(dir string) (bool, bool) { - if rc == nil || rc.m == nil { - return false, false - } - v, ok := rc.m[dir] - return v, ok -} - -// Store stores a new value for dir. It is the callers responsibility to call -// rc.Lock and rc.Unlock before using this method, otherwise data races may -// occur. -func (rc *rejectionCache) Store(dir string, rejected bool) { - if rc == nil { - return - } - if rc.m == nil { - rc.m = make(map[string]bool) - } - rc.m[dir] = rejected -} - -// RejectByNameFunc is a function that takes a filename of a -// file that would be included in the backup. The function returns true if it -// should be excluded (rejected) from the backup. -type RejectByNameFunc func(path string) bool - -// RejectFunc is a function that takes a filename and os.FileInfo of a -// file that would be included in the backup. The function returns true if it -// should be excluded (rejected) from the backup. -type RejectFunc func(path string, fi os.FileInfo) bool - -// rejectByPattern returns a RejectByNameFunc which rejects files that match -// one of the patterns. -func rejectByPattern(patterns []string) RejectByNameFunc { - parsedPatterns := filter.ParsePatterns(patterns) - return func(item string) bool { - matched, err := filter.List(parsedPatterns, item) - if err != nil { - Warnf("error for exclude pattern: %v", err) - } - - if matched { - debug.Log("path %q excluded by an exclude pattern", item) - return true - } - - return false - } -} - -// Same as `rejectByPattern` but case insensitive. -func rejectByInsensitivePattern(patterns []string) RejectByNameFunc { - for index, path := range patterns { - patterns[index] = strings.ToLower(path) - } - - rejFunc := rejectByPattern(patterns) - return func(item string) bool { - return rejFunc(strings.ToLower(item)) - } -} - -// rejectIfPresent returns a RejectByNameFunc which itself returns whether a path -// should be excluded. The RejectByNameFunc considers a file to be excluded when -// it resides in a directory with an exclusion file, that is specified by -// excludeFileSpec in the form "filename[:content]". The returned error is -// non-nil if the filename component of excludeFileSpec is empty. If rc is -// non-nil, it is going to be used in the RejectByNameFunc to expedite the evaluation -// of a directory based on previous visits. -func rejectIfPresent(excludeFileSpec string) (RejectByNameFunc, error) { - if excludeFileSpec == "" { - return nil, errors.New("name for exclusion tagfile is empty") - } - colon := strings.Index(excludeFileSpec, ":") - if colon == 0 { - return nil, fmt.Errorf("no name for exclusion tagfile provided") - } - tf, tc := "", "" - if colon > 0 { - tf = excludeFileSpec[:colon] - tc = excludeFileSpec[colon+1:] - } else { - tf = excludeFileSpec - } - debug.Log("using %q as exclusion tagfile", tf) - rc := &rejectionCache{} - fn := func(filename string) bool { - return isExcludedByFile(filename, tf, tc, rc) - } - return fn, nil -} - -// isExcludedByFile interprets filename as a path and returns true if that file -// is in an excluded directory. A directory is identified as excluded if it contains a -// tagfile which bears the name specified in tagFilename and starts with -// header. If rc is non-nil, it is used to expedite the evaluation of a -// directory based on previous visits. -func isExcludedByFile(filename, tagFilename, header string, rc *rejectionCache) bool { - if tagFilename == "" { - return false - } - dir, base := filepath.Split(filename) - if base == tagFilename { - return false // do not exclude the tagfile itself - } - rc.Lock() - defer rc.Unlock() - - rejected, visited := rc.Get(dir) - if visited { - return rejected - } - rejected = isDirExcludedByFile(dir, tagFilename, header) - rc.Store(dir, rejected) - return rejected -} - -func isDirExcludedByFile(dir, tagFilename, header string) bool { - tf := filepath.Join(dir, tagFilename) - _, err := fs.Lstat(tf) - if os.IsNotExist(err) { - return false - } - if err != nil { - Warnf("could not access exclusion tagfile: %v", err) - return false - } - // when no signature is given, the mere presence of tf is enough reason - // to exclude filename - if len(header) == 0 { - return true - } - // From this stage, errors mean tagFilename exists but it is malformed. - // Warnings will be generated so that the user is informed that the - // indented ignore-action is not performed. - f, err := os.Open(tf) - if err != nil { - Warnf("could not open exclusion tagfile: %v", err) - return false - } - defer func() { - _ = f.Close() - }() - buf := make([]byte, len(header)) - _, err = io.ReadFull(f, buf) - // EOF is handled with a dedicated message, otherwise the warning were too cryptic - if err == io.EOF { - Warnf("invalid (too short) signature in exclusion tagfile %q\n", tf) - return false - } - if err != nil { - Warnf("could not read signature from exclusion tagfile %q: %v\n", tf, err) - return false - } - if !bytes.Equal(buf, []byte(header)) { - Warnf("invalid signature in exclusion tagfile %q\n", tf) - return false - } - return true -} - -// DeviceMap is used to track allowed source devices for backup. This is used to -// check for crossing mount points during backup (for --one-file-system). It -// maps the name of a source path to its device ID. -type DeviceMap map[string]uint64 - -// NewDeviceMap creates a new device map from the list of source paths. -func NewDeviceMap(allowedSourcePaths []string) (DeviceMap, error) { - deviceMap := make(map[string]uint64) - - for _, item := range allowedSourcePaths { - item, err := filepath.Abs(filepath.Clean(item)) - if err != nil { - return nil, err - } - - fi, err := fs.Lstat(item) - if err != nil { - return nil, err - } - - id, err := fs.DeviceID(fi) - if err != nil { - return nil, err - } - - deviceMap[item] = id - } - - if len(deviceMap) == 0 { - return nil, errors.New("zero allowed devices") - } - - return deviceMap, nil -} - -// IsAllowed returns true if the path is located on an allowed device. -func (m DeviceMap) IsAllowed(item string, deviceID uint64) (bool, error) { - for dir := item; ; dir = filepath.Dir(dir) { - debug.Log("item %v, test dir %v", item, dir) - - // find a parent directory that is on an allowed device (otherwise - // we would not traverse the directory at all) - allowedID, ok := m[dir] - if !ok { - if dir == filepath.Dir(dir) { - // arrived at root, no allowed device found. this should not happen. - break - } - continue - } - - // if the item has a different device ID than the parent directory, - // we crossed a file system boundary - if allowedID != deviceID { - debug.Log("item %v (dir %v) on disallowed device %d", item, dir, deviceID) - return false, nil - } - - // item is on allowed device, accept it - debug.Log("item %v allowed", item) - return true, nil - } - - return false, fmt.Errorf("item %v (device ID %v) not found, deviceMap: %v", item, deviceID, m) -} - -// rejectByDevice returns a RejectFunc that rejects files which are on a -// different file systems than the files/dirs in samples. -func rejectByDevice(samples []string) (RejectFunc, error) { - deviceMap, err := NewDeviceMap(samples) - if err != nil { - return nil, err - } - debug.Log("allowed devices: %v\n", deviceMap) - - return func(item string, fi os.FileInfo) bool { - id, err := fs.DeviceID(fi) - if err != nil { - // This should never happen because gatherDevices() would have - // errored out earlier. If it still does that's a reason to panic. - panic(err) - } - - allowed, err := deviceMap.IsAllowed(filepath.Clean(item), id) - if err != nil { - // this should not happen - panic(fmt.Sprintf("error checking device ID of %v: %v", item, err)) - } - - if allowed { - // accept item - return false - } - - // reject everything except directories - if !fi.IsDir() { - return true - } - - // special case: make sure we keep mountpoints (directories which - // contain a mounted file system). Test this by checking if the parent - // directory would be included. - parentDir := filepath.Dir(filepath.Clean(item)) - - parentFI, err := fs.Lstat(parentDir) - if err != nil { - debug.Log("item %v: error running lstat() on parent directory: %v", item, err) - // if in doubt, reject - return true - } - - parentDeviceID, err := fs.DeviceID(parentFI) - if err != nil { - debug.Log("item %v: getting device ID of parent directory: %v", item, err) - // if in doubt, reject - return true - } - - parentAllowed, err := deviceMap.IsAllowed(parentDir, parentDeviceID) - if err != nil { - debug.Log("item %v: error checking parent directory: %v", item, err) - // if in doubt, reject - return true - } - - if parentAllowed { - // we found a mount point, so accept the directory - return false - } - - // reject everything else - return true - }, nil -} - // rejectResticCache returns a RejectByNameFunc that rejects the restic cache // directory (if set). -func rejectResticCache(repo *repository.Repository) (RejectByNameFunc, error) { +func rejectResticCache(repo *repository.Repository) (archiver.RejectByNameFunc, error) { if repo.Cache == nil { return func(string) bool { return false @@ -362,137 +31,3 @@ func rejectResticCache(repo *repository.Repository) (RejectByNameFunc, error) { return false }, nil } - -func rejectBySize(maxSizeStr string) (RejectFunc, error) { - maxSize, err := ui.ParseBytes(maxSizeStr) - if err != nil { - return nil, err - } - - return func(item string, fi os.FileInfo) bool { - // directory will be ignored - if fi.IsDir() { - return false - } - - filesize := fi.Size() - if filesize > maxSize { - debug.Log("file %s is oversize: %d", item, filesize) - return true - } - - return false - }, nil -} - -// readPatternsFromFiles reads all files and returns the list of -// patterns. For each line, leading and trailing white space is removed -// and comment lines are ignored. For each remaining pattern, environment -// variables are resolved. For adding a literal dollar sign ($), write $$ to -// the file. -func readPatternsFromFiles(files []string) ([]string, error) { - getenvOrDollar := func(s string) string { - if s == "$" { - return "$" - } - return os.Getenv(s) - } - - var patterns []string - for _, filename := range files { - err := func() (err error) { - data, err := textfile.Read(filename) - if err != nil { - return err - } - - scanner := bufio.NewScanner(bytes.NewReader(data)) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - - // ignore empty lines - if line == "" { - continue - } - - // strip comments - if strings.HasPrefix(line, "#") { - continue - } - - line = os.Expand(line, getenvOrDollar) - patterns = append(patterns, line) - } - return scanner.Err() - }() - if err != nil { - return nil, fmt.Errorf("failed to read patterns from file %q: %w", filename, err) - } - } - return patterns, nil -} - -type excludePatternOptions struct { - Excludes []string - InsensitiveExcludes []string - ExcludeFiles []string - InsensitiveExcludeFiles []string -} - -func initExcludePatternOptions(f *pflag.FlagSet, opts *excludePatternOptions) { - f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") - f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames") - f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)") - f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns") -} - -func (opts *excludePatternOptions) Empty() bool { - return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0 -} - -func (opts excludePatternOptions) CollectPatterns() ([]RejectByNameFunc, error) { - var fs []RejectByNameFunc - // add patterns from file - if len(opts.ExcludeFiles) > 0 { - excludePatterns, err := readPatternsFromFiles(opts.ExcludeFiles) - if err != nil { - return nil, err - } - - if err := filter.ValidatePatterns(excludePatterns); err != nil { - return nil, errors.Fatalf("--exclude-file: %s", err) - } - - opts.Excludes = append(opts.Excludes, excludePatterns...) - } - - if len(opts.InsensitiveExcludeFiles) > 0 { - excludes, err := readPatternsFromFiles(opts.InsensitiveExcludeFiles) - if err != nil { - return nil, err - } - - if err := filter.ValidatePatterns(excludes); err != nil { - return nil, errors.Fatalf("--iexclude-file: %s", err) - } - - opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...) - } - - if len(opts.InsensitiveExcludes) > 0 { - if err := filter.ValidatePatterns(opts.InsensitiveExcludes); err != nil { - return nil, errors.Fatalf("--iexclude: %s", err) - } - - fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes)) - } - - if len(opts.Excludes) > 0 { - if err := filter.ValidatePatterns(opts.Excludes); err != nil { - return nil, errors.Fatalf("--exclude: %s", err) - } - - fs = append(fs, rejectByPattern(opts.Excludes)) - } - return fs, nil -} diff --git a/cmd/restic/integration_filter_pattern_test.go b/cmd/restic/integration_filter_pattern_test.go index dccbcc0a0..46badbe4f 100644 --- a/cmd/restic/integration_filter_pattern_test.go +++ b/cmd/restic/integration_filter_pattern_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "testing" + "github.com/restic/restic/internal/filter" rtest "github.com/restic/restic/internal/test" ) @@ -17,14 +18,14 @@ func TestBackupFailsWhenUsingInvalidPatterns(t *testing.T) { var err error // Test --exclude - err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) // Test --iexclude - err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided: *[._]log[.-][0-9] @@ -47,14 +48,14 @@ func TestBackupFailsWhenUsingInvalidPatternsFromFile(t *testing.T) { var err error // Test --exclude-file: - err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts) + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts) rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) // Test --iexclude-file - err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{excludeFile}}}, env.gopts) + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludeFiles: []string{excludeFile}}}, env.gopts) rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided: *[._]log[.-][0-9] @@ -70,28 +71,28 @@ func TestRestoreFailsWhenUsingInvalidPatterns(t *testing.T) { var err error // Test --exclude - err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) // Test --iexclude - err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) // Test --include - err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{Includes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --include: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) // Test --iinclude - err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{InsensitiveIncludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --iinclude: invalid pattern(s) provided: *[._]log[.-][0-9] @@ -111,22 +112,22 @@ func TestRestoreFailsWhenUsingInvalidPatternsFromFile(t *testing.T) { t.Fatalf("Could not write include file: %v", fileErr) } - err := testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts) + err := testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts) rtest.Equals(t, `Fatal: --include-file: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) - err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts) rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) - err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts) rtest.Equals(t, `Fatal: --iinclude-file: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) - err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts) rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 4f0990843..a783f6c7f 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -25,7 +25,7 @@ type SelectByNameFunc func(item string) bool // SelectFunc returns true for all items that should be included (files and // dirs). If false is returned, files are ignored and dirs are not even walked. -type SelectFunc func(item string, fi os.FileInfo) bool +type SelectFunc func(item string, fi os.FileInfo, fs fs.FS) bool // ErrorFunc is called when an error during archiving occurs. When nil is // returned, the archiver continues, otherwise it aborts and passes the error @@ -75,6 +75,14 @@ type archiverRepo interface { } // Archiver saves a directory structure to the repo. +// +// An Archiver has a number of worker goroutines handling saving the different +// data structures to the repository, the details are implemented by the +// fileSaver, blobSaver, and treeSaver types. +// +// The main goroutine (the one calling Snapshot()) traverses the directory tree +// and delegates all work to these worker pools. They return a futureNode which +// can be resolved later, by calling Wait() on it. type Archiver struct { Repo archiverRepo SelectByName SelectByNameFunc @@ -82,9 +90,9 @@ type Archiver struct { FS fs.FS Options Options - blobSaver *BlobSaver - fileSaver *FileSaver - treeSaver *TreeSaver + blobSaver *blobSaver + fileSaver *fileSaver + treeSaver *treeSaver mu sync.Mutex summary *Summary @@ -160,7 +168,7 @@ func (o Options) ApplyDefaults() Options { if o.SaveTreeConcurrency == 0 { // can either wait for a file, wait for a tree, serialize a tree or wait for saveblob // the last two are cpu-bound and thus mutually exclusive. - // Also allow waiting for FileReadConcurrency files, this is the maximum of FutureFiles + // Also allow waiting for FileReadConcurrency files, this is the maximum of files // which currently can be in progress. The main backup loop blocks when trying to queue // more files to read. o.SaveTreeConcurrency = uint(runtime.GOMAXPROCS(0)) + o.ReadConcurrency @@ -170,12 +178,12 @@ func (o Options) ApplyDefaults() Options { } // New initializes a new archiver. -func New(repo archiverRepo, fs fs.FS, opts Options) *Archiver { +func New(repo archiverRepo, filesystem fs.FS, opts Options) *Archiver { arch := &Archiver{ Repo: repo, SelectByName: func(_ string) bool { return true }, - Select: func(_ string, _ os.FileInfo) bool { return true }, - FS: fs, + Select: func(_ string, _ os.FileInfo, _ fs.FS) bool { return true }, + FS: filesystem, Options: opts.ApplyDefaults(), CompleteItem: func(string, *restic.Node, *restic.Node, ItemStats, time.Duration) {}, @@ -297,27 +305,27 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error { // saveDir stores a directory in the repo and returns the node. snPath is the // path within the current snapshot. -func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) { +func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete fileCompleteFunc) (d futureNode, err error) { debug.Log("%v %v", snPath, dir) treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi, false) if err != nil { - return FutureNode{}, err + return futureNode{}, err } names, err := fs.Readdirnames(arch.FS, dir, fs.O_NOFOLLOW) if err != nil { - return FutureNode{}, err + return futureNode{}, err } sort.Strings(names) - nodes := make([]FutureNode, 0, len(names)) + nodes := make([]futureNode, 0, len(names)) for _, name := range names { // test if context has been cancelled if ctx.Err() != nil { debug.Log("context has been cancelled, aborting") - return FutureNode{}, ctx.Err() + return futureNode{}, ctx.Err() } pathname := arch.FS.Join(dir, name) @@ -333,7 +341,7 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi continue } - return FutureNode{}, err + return futureNode{}, err } if excluded { @@ -348,11 +356,11 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi return fn, nil } -// FutureNode holds a reference to a channel that returns a FutureNodeResult +// futureNode holds a reference to a channel that returns a FutureNodeResult // or a reference to an already existing result. If the result is available // immediately, then storing a reference directly requires less memory than // using the indirection via a channel. -type FutureNode struct { +type futureNode struct { ch <-chan futureNodeResult res *futureNodeResult } @@ -365,18 +373,18 @@ type futureNodeResult struct { err error } -func newFutureNode() (FutureNode, chan<- futureNodeResult) { +func newFutureNode() (futureNode, chan<- futureNodeResult) { ch := make(chan futureNodeResult, 1) - return FutureNode{ch: ch}, ch + return futureNode{ch: ch}, ch } -func newFutureNodeWithResult(res futureNodeResult) FutureNode { - return FutureNode{ +func newFutureNodeWithResult(res futureNodeResult) futureNode { + return futureNode{ res: &res, } } -func (fn *FutureNode) take(ctx context.Context) futureNodeResult { +func (fn *futureNode) take(ctx context.Context) futureNodeResult { if fn.res != nil { res := fn.res // free result @@ -415,19 +423,19 @@ func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool { // Errors and completion needs to be handled by the caller. // // snPath is the path within the current snapshot. -func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) { +func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *restic.Node) (fn futureNode, excluded bool, err error) { start := time.Now() debug.Log("%v target %q, previous %v", snPath, target, previous) abstarget, err := arch.FS.Abs(target) if err != nil { - return FutureNode{}, false, err + return futureNode{}, false, err } // exclude files by path before running Lstat to reduce number of lstat calls if !arch.SelectByName(abstarget) { debug.Log("%v is excluded by path", target) - return FutureNode{}, true, nil + return futureNode{}, true, nil } // get file info and run remaining select functions that require file information @@ -436,13 +444,13 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous debug.Log("lstat() for %v returned error: %v", target, err) err = arch.error(abstarget, err) if err != nil { - return FutureNode{}, false, errors.WithStack(err) + return futureNode{}, false, errors.WithStack(err) } - return FutureNode{}, true, nil + return futureNode{}, true, nil } - if !arch.Select(abstarget, fi) { + if !arch.Select(abstarget, fi, arch.FS) { debug.Log("%v is excluded", target) - return FutureNode{}, true, nil + return futureNode{}, true, nil } switch { @@ -451,14 +459,14 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous // check if the file has not changed before performing a fopen operation (more expensive, specially // in network filesystems) - if previous != nil && !fileChanged(fi, previous, arch.ChangeIgnoreFlags) { + if previous != nil && !fileChanged(arch.FS, fi, previous, arch.ChangeIgnoreFlags) { if arch.allBlobsPresent(previous) { debug.Log("%v hasn't changed, using old list of blobs", target) arch.trackItem(snPath, previous, previous, ItemStats{}, time.Since(start)) arch.CompleteBlob(previous.Size) node, err := arch.nodeFromFileInfo(snPath, target, fi, false) if err != nil { - return FutureNode{}, false, err + return futureNode{}, false, err } // copy list of blobs @@ -477,7 +485,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous err := errors.Errorf("parts of %v not found in the repository index; storing the file again", target) err = arch.error(abstarget, err) if err != nil { - return FutureNode{}, false, err + return futureNode{}, false, err } } @@ -488,9 +496,9 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous debug.Log("Openfile() for %v returned error: %v", target, err) err = arch.error(abstarget, err) if err != nil { - return FutureNode{}, false, errors.WithStack(err) + return futureNode{}, false, errors.WithStack(err) } - return FutureNode{}, true, nil + return futureNode{}, true, nil } fi, err = file.Stat() @@ -499,9 +507,9 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous _ = file.Close() err = arch.error(abstarget, err) if err != nil { - return FutureNode{}, false, errors.WithStack(err) + return futureNode{}, false, errors.WithStack(err) } - return FutureNode{}, true, nil + return futureNode{}, true, nil } // make sure it's still a file @@ -510,9 +518,9 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous _ = file.Close() err = arch.error(abstarget, err) if err != nil { - return FutureNode{}, false, err + return futureNode{}, false, err } - return FutureNode{}, true, nil + return futureNode{}, true, nil } // Save will close the file, we don't need to do that @@ -533,7 +541,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous err = arch.error(abstarget, err) } if err != nil { - return FutureNode{}, false, err + return futureNode{}, false, err } fn, err = arch.saveDir(ctx, snPath, target, fi, oldSubtree, @@ -542,19 +550,19 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous }) if err != nil { debug.Log("SaveDir for %v returned error: %v", snPath, err) - return FutureNode{}, false, err + return futureNode{}, false, err } case fi.Mode()&os.ModeSocket > 0: debug.Log(" %v is a socket, ignoring", target) - return FutureNode{}, true, nil + return futureNode{}, true, nil default: debug.Log(" %v other", target) node, err := arch.nodeFromFileInfo(snPath, target, fi, false) if err != nil { - return FutureNode{}, false, err + return futureNode{}, false, err } fn = newFutureNodeWithResult(futureNodeResult{ snPath: snPath, @@ -571,7 +579,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous // fileChanged tries to detect whether a file's content has changed compared // to the contents of node, which describes the same path in the parent backup. // It should only be run for regular files. -func fileChanged(fi os.FileInfo, node *restic.Node, ignoreFlags uint) bool { +func fileChanged(fs fs.FS, fi os.FileInfo, node *restic.Node, ignoreFlags uint) bool { switch { case node == nil: return true @@ -621,17 +629,17 @@ func (arch *Archiver) statDir(dir string) (os.FileInfo, error) { // saveTree stores a Tree in the repo, returned is the tree. snPath is the path // within the current snapshot. -func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) { +func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *tree, previous *restic.Tree, complete fileCompleteFunc) (futureNode, int, error) { var node *restic.Node if snPath != "/" { if atree.FileInfoPath == "" { - return FutureNode{}, 0, errors.Errorf("FileInfoPath for %v is empty", snPath) + return futureNode{}, 0, errors.Errorf("FileInfoPath for %v is empty", snPath) } fi, err := arch.statDir(atree.FileInfoPath) if err != nil { - return FutureNode{}, 0, err + return futureNode{}, 0, err } debug.Log("%v, dir node data loaded from %v", snPath, atree.FileInfoPath) @@ -639,7 +647,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *Tree, // thus ignore errors for such folders. node, err = arch.nodeFromFileInfo(snPath, atree.FileInfoPath, fi, true) if err != nil { - return FutureNode{}, 0, err + return futureNode{}, 0, err } } else { // fake root node @@ -648,7 +656,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *Tree, debug.Log("%v (%v nodes), parent %v", snPath, len(atree.Nodes), previous) nodeNames := atree.NodeNames() - nodes := make([]FutureNode, 0, len(nodeNames)) + nodes := make([]futureNode, 0, len(nodeNames)) // iterate over the nodes of atree in lexicographic (=deterministic) order for _, name := range nodeNames { @@ -656,7 +664,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *Tree, // test if context has been cancelled if ctx.Err() != nil { - return FutureNode{}, 0, ctx.Err() + return futureNode{}, 0, ctx.Err() } // this is a leaf node @@ -669,11 +677,11 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *Tree, // ignore error continue } - return FutureNode{}, 0, err + return futureNode{}, 0, err } if err != nil { - return FutureNode{}, 0, err + return futureNode{}, 0, err } if !excluded { @@ -691,7 +699,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *Tree, err = arch.error(join(snPath, name), err) } if err != nil { - return FutureNode{}, 0, err + return futureNode{}, 0, err } // not a leaf node, archive subtree @@ -699,7 +707,7 @@ func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *Tree, arch.trackItem(snItem, oldNode, n, is, time.Since(start)) }) if err != nil { - return FutureNode{}, 0, err + return futureNode{}, 0, err } nodes = append(nodes, fn) } @@ -779,16 +787,16 @@ func (arch *Archiver) loadParentTree(ctx context.Context, sn *restic.Snapshot) * // runWorkers starts the worker pools, which are stopped when the context is cancelled. func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group) { - arch.blobSaver = NewBlobSaver(ctx, wg, arch.Repo, arch.Options.SaveBlobConcurrency) + arch.blobSaver = newBlobSaver(ctx, wg, arch.Repo, arch.Options.SaveBlobConcurrency) - arch.fileSaver = NewFileSaver(ctx, wg, + arch.fileSaver = newFileSaver(ctx, wg, arch.blobSaver.Save, arch.Repo.Config().ChunkerPolynomial, arch.Options.ReadConcurrency, arch.Options.SaveBlobConcurrency) arch.fileSaver.CompleteBlob = arch.CompleteBlob arch.fileSaver.NodeFromFileInfo = arch.nodeFromFileInfo - arch.treeSaver = NewTreeSaver(ctx, wg, arch.Options.SaveTreeConcurrency, arch.blobSaver.Save, arch.Error) + arch.treeSaver = newTreeSaver(ctx, wg, arch.Options.SaveTreeConcurrency, arch.blobSaver.Save, arch.Error) } func (arch *Archiver) stopWorkers() { @@ -809,7 +817,7 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps return nil, restic.ID{}, nil, err } - atree, err := NewTree(arch.FS, cleanTargets) + atree, err := newTree(arch.FS, cleanTargets) if err != nil { return nil, restic.ID{}, nil, err } diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index d67b5b06a..962fd5481 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -686,10 +686,11 @@ func TestFileChanged(t *testing.T) { } save(t, filename, content) + fs := &fs.Local{} fiBefore := lstat(t, filename) node := nodeFromFI(t, filename, fiBefore) - if fileChanged(fiBefore, node, 0) { + if fileChanged(fs, fiBefore, node, 0) { t.Fatalf("unchanged file detected as changed") } @@ -699,12 +700,12 @@ func TestFileChanged(t *testing.T) { if test.SameFile { // file should be detected as unchanged - if fileChanged(fiAfter, node, test.ChangeIgnore) { + if fileChanged(fs, fiAfter, node, test.ChangeIgnore) { t.Fatalf("unmodified file detected as changed") } } else { // file should be detected as changed - if !fileChanged(fiAfter, node, test.ChangeIgnore) && !test.SameFile { + if !fileChanged(fs, fiAfter, node, test.ChangeIgnore) && !test.SameFile { t.Fatalf("modified file detected as unchanged") } } @@ -721,7 +722,7 @@ func TestFilChangedSpecialCases(t *testing.T) { t.Run("nil-node", func(t *testing.T) { fi := lstat(t, filename) - if !fileChanged(fi, nil, 0) { + if !fileChanged(&fs.Local{}, fi, nil, 0) { t.Fatal("nil node detected as unchanged") } }) @@ -730,7 +731,7 @@ func TestFilChangedSpecialCases(t *testing.T) { fi := lstat(t, filename) node := nodeFromFI(t, filename, fi) node.Type = "symlink" - if !fileChanged(fi, node, 0) { + if !fileChanged(&fs.Local{}, fi, node, 0) { t.Fatal("node with changed type detected as unchanged") } }) @@ -1121,7 +1122,7 @@ func TestArchiverSaveTree(t *testing.T) { test.prepare(t) } - atree, err := NewTree(testFS, test.targets) + atree, err := newTree(testFS, test.targets) if err != nil { t.Fatal(err) } @@ -1529,7 +1530,7 @@ func TestArchiverSnapshotSelect(t *testing.T) { }, "other": TestFile{Content: "another file"}, }, - selFn: func(item string, fi os.FileInfo) bool { + selFn: func(item string, fi os.FileInfo, _ fs.FS) bool { return true }, }, @@ -1546,7 +1547,7 @@ func TestArchiverSnapshotSelect(t *testing.T) { }, "other": TestFile{Content: "another file"}, }, - selFn: func(item string, fi os.FileInfo) bool { + selFn: func(item string, fi os.FileInfo, _ fs.FS) bool { return false }, err: "snapshot is empty", @@ -1573,7 +1574,7 @@ func TestArchiverSnapshotSelect(t *testing.T) { }, "other": TestFile{Content: "another file"}, }, - selFn: func(item string, fi os.FileInfo) bool { + selFn: func(item string, fi os.FileInfo, _ fs.FS) bool { return filepath.Ext(item) != ".txt" }, }, @@ -1597,8 +1598,8 @@ func TestArchiverSnapshotSelect(t *testing.T) { }, "other": TestFile{Content: "another file"}, }, - selFn: func(item string, fi os.FileInfo) bool { - return filepath.Base(item) != "subdir" + selFn: func(item string, fi os.FileInfo, fs fs.FS) bool { + return fs.Base(item) != "subdir" }, }, { @@ -1606,8 +1607,8 @@ func TestArchiverSnapshotSelect(t *testing.T) { src: TestDir{ "foo": TestFile{Content: "foo"}, }, - selFn: func(item string, fi os.FileInfo) bool { - return filepath.IsAbs(item) + selFn: func(item string, fi os.FileInfo, fs fs.FS) bool { + return fs.IsAbs(item) }, }, } diff --git a/internal/archiver/blob_saver.go b/internal/archiver/blob_saver.go index d4347a169..356a32ce2 100644 --- a/internal/archiver/blob_saver.go +++ b/internal/archiver/blob_saver.go @@ -9,22 +9,22 @@ import ( "golang.org/x/sync/errgroup" ) -// Saver allows saving a blob. -type Saver interface { +// saver allows saving a blob. +type saver interface { SaveBlob(ctx context.Context, t restic.BlobType, data []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) } -// BlobSaver concurrently saves incoming blobs to the repo. -type BlobSaver struct { - repo Saver +// blobSaver concurrently saves incoming blobs to the repo. +type blobSaver struct { + repo saver ch chan<- saveBlobJob } -// NewBlobSaver returns a new blob. A worker pool is started, it is stopped +// newBlobSaver returns a new blob. A worker pool is started, it is stopped // when ctx is cancelled. -func NewBlobSaver(ctx context.Context, wg *errgroup.Group, repo Saver, workers uint) *BlobSaver { +func newBlobSaver(ctx context.Context, wg *errgroup.Group, repo saver, workers uint) *blobSaver { ch := make(chan saveBlobJob) - s := &BlobSaver{ + s := &blobSaver{ repo: repo, ch: ch, } @@ -38,13 +38,13 @@ func NewBlobSaver(ctx context.Context, wg *errgroup.Group, repo Saver, workers u return s } -func (s *BlobSaver) TriggerShutdown() { +func (s *blobSaver) TriggerShutdown() { close(s.ch) } // Save stores a blob in the repo. It checks the index and the known blobs // before saving anything. It takes ownership of the buffer passed in. -func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer, filename string, cb func(res SaveBlobResponse)) { +func (s *blobSaver) Save(ctx context.Context, t restic.BlobType, buf *buffer, filename string, cb func(res saveBlobResponse)) { select { case s.ch <- saveBlobJob{BlobType: t, buf: buf, fn: filename, cb: cb}: case <-ctx.Done(): @@ -54,26 +54,26 @@ func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer, fi type saveBlobJob struct { restic.BlobType - buf *Buffer + buf *buffer fn string - cb func(res SaveBlobResponse) + cb func(res saveBlobResponse) } -type SaveBlobResponse struct { +type saveBlobResponse struct { id restic.ID length int sizeInRepo int known bool } -func (s *BlobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (SaveBlobResponse, error) { +func (s *blobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (saveBlobResponse, error) { id, known, sizeInRepo, err := s.repo.SaveBlob(ctx, t, buf, restic.ID{}, false) if err != nil { - return SaveBlobResponse{}, err + return saveBlobResponse{}, err } - return SaveBlobResponse{ + return saveBlobResponse{ id: id, length: len(buf), sizeInRepo: sizeInRepo, @@ -81,7 +81,7 @@ func (s *BlobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) }, nil } -func (s *BlobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error { +func (s *blobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error { for { var job saveBlobJob var ok bool diff --git a/internal/archiver/blob_saver_test.go b/internal/archiver/blob_saver_test.go index f7ef2f47d..e23ed12e5 100644 --- a/internal/archiver/blob_saver_test.go +++ b/internal/archiver/blob_saver_test.go @@ -38,20 +38,20 @@ func TestBlobSaver(t *testing.T) { wg, ctx := errgroup.WithContext(ctx) saver := &saveFail{} - b := NewBlobSaver(ctx, wg, saver, uint(runtime.NumCPU())) + b := newBlobSaver(ctx, wg, saver, uint(runtime.NumCPU())) var wait sync.WaitGroup - var results []SaveBlobResponse + var results []saveBlobResponse var lock sync.Mutex wait.Add(20) for i := 0; i < 20; i++ { - buf := &Buffer{Data: []byte(fmt.Sprintf("foo%d", i))} + buf := &buffer{Data: []byte(fmt.Sprintf("foo%d", i))} idx := i lock.Lock() - results = append(results, SaveBlobResponse{}) + results = append(results, saveBlobResponse{}) lock.Unlock() - b.Save(ctx, restic.DataBlob, buf, "file", func(res SaveBlobResponse) { + b.Save(ctx, restic.DataBlob, buf, "file", func(res saveBlobResponse) { lock.Lock() results[idx] = res lock.Unlock() @@ -95,11 +95,11 @@ func TestBlobSaverError(t *testing.T) { failAt: int32(test.failAt), } - b := NewBlobSaver(ctx, wg, saver, uint(runtime.NumCPU())) + b := newBlobSaver(ctx, wg, saver, uint(runtime.NumCPU())) for i := 0; i < test.blobs; i++ { - buf := &Buffer{Data: []byte(fmt.Sprintf("foo%d", i))} - b.Save(ctx, restic.DataBlob, buf, "errfile", func(res SaveBlobResponse) {}) + buf := &buffer{Data: []byte(fmt.Sprintf("foo%d", i))} + b.Save(ctx, restic.DataBlob, buf, "errfile", func(res saveBlobResponse) {}) } b.TriggerShutdown() diff --git a/internal/archiver/buffer.go b/internal/archiver/buffer.go index 39bda2668..d5bfb46b3 100644 --- a/internal/archiver/buffer.go +++ b/internal/archiver/buffer.go @@ -1,14 +1,14 @@ package archiver -// Buffer is a reusable buffer. After the buffer has been used, Release should +// buffer is a reusable buffer. After the buffer has been used, Release should // be called so the underlying slice is put back into the pool. -type Buffer struct { +type buffer struct { Data []byte - pool *BufferPool + pool *bufferPool } // Release puts the buffer back into the pool it came from. -func (b *Buffer) Release() { +func (b *buffer) Release() { pool := b.pool if pool == nil || cap(b.Data) > pool.defaultSize { return @@ -20,32 +20,32 @@ func (b *Buffer) Release() { } } -// BufferPool implements a limited set of reusable buffers. -type BufferPool struct { - ch chan *Buffer +// bufferPool implements a limited set of reusable buffers. +type bufferPool struct { + ch chan *buffer defaultSize int } -// NewBufferPool initializes a new buffer pool. The pool stores at most max +// newBufferPool initializes a new buffer pool. The pool stores at most max // items. New buffers are created with defaultSize. Buffers that have grown // larger are not put back. -func NewBufferPool(max int, defaultSize int) *BufferPool { - b := &BufferPool{ - ch: make(chan *Buffer, max), +func newBufferPool(max int, defaultSize int) *bufferPool { + b := &bufferPool{ + ch: make(chan *buffer, max), defaultSize: defaultSize, } return b } // Get returns a new buffer, either from the pool or newly allocated. -func (pool *BufferPool) Get() *Buffer { +func (pool *bufferPool) Get() *buffer { select { case buf := <-pool.ch: return buf default: } - b := &Buffer{ + b := &buffer{ Data: make([]byte, pool.defaultSize), pool: pool, } diff --git a/internal/archiver/doc.go b/internal/archiver/doc.go index 928145aa2..1b9603975 100644 --- a/internal/archiver/doc.go +++ b/internal/archiver/doc.go @@ -1,12 +1,3 @@ // Package archiver contains the code which reads files, splits them into // chunks and saves the data to the repository. -// -// An Archiver has a number of worker goroutines handling saving the different -// data structures to the repository, the details are implemented by the -// FileSaver, BlobSaver, and TreeSaver types. -// -// The main goroutine (the one calling Snapshot()) traverses the directory tree -// and delegates all work to these worker pools. They return a type -// (FutureFile, FutureBlob, and FutureTree) which can be resolved later, by -// calling Wait() on it. package archiver diff --git a/internal/archiver/exclude.go b/internal/archiver/exclude.go new file mode 100644 index 000000000..1e855fc3a --- /dev/null +++ b/internal/archiver/exclude.go @@ -0,0 +1,332 @@ +package archiver + +import ( + "bytes" + "fmt" + "io" + "os" + "strings" + "sync" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" +) + +// RejectByNameFunc is a function that takes a filename of a +// file that would be included in the backup. The function returns true if it +// should be excluded (rejected) from the backup. +type RejectByNameFunc func(path string) bool + +// RejectFunc is a function that takes a filename and os.FileInfo of a +// file that would be included in the backup. The function returns true if it +// should be excluded (rejected) from the backup. +type RejectFunc func(path string, fi os.FileInfo, fs fs.FS) bool + +func CombineRejectByNames(funcs []RejectByNameFunc) SelectByNameFunc { + return func(item string) bool { + for _, reject := range funcs { + if reject(item) { + return false + } + } + return true + } +} + +func CombineRejects(funcs []RejectFunc) SelectFunc { + return func(item string, fi os.FileInfo, fs fs.FS) bool { + for _, reject := range funcs { + if reject(item, fi, fs) { + return false + } + } + return true + } +} + +type rejectionCache struct { + m map[string]bool + mtx sync.Mutex +} + +func newRejectionCache() *rejectionCache { + return &rejectionCache{m: make(map[string]bool)} +} + +// Lock locks the mutex in rc. +func (rc *rejectionCache) Lock() { + rc.mtx.Lock() +} + +// Unlock unlocks the mutex in rc. +func (rc *rejectionCache) Unlock() { + rc.mtx.Unlock() +} + +// Get returns the last stored value for dir and a second boolean that +// indicates whether that value was actually written to the cache. It is the +// callers responsibility to call rc.Lock and rc.Unlock before using this +// method, otherwise data races may occur. +func (rc *rejectionCache) Get(dir string) (bool, bool) { + v, ok := rc.m[dir] + return v, ok +} + +// Store stores a new value for dir. It is the callers responsibility to call +// rc.Lock and rc.Unlock before using this method, otherwise data races may +// occur. +func (rc *rejectionCache) Store(dir string, rejected bool) { + rc.m[dir] = rejected +} + +// RejectIfPresent returns a RejectByNameFunc which itself returns whether a path +// should be excluded. The RejectByNameFunc considers a file to be excluded when +// it resides in a directory with an exclusion file, that is specified by +// excludeFileSpec in the form "filename[:content]". The returned error is +// non-nil if the filename component of excludeFileSpec is empty. If rc is +// non-nil, it is going to be used in the RejectByNameFunc to expedite the evaluation +// of a directory based on previous visits. +func RejectIfPresent(excludeFileSpec string, warnf func(msg string, args ...interface{})) (RejectFunc, error) { + if excludeFileSpec == "" { + return nil, errors.New("name for exclusion tagfile is empty") + } + colon := strings.Index(excludeFileSpec, ":") + if colon == 0 { + return nil, fmt.Errorf("no name for exclusion tagfile provided") + } + tf, tc := "", "" + if colon > 0 { + tf = excludeFileSpec[:colon] + tc = excludeFileSpec[colon+1:] + } else { + tf = excludeFileSpec + } + debug.Log("using %q as exclusion tagfile", tf) + rc := newRejectionCache() + return func(filename string, _ os.FileInfo, fs fs.FS) bool { + return isExcludedByFile(filename, tf, tc, rc, fs, warnf) + }, nil +} + +// isExcludedByFile interprets filename as a path and returns true if that file +// is in an excluded directory. A directory is identified as excluded if it contains a +// tagfile which bears the name specified in tagFilename and starts with +// header. If rc is non-nil, it is used to expedite the evaluation of a +// directory based on previous visits. +func isExcludedByFile(filename, tagFilename, header string, rc *rejectionCache, fs fs.FS, warnf func(msg string, args ...interface{})) bool { + if tagFilename == "" { + return false + } + + if fs.Base(filename) == tagFilename { + return false // do not exclude the tagfile itself + } + rc.Lock() + defer rc.Unlock() + + dir := fs.Dir(filename) + rejected, visited := rc.Get(dir) + if visited { + return rejected + } + rejected = isDirExcludedByFile(dir, tagFilename, header, fs, warnf) + rc.Store(dir, rejected) + return rejected +} + +func isDirExcludedByFile(dir, tagFilename, header string, fs fs.FS, warnf func(msg string, args ...interface{})) bool { + tf := fs.Join(dir, tagFilename) + _, err := fs.Lstat(tf) + if errors.Is(err, os.ErrNotExist) { + return false + } + if err != nil { + warnf("could not access exclusion tagfile: %v", err) + return false + } + // when no signature is given, the mere presence of tf is enough reason + // to exclude filename + if len(header) == 0 { + return true + } + // From this stage, errors mean tagFilename exists but it is malformed. + // Warnings will be generated so that the user is informed that the + // indented ignore-action is not performed. + f, err := fs.OpenFile(tf, os.O_RDONLY, 0) + if err != nil { + warnf("could not open exclusion tagfile: %v", err) + return false + } + defer func() { + _ = f.Close() + }() + buf := make([]byte, len(header)) + _, err = io.ReadFull(f, buf) + // EOF is handled with a dedicated message, otherwise the warning were too cryptic + if err == io.EOF { + warnf("invalid (too short) signature in exclusion tagfile %q\n", tf) + return false + } + if err != nil { + warnf("could not read signature from exclusion tagfile %q: %v\n", tf, err) + return false + } + if !bytes.Equal(buf, []byte(header)) { + warnf("invalid signature in exclusion tagfile %q\n", tf) + return false + } + return true +} + +// deviceMap is used to track allowed source devices for backup. This is used to +// check for crossing mount points during backup (for --one-file-system). It +// maps the name of a source path to its device ID. +type deviceMap map[string]uint64 + +// newDeviceMap creates a new device map from the list of source paths. +func newDeviceMap(allowedSourcePaths []string, fs fs.FS) (deviceMap, error) { + deviceMap := make(map[string]uint64) + + for _, item := range allowedSourcePaths { + item, err := fs.Abs(fs.Clean(item)) + if err != nil { + return nil, err + } + + fi, err := fs.Lstat(item) + if err != nil { + return nil, err + } + + id, err := fs.DeviceID(fi) + if err != nil { + return nil, err + } + + deviceMap[item] = id + } + + if len(deviceMap) == 0 { + return nil, errors.New("zero allowed devices") + } + + return deviceMap, nil +} + +// IsAllowed returns true if the path is located on an allowed device. +func (m deviceMap) IsAllowed(item string, deviceID uint64, fs fs.FS) (bool, error) { + for dir := item; ; dir = fs.Dir(dir) { + debug.Log("item %v, test dir %v", item, dir) + + // find a parent directory that is on an allowed device (otherwise + // we would not traverse the directory at all) + allowedID, ok := m[dir] + if !ok { + if dir == fs.Dir(dir) { + // arrived at root, no allowed device found. this should not happen. + break + } + continue + } + + // if the item has a different device ID than the parent directory, + // we crossed a file system boundary + if allowedID != deviceID { + debug.Log("item %v (dir %v) on disallowed device %d", item, dir, deviceID) + return false, nil + } + + // item is on allowed device, accept it + debug.Log("item %v allowed", item) + return true, nil + } + + return false, fmt.Errorf("item %v (device ID %v) not found, deviceMap: %v", item, deviceID, m) +} + +// RejectByDevice returns a RejectFunc that rejects files which are on a +// different file systems than the files/dirs in samples. +func RejectByDevice(samples []string, filesystem fs.FS) (RejectFunc, error) { + deviceMap, err := newDeviceMap(samples, filesystem) + if err != nil { + return nil, err + } + debug.Log("allowed devices: %v\n", deviceMap) + + return func(item string, fi os.FileInfo, fs fs.FS) bool { + id, err := fs.DeviceID(fi) + if err != nil { + // This should never happen because gatherDevices() would have + // errored out earlier. If it still does that's a reason to panic. + panic(err) + } + + allowed, err := deviceMap.IsAllowed(fs.Clean(item), id, fs) + if err != nil { + // this should not happen + panic(fmt.Sprintf("error checking device ID of %v: %v", item, err)) + } + + if allowed { + // accept item + return false + } + + // reject everything except directories + if !fi.IsDir() { + return true + } + + // special case: make sure we keep mountpoints (directories which + // contain a mounted file system). Test this by checking if the parent + // directory would be included. + parentDir := fs.Dir(fs.Clean(item)) + + parentFI, err := fs.Lstat(parentDir) + if err != nil { + debug.Log("item %v: error running lstat() on parent directory: %v", item, err) + // if in doubt, reject + return true + } + + parentDeviceID, err := fs.DeviceID(parentFI) + if err != nil { + debug.Log("item %v: getting device ID of parent directory: %v", item, err) + // if in doubt, reject + return true + } + + parentAllowed, err := deviceMap.IsAllowed(parentDir, parentDeviceID, fs) + if err != nil { + debug.Log("item %v: error checking parent directory: %v", item, err) + // if in doubt, reject + return true + } + + if parentAllowed { + // we found a mount point, so accept the directory + return false + } + + // reject everything else + return true + }, nil +} + +func RejectBySize(maxSize int64) (RejectFunc, error) { + return func(item string, fi os.FileInfo, _ fs.FS) bool { + // directory will be ignored + if fi.IsDir() { + return false + } + + filesize := fi.Size() + if filesize > maxSize { + debug.Log("file %s is oversize: %d", item, filesize) + return true + } + + return false + }, nil +} diff --git a/cmd/restic/exclude_test.go b/internal/archiver/exclude_test.go similarity index 78% rename from cmd/restic/exclude_test.go rename to internal/archiver/exclude_test.go index 9a24418ae..7eb24b08b 100644 --- a/cmd/restic/exclude_test.go +++ b/internal/archiver/exclude_test.go @@ -1,67 +1,14 @@ -package main +package archiver import ( "os" "path/filepath" "testing" + "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/test" ) -func TestRejectByPattern(t *testing.T) { - var tests = []struct { - filename string - reject bool - }{ - {filename: "/home/user/foo.go", reject: true}, - {filename: "/home/user/foo.c", reject: false}, - {filename: "/home/user/foobar", reject: false}, - {filename: "/home/user/foobar/x", reject: true}, - {filename: "/home/user/README", reject: false}, - {filename: "/home/user/README.md", reject: true}, - } - - patterns := []string{"*.go", "README.md", "/home/user/foobar/*"} - - for _, tc := range tests { - t.Run("", func(t *testing.T) { - reject := rejectByPattern(patterns) - res := reject(tc.filename) - if res != tc.reject { - t.Fatalf("wrong result for filename %v: want %v, got %v", - tc.filename, tc.reject, res) - } - }) - } -} - -func TestRejectByInsensitivePattern(t *testing.T) { - var tests = []struct { - filename string - reject bool - }{ - {filename: "/home/user/foo.GO", reject: true}, - {filename: "/home/user/foo.c", reject: false}, - {filename: "/home/user/foobar", reject: false}, - {filename: "/home/user/FOObar/x", reject: true}, - {filename: "/home/user/README", reject: false}, - {filename: "/home/user/readme.md", reject: true}, - } - - patterns := []string{"*.go", "README.md", "/home/user/foobar/*"} - - for _, tc := range tests { - t.Run("", func(t *testing.T) { - reject := rejectByInsensitivePattern(patterns) - res := reject(tc.filename) - if res != tc.reject { - t.Fatalf("wrong result for filename %v: want %v, got %v", - tc.filename, tc.reject, res) - } - }) - } -} - func TestIsExcludedByFile(t *testing.T) { const ( tagFilename = "CACHEDIR.TAG" @@ -102,7 +49,7 @@ func TestIsExcludedByFile(t *testing.T) { if tc.content == "" { h = "" } - if got := isExcludedByFile(foo, tagFilename, h, nil); tc.want != got { + if got := isExcludedByFile(foo, tagFilename, h, newRejectionCache(), &fs.Local{}, func(msg string, args ...interface{}) { t.Logf(msg, args...) }); tc.want != got { t.Fatalf("expected %v, got %v", tc.want, got) } }) @@ -153,8 +100,8 @@ func TestMultipleIsExcludedByFile(t *testing.T) { // create two rejection functions, one that tests for the NOFOO file // and one for the NOBAR file - fooExclude, _ := rejectIfPresent("NOFOO") - barExclude, _ := rejectIfPresent("NOBAR") + fooExclude, _ := RejectIfPresent("NOFOO", nil) + barExclude, _ := RejectIfPresent("NOBAR", nil) // To mock the archiver scanning walk, we create filepath.WalkFn // that tests against the two rejection functions and stores @@ -164,8 +111,8 @@ func TestMultipleIsExcludedByFile(t *testing.T) { if err != nil { return err } - excludedByFoo := fooExclude(p) - excludedByBar := barExclude(p) + excludedByFoo := fooExclude(p, nil, &fs.Local{}) + excludedByBar := barExclude(p, nil, &fs.Local{}) excluded := excludedByFoo || excludedByBar // the log message helps debugging in case the test fails t.Logf("%q: %v || %v = %v", p, excludedByFoo, excludedByBar, excluded) @@ -192,9 +139,6 @@ func TestMultipleIsExcludedByFile(t *testing.T) { func TestIsExcludedByFileSize(t *testing.T) { tempDir := test.TempDir(t) - // Max size of file is set to be 1k - maxSizeStr := "1k" - // Create some files in a temporary directory. // Files in UPPERCASE will be used as exclusion triggers later on. // We will test the inclusion later, so we add the expected value as @@ -238,7 +182,7 @@ func TestIsExcludedByFileSize(t *testing.T) { test.OKs(t, errs) // see if anything went wrong during the creation // create rejection function - sizeExclude, _ := rejectBySize(maxSizeStr) + sizeExclude, _ := RejectBySize(1024) // To mock the archiver scanning walk, we create filepath.WalkFn // that tests against the two rejection functions and stores @@ -249,7 +193,7 @@ func TestIsExcludedByFileSize(t *testing.T) { return err } - excluded := sizeExclude(p, fi) + excluded := sizeExclude(p, fi, nil) // the log message helps debugging in case the test fails t.Logf("%q: dir:%t; size:%d; excluded:%v", p, fi.IsDir(), fi.Size(), excluded) m[p] = !excluded @@ -268,7 +212,7 @@ func TestIsExcludedByFileSize(t *testing.T) { } func TestDeviceMap(t *testing.T) { - deviceMap := DeviceMap{ + deviceMap := deviceMap{ filepath.FromSlash("/"): 1, filepath.FromSlash("/usr/local"): 5, } @@ -299,7 +243,7 @@ func TestDeviceMap(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { - res, err := deviceMap.IsAllowed(filepath.FromSlash(test.item), test.deviceID) + res, err := deviceMap.IsAllowed(filepath.FromSlash(test.item), test.deviceID, &fs.Local{}) if err != nil { t.Fatal(err) } diff --git a/internal/archiver/file_saver.go b/internal/archiver/file_saver.go index d10334301..fa19cab86 100644 --- a/internal/archiver/file_saver.go +++ b/internal/archiver/file_saver.go @@ -15,13 +15,13 @@ import ( "golang.org/x/sync/errgroup" ) -// SaveBlobFn saves a blob to a repo. -type SaveBlobFn func(context.Context, restic.BlobType, *Buffer, string, func(res SaveBlobResponse)) +// saveBlobFn saves a blob to a repo. +type saveBlobFn func(context.Context, restic.BlobType, *buffer, string, func(res saveBlobResponse)) -// FileSaver concurrently saves incoming files to the repo. -type FileSaver struct { - saveFilePool *BufferPool - saveBlob SaveBlobFn +// fileSaver concurrently saves incoming files to the repo. +type fileSaver struct { + saveFilePool *bufferPool + saveBlob saveBlobFn pol chunker.Pol @@ -32,18 +32,18 @@ type FileSaver struct { NodeFromFileInfo func(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) } -// NewFileSaver returns a new file saver. A worker pool with fileWorkers is +// newFileSaver returns a new file saver. A worker pool with fileWorkers is // started, it is stopped when ctx is cancelled. -func NewFileSaver(ctx context.Context, wg *errgroup.Group, save SaveBlobFn, pol chunker.Pol, fileWorkers, blobWorkers uint) *FileSaver { +func newFileSaver(ctx context.Context, wg *errgroup.Group, save saveBlobFn, pol chunker.Pol, fileWorkers, blobWorkers uint) *fileSaver { ch := make(chan saveFileJob) debug.Log("new file saver with %v file workers and %v blob workers", fileWorkers, blobWorkers) poolSize := fileWorkers + blobWorkers - s := &FileSaver{ + s := &fileSaver{ saveBlob: save, - saveFilePool: NewBufferPool(int(poolSize), chunker.MaxSize), + saveFilePool: newBufferPool(int(poolSize), chunker.MaxSize), pol: pol, ch: ch, @@ -60,18 +60,18 @@ func NewFileSaver(ctx context.Context, wg *errgroup.Group, save SaveBlobFn, pol return s } -func (s *FileSaver) TriggerShutdown() { +func (s *fileSaver) TriggerShutdown() { close(s.ch) } -// CompleteFunc is called when the file has been saved. -type CompleteFunc func(*restic.Node, ItemStats) +// fileCompleteFunc is called when the file has been saved. +type fileCompleteFunc func(*restic.Node, ItemStats) // Save stores the file f and returns the data once it has been completed. The // file is closed by Save. completeReading is only called if the file was read // successfully. complete is always called. If completeReading is called, then // this will always happen before calling complete. -func (s *FileSaver) Save(ctx context.Context, snPath string, target string, file fs.File, fi os.FileInfo, start func(), completeReading func(), complete CompleteFunc) FutureNode { +func (s *fileSaver) Save(ctx context.Context, snPath string, target string, file fs.File, fi os.FileInfo, start func(), completeReading func(), complete fileCompleteFunc) futureNode { fn, ch := newFutureNode() job := saveFileJob{ snPath: snPath, @@ -105,11 +105,11 @@ type saveFileJob struct { start func() completeReading func() - complete CompleteFunc + complete fileCompleteFunc } // saveFile stores the file f in the repo, then closes it. -func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPath string, target string, f fs.File, fi os.FileInfo, start func(), finishReading func(), finish func(res futureNodeResult)) { +func (s *fileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPath string, target string, f fs.File, fi os.FileInfo, start func(), finishReading func(), finish func(res futureNodeResult)) { start() fnr := futureNodeResult{ @@ -205,7 +205,7 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat node.Content = append(node.Content, restic.ID{}) lock.Unlock() - s.saveBlob(ctx, restic.DataBlob, buf, target, func(sbr SaveBlobResponse) { + s.saveBlob(ctx, restic.DataBlob, buf, target, func(sbr saveBlobResponse) { lock.Lock() if !sbr.known { fnr.stats.DataBlobs++ @@ -246,7 +246,7 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat completeBlob() } -func (s *FileSaver) worker(ctx context.Context, jobs <-chan saveFileJob) { +func (s *fileSaver) worker(ctx context.Context, jobs <-chan saveFileJob) { // a worker has one chunker which is reused for each file (because it contains a rather large buffer) chnker := chunker.New(nil, s.pol) diff --git a/internal/archiver/file_saver_test.go b/internal/archiver/file_saver_test.go index 948d7ce3c..ede616e28 100644 --- a/internal/archiver/file_saver_test.go +++ b/internal/archiver/file_saver_test.go @@ -30,11 +30,11 @@ func createTestFiles(t testing.TB, num int) (files []string) { return files } -func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Context, *errgroup.Group) { +func startFileSaver(ctx context.Context, t testing.TB) (*fileSaver, context.Context, *errgroup.Group) { wg, ctx := errgroup.WithContext(ctx) - saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *Buffer, _ string, cb func(SaveBlobResponse)) { - cb(SaveBlobResponse{ + saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *buffer, _ string, cb func(saveBlobResponse)) { + cb(saveBlobResponse{ id: restic.Hash(buf.Data), length: len(buf.Data), sizeInRepo: len(buf.Data), @@ -48,7 +48,7 @@ func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Cont t.Fatal(err) } - s := NewFileSaver(ctx, wg, saveBlob, pol, workers, workers) + s := newFileSaver(ctx, wg, saveBlob, pol, workers, workers) s.NodeFromFileInfo = func(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) { return fs.NodeFromFileInfo(filename, fi, ignoreXattrListError) } @@ -69,7 +69,7 @@ func TestFileSaver(t *testing.T) { testFs := fs.Local{} s, ctx, wg := startFileSaver(ctx, t) - var results []FutureNode + var results []futureNode for _, filename := range files { f, err := testFs.OpenFile(filename, os.O_RDONLY, 0) diff --git a/internal/archiver/scanner.go b/internal/archiver/scanner.go index d61e5ce47..debd09aa3 100644 --- a/internal/archiver/scanner.go +++ b/internal/archiver/scanner.go @@ -3,7 +3,6 @@ package archiver import ( "context" "os" - "path/filepath" "sort" "github.com/restic/restic/internal/debug" @@ -22,11 +21,11 @@ type Scanner struct { } // NewScanner initializes a new Scanner. -func NewScanner(fs fs.FS) *Scanner { +func NewScanner(filesystem fs.FS) *Scanner { return &Scanner{ - FS: fs, + FS: filesystem, SelectByName: func(_ string) bool { return true }, - Select: func(_ string, _ os.FileInfo) bool { return true }, + Select: func(_ string, _ os.FileInfo, _ fs.FS) bool { return true }, Error: func(_ string, err error) error { return err }, Result: func(_ string, _ ScanStats) {}, } @@ -38,7 +37,7 @@ type ScanStats struct { Bytes uint64 } -func (s *Scanner) scanTree(ctx context.Context, stats ScanStats, tree Tree) (ScanStats, error) { +func (s *Scanner) scanTree(ctx context.Context, stats ScanStats, tree tree) (ScanStats, error) { // traverse the path in the file system for all leaf nodes if tree.Leaf() { abstarget, err := s.FS.Abs(tree.Path) @@ -83,7 +82,7 @@ func (s *Scanner) Scan(ctx context.Context, targets []string) error { debug.Log("clean targets %v", cleanTargets) // we're using the same tree representation as the archiver does - tree, err := NewTree(s.FS, cleanTargets) + tree, err := newTree(s.FS, cleanTargets) if err != nil { return err } @@ -115,7 +114,7 @@ func (s *Scanner) scan(ctx context.Context, stats ScanStats, target string) (Sca } // run remaining select functions that require file information - if !s.Select(target, fi) { + if !s.Select(target, fi, s.FS) { return stats, nil } @@ -131,7 +130,7 @@ func (s *Scanner) scan(ctx context.Context, stats ScanStats, target string) (Sca sort.Strings(names) for _, name := range names { - stats, err = s.scan(ctx, stats, filepath.Join(target, name)) + stats, err = s.scan(ctx, stats, s.FS.Join(target, name)) if err != nil { return stats, err } diff --git a/internal/archiver/scanner_test.go b/internal/archiver/scanner_test.go index b5b7057b8..e4e2c9f59 100644 --- a/internal/archiver/scanner_test.go +++ b/internal/archiver/scanner_test.go @@ -56,7 +56,7 @@ func TestScanner(t *testing.T) { }, }, }, - selFn: func(item string, fi os.FileInfo) bool { + selFn: func(item string, fi os.FileInfo, fs fs.FS) bool { if fi.IsDir() { return true } diff --git a/internal/archiver/tree.go b/internal/archiver/tree.go index cd03ba521..f4eb1abde 100644 --- a/internal/archiver/tree.go +++ b/internal/archiver/tree.go @@ -9,7 +9,7 @@ import ( "github.com/restic/restic/internal/fs" ) -// Tree recursively defines how a snapshot should look like when +// tree recursively defines how a snapshot should look like when // archived. // // When `Path` is set, this is a leaf node and the contents of `Path` should be @@ -20,8 +20,8 @@ import ( // // `FileInfoPath` is used to extract metadata for intermediate (=non-leaf) // trees. -type Tree struct { - Nodes map[string]Tree +type tree struct { + Nodes map[string]tree Path string // where the files/dirs to be saved are found FileInfoPath string // where the dir can be found that is not included itself, but its subdirs Root string // parent directory of the tree @@ -95,13 +95,13 @@ func rootDirectory(fs fs.FS, target string) string { } // Add adds a new file or directory to the tree. -func (t *Tree) Add(fs fs.FS, path string) error { +func (t *tree) Add(fs fs.FS, path string) error { if path == "" { panic("invalid path (empty string)") } if t.Nodes == nil { - t.Nodes = make(map[string]Tree) + t.Nodes = make(map[string]tree) } pc, virtualPrefix := pathComponents(fs, path, false) @@ -111,7 +111,7 @@ func (t *Tree) Add(fs fs.FS, path string) error { name := pc[0] root := rootDirectory(fs, path) - tree := Tree{Root: root} + tree := tree{Root: root} origName := name i := 0 @@ -152,63 +152,63 @@ func (t *Tree) Add(fs fs.FS, path string) error { } // add adds a new target path into the tree. -func (t *Tree) add(fs fs.FS, target, root string, pc []string) error { +func (t *tree) add(fs fs.FS, target, root string, pc []string) error { if len(pc) == 0 { return errors.Errorf("invalid path %q", target) } if t.Nodes == nil { - t.Nodes = make(map[string]Tree) + t.Nodes = make(map[string]tree) } name := pc[0] if len(pc) == 1 { - tree, ok := t.Nodes[name] + node, ok := t.Nodes[name] if !ok { - t.Nodes[name] = Tree{Path: target} + t.Nodes[name] = tree{Path: target} return nil } - if tree.Path != "" { + if node.Path != "" { return errors.Errorf("path is already set for target %v", target) } - tree.Path = target - t.Nodes[name] = tree + node.Path = target + t.Nodes[name] = node return nil } - tree := Tree{} + node := tree{} if other, ok := t.Nodes[name]; ok { - tree = other + node = other } subroot := fs.Join(root, name) - tree.FileInfoPath = subroot + node.FileInfoPath = subroot - err := tree.add(fs, target, subroot, pc[1:]) + err := node.add(fs, target, subroot, pc[1:]) if err != nil { return err } - t.Nodes[name] = tree + t.Nodes[name] = node return nil } -func (t Tree) String() string { +func (t tree) String() string { return formatTree(t, "") } // Leaf returns true if this is a leaf node, which means Path is set to a // non-empty string and the contents of Path should be inserted at this point // in the tree. -func (t Tree) Leaf() bool { +func (t tree) Leaf() bool { return t.Path != "" } // NodeNames returns the sorted list of subtree names. -func (t Tree) NodeNames() []string { +func (t tree) NodeNames() []string { // iterate over the nodes of atree in lexicographic (=deterministic) order names := make([]string, 0, len(t.Nodes)) for name := range t.Nodes { @@ -219,7 +219,7 @@ func (t Tree) NodeNames() []string { } // formatTree returns a text representation of the tree t. -func formatTree(t Tree, indent string) (s string) { +func formatTree(t tree, indent string) (s string) { for name, node := range t.Nodes { s += fmt.Sprintf("%v/%v, root %q, path %q, meta %q\n", indent, name, node.Root, node.Path, node.FileInfoPath) s += formatTree(node, indent+" ") @@ -228,7 +228,7 @@ func formatTree(t Tree, indent string) (s string) { } // unrollTree unrolls the tree so that only leaf nodes have Path set. -func unrollTree(f fs.FS, t *Tree) error { +func unrollTree(f fs.FS, t *tree) error { // if the current tree is a leaf node (Path is set) and has additional // nodes, add the contents of Path to the nodes. if t.Path != "" && len(t.Nodes) > 0 { @@ -252,7 +252,7 @@ func unrollTree(f fs.FS, t *Tree) error { return errors.Errorf("tree unrollTree: collision on path, node %#v, path %q", node, f.Join(t.Path, entry)) } - t.Nodes[entry] = Tree{Path: f.Join(t.Path, entry)} + t.Nodes[entry] = tree{Path: f.Join(t.Path, entry)} } t.Path = "" } @@ -269,10 +269,10 @@ func unrollTree(f fs.FS, t *Tree) error { return nil } -// NewTree creates a Tree from the target files/directories. -func NewTree(fs fs.FS, targets []string) (*Tree, error) { +// newTree creates a Tree from the target files/directories. +func newTree(fs fs.FS, targets []string) (*tree, error) { debug.Log("targets: %v", targets) - tree := &Tree{} + tree := &tree{} seen := make(map[string]struct{}) for _, target := range targets { target = fs.Clean(target) diff --git a/internal/archiver/tree_saver.go b/internal/archiver/tree_saver.go index 9c11b48f0..aeedefef5 100644 --- a/internal/archiver/tree_saver.go +++ b/internal/archiver/tree_saver.go @@ -9,20 +9,20 @@ import ( "golang.org/x/sync/errgroup" ) -// TreeSaver concurrently saves incoming trees to the repo. -type TreeSaver struct { - saveBlob SaveBlobFn +// treeSaver concurrently saves incoming trees to the repo. +type treeSaver struct { + saveBlob saveBlobFn errFn ErrorFunc ch chan<- saveTreeJob } -// NewTreeSaver returns a new tree saver. A worker pool with treeWorkers is +// newTreeSaver returns a new tree saver. A worker pool with treeWorkers is // started, it is stopped when ctx is cancelled. -func NewTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, saveBlob SaveBlobFn, errFn ErrorFunc) *TreeSaver { +func newTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, saveBlob saveBlobFn, errFn ErrorFunc) *treeSaver { ch := make(chan saveTreeJob) - s := &TreeSaver{ + s := &treeSaver{ ch: ch, saveBlob: saveBlob, errFn: errFn, @@ -37,12 +37,12 @@ func NewTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, sav return s } -func (s *TreeSaver) TriggerShutdown() { +func (s *treeSaver) TriggerShutdown() { close(s.ch) } // Save stores the dir d and returns the data once it has been completed. -func (s *TreeSaver) Save(ctx context.Context, snPath string, target string, node *restic.Node, nodes []FutureNode, complete CompleteFunc) FutureNode { +func (s *treeSaver) Save(ctx context.Context, snPath string, target string, node *restic.Node, nodes []futureNode, complete fileCompleteFunc) futureNode { fn, ch := newFutureNode() job := saveTreeJob{ snPath: snPath, @@ -66,13 +66,13 @@ type saveTreeJob struct { snPath string target string node *restic.Node - nodes []FutureNode + nodes []futureNode ch chan<- futureNodeResult - complete CompleteFunc + complete fileCompleteFunc } // save stores the nodes as a tree in the repo. -func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, ItemStats, error) { +func (s *treeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, ItemStats, error) { var stats ItemStats node := job.node nodes := job.nodes @@ -84,7 +84,7 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I for i, fn := range nodes { // fn is a copy, so clear the original value explicitly - nodes[i] = FutureNode{} + nodes[i] = futureNode{} fnr := fn.take(ctx) // return the error if it wasn't ignored @@ -128,9 +128,9 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I return nil, stats, err } - b := &Buffer{Data: buf} - ch := make(chan SaveBlobResponse, 1) - s.saveBlob(ctx, restic.TreeBlob, b, job.target, func(res SaveBlobResponse) { + b := &buffer{Data: buf} + ch := make(chan saveBlobResponse, 1) + s.saveBlob(ctx, restic.TreeBlob, b, job.target, func(res saveBlobResponse) { ch <- res }) @@ -149,7 +149,7 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I } } -func (s *TreeSaver) worker(ctx context.Context, jobs <-chan saveTreeJob) error { +func (s *treeSaver) worker(ctx context.Context, jobs <-chan saveTreeJob) error { for { var job saveTreeJob var ok bool diff --git a/internal/archiver/tree_saver_test.go b/internal/archiver/tree_saver_test.go index 47a3f3842..4aa4c51f1 100644 --- a/internal/archiver/tree_saver_test.go +++ b/internal/archiver/tree_saver_test.go @@ -12,8 +12,8 @@ import ( "golang.org/x/sync/errgroup" ) -func treeSaveHelper(_ context.Context, _ restic.BlobType, buf *Buffer, _ string, cb func(res SaveBlobResponse)) { - cb(SaveBlobResponse{ +func treeSaveHelper(_ context.Context, _ restic.BlobType, buf *buffer, _ string, cb func(res saveBlobResponse)) { + cb(saveBlobResponse{ id: restic.NewRandomID(), known: false, length: len(buf.Data), @@ -21,7 +21,7 @@ func treeSaveHelper(_ context.Context, _ restic.BlobType, buf *Buffer, _ string, }) } -func setupTreeSaver() (context.Context, context.CancelFunc, *TreeSaver, func() error) { +func setupTreeSaver() (context.Context, context.CancelFunc, *treeSaver, func() error) { ctx, cancel := context.WithCancel(context.Background()) wg, ctx := errgroup.WithContext(ctx) @@ -29,7 +29,7 @@ func setupTreeSaver() (context.Context, context.CancelFunc, *TreeSaver, func() e return err } - b := NewTreeSaver(ctx, wg, uint(runtime.NumCPU()), treeSaveHelper, errFn) + b := newTreeSaver(ctx, wg, uint(runtime.NumCPU()), treeSaveHelper, errFn) shutdown := func() error { b.TriggerShutdown() @@ -43,7 +43,7 @@ func TestTreeSaver(t *testing.T) { ctx, cancel, b, shutdown := setupTreeSaver() defer cancel() - var results []FutureNode + var results []futureNode for i := 0; i < 20; i++ { node := &restic.Node{ @@ -83,13 +83,13 @@ func TestTreeSaverError(t *testing.T) { ctx, cancel, b, shutdown := setupTreeSaver() defer cancel() - var results []FutureNode + var results []futureNode for i := 0; i < test.trees; i++ { node := &restic.Node{ Name: fmt.Sprintf("file-%d", i), } - nodes := []FutureNode{ + nodes := []futureNode{ newFutureNodeWithResult(futureNodeResult{node: &restic.Node{ Name: fmt.Sprintf("child-%d", i), }}), @@ -128,7 +128,7 @@ func TestTreeSaverDuplicates(t *testing.T) { node := &restic.Node{ Name: "file", } - nodes := []FutureNode{ + nodes := []futureNode{ newFutureNodeWithResult(futureNodeResult{node: &restic.Node{ Name: "child", }}), diff --git a/internal/archiver/tree_test.go b/internal/archiver/tree_test.go index a9d2d97ff..c9fe776b1 100644 --- a/internal/archiver/tree_test.go +++ b/internal/archiver/tree_test.go @@ -12,7 +12,7 @@ import ( ) // debug.Log requires Tree.String. -var _ fmt.Stringer = Tree{} +var _ fmt.Stringer = tree{} func TestPathComponents(t *testing.T) { var tests = []struct { @@ -142,20 +142,20 @@ func TestTree(t *testing.T) { var tests = []struct { targets []string src TestDir - want Tree + want tree unix bool win bool mustError bool }{ { targets: []string{"foo"}, - want: Tree{Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ "foo": {Path: "foo", Root: "."}, }}, }, { targets: []string{"foo", "bar", "baz"}, - want: Tree{Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ "foo": {Path: "foo", Root: "."}, "bar": {Path: "bar", Root: "."}, "baz": {Path: "baz", Root: "."}, @@ -163,8 +163,8 @@ func TestTree(t *testing.T) { }, { targets: []string{"foo/user1", "foo/user2", "foo/other"}, - want: Tree{Nodes: map[string]Tree{ - "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ + "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{ "user1": {Path: filepath.FromSlash("foo/user1")}, "user2": {Path: filepath.FromSlash("foo/user2")}, "other": {Path: filepath.FromSlash("foo/other")}, @@ -173,9 +173,9 @@ func TestTree(t *testing.T) { }, { targets: []string{"foo/work/user1", "foo/work/user2"}, - want: Tree{Nodes: map[string]Tree{ - "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{ - "work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ + "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{ + "work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]tree{ "user1": {Path: filepath.FromSlash("foo/work/user1")}, "user2": {Path: filepath.FromSlash("foo/work/user2")}, }}, @@ -184,50 +184,50 @@ func TestTree(t *testing.T) { }, { targets: []string{"foo/user1", "bar/user1", "foo/other"}, - want: Tree{Nodes: map[string]Tree{ - "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ + "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{ "user1": {Path: filepath.FromSlash("foo/user1")}, "other": {Path: filepath.FromSlash("foo/other")}, }}, - "bar": {Root: ".", FileInfoPath: "bar", Nodes: map[string]Tree{ + "bar": {Root: ".", FileInfoPath: "bar", Nodes: map[string]tree{ "user1": {Path: filepath.FromSlash("bar/user1")}, }}, }}, }, { targets: []string{"../work"}, - want: Tree{Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ "work": {Root: "..", Path: filepath.FromSlash("../work")}, }}, }, { targets: []string{"../work/other"}, - want: Tree{Nodes: map[string]Tree{ - "work": {Root: "..", FileInfoPath: filepath.FromSlash("../work"), Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ + "work": {Root: "..", FileInfoPath: filepath.FromSlash("../work"), Nodes: map[string]tree{ "other": {Path: filepath.FromSlash("../work/other")}, }}, }}, }, { targets: []string{"foo/user1", "../work/other", "foo/user2"}, - want: Tree{Nodes: map[string]Tree{ - "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ + "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{ "user1": {Path: filepath.FromSlash("foo/user1")}, "user2": {Path: filepath.FromSlash("foo/user2")}, }}, - "work": {Root: "..", FileInfoPath: filepath.FromSlash("../work"), Nodes: map[string]Tree{ + "work": {Root: "..", FileInfoPath: filepath.FromSlash("../work"), Nodes: map[string]tree{ "other": {Path: filepath.FromSlash("../work/other")}, }}, }}, }, { targets: []string{"foo/user1", "../foo/other", "foo/user2"}, - want: Tree{Nodes: map[string]Tree{ - "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ + "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{ "user1": {Path: filepath.FromSlash("foo/user1")}, "user2": {Path: filepath.FromSlash("foo/user2")}, }}, - "foo-1": {Root: "..", FileInfoPath: filepath.FromSlash("../foo"), Nodes: map[string]Tree{ + "foo-1": {Root: "..", FileInfoPath: filepath.FromSlash("../foo"), Nodes: map[string]tree{ "other": {Path: filepath.FromSlash("../foo/other")}, }}, }}, @@ -240,11 +240,11 @@ func TestTree(t *testing.T) { }, }, targets: []string{"foo", "foo/work"}, - want: Tree{Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ "foo": { Root: ".", FileInfoPath: "foo", - Nodes: map[string]Tree{ + Nodes: map[string]tree{ "file": {Path: filepath.FromSlash("foo/file")}, "work": {Path: filepath.FromSlash("foo/work")}, }, @@ -261,11 +261,11 @@ func TestTree(t *testing.T) { }, }, targets: []string{"foo/work", "foo"}, - want: Tree{Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ "foo": { Root: ".", FileInfoPath: "foo", - Nodes: map[string]Tree{ + Nodes: map[string]tree{ "file": {Path: filepath.FromSlash("foo/file")}, "work": {Path: filepath.FromSlash("foo/work")}, }, @@ -282,11 +282,11 @@ func TestTree(t *testing.T) { }, }, targets: []string{"foo/work", "foo/work/user2"}, - want: Tree{Nodes: map[string]Tree{ - "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ + "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{ "work": { FileInfoPath: filepath.FromSlash("foo/work"), - Nodes: map[string]Tree{ + Nodes: map[string]tree{ "user1": {Path: filepath.FromSlash("foo/work/user1")}, "user2": {Path: filepath.FromSlash("foo/work/user2")}, }, @@ -304,10 +304,10 @@ func TestTree(t *testing.T) { }, }, targets: []string{"foo/work/user2", "foo/work"}, - want: Tree{Nodes: map[string]Tree{ - "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ + "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{ "work": {FileInfoPath: filepath.FromSlash("foo/work"), - Nodes: map[string]Tree{ + Nodes: map[string]tree{ "user1": {Path: filepath.FromSlash("foo/work/user1")}, "user2": {Path: filepath.FromSlash("foo/work/user2")}, }, @@ -332,12 +332,12 @@ func TestTree(t *testing.T) { }, }, targets: []string{"foo/work/user2/data/secret", "foo"}, - want: Tree{Nodes: map[string]Tree{ - "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ + "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{ "other": {Path: filepath.FromSlash("foo/other")}, - "work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]Tree{ - "user2": {FileInfoPath: filepath.FromSlash("foo/work/user2"), Nodes: map[string]Tree{ - "data": {FileInfoPath: filepath.FromSlash("foo/work/user2/data"), Nodes: map[string]Tree{ + "work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]tree{ + "user2": {FileInfoPath: filepath.FromSlash("foo/work/user2"), Nodes: map[string]tree{ + "data": {FileInfoPath: filepath.FromSlash("foo/work/user2/data"), Nodes: map[string]tree{ "secret": { Path: filepath.FromSlash("foo/work/user2/data/secret"), }, @@ -368,10 +368,10 @@ func TestTree(t *testing.T) { }, unix: true, targets: []string{"mnt/driveA", "mnt/driveA/work/driveB"}, - want: Tree{Nodes: map[string]Tree{ - "mnt": {Root: ".", FileInfoPath: filepath.FromSlash("mnt"), Nodes: map[string]Tree{ - "driveA": {FileInfoPath: filepath.FromSlash("mnt/driveA"), Nodes: map[string]Tree{ - "work": {FileInfoPath: filepath.FromSlash("mnt/driveA/work"), Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ + "mnt": {Root: ".", FileInfoPath: filepath.FromSlash("mnt"), Nodes: map[string]tree{ + "driveA": {FileInfoPath: filepath.FromSlash("mnt/driveA"), Nodes: map[string]tree{ + "work": {FileInfoPath: filepath.FromSlash("mnt/driveA/work"), Nodes: map[string]tree{ "driveB": { Path: filepath.FromSlash("mnt/driveA/work/driveB"), }, @@ -384,9 +384,9 @@ func TestTree(t *testing.T) { }, { targets: []string{"foo/work/user", "foo/work/user"}, - want: Tree{Nodes: map[string]Tree{ - "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{ - "work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ + "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{ + "work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]tree{ "user": {Path: filepath.FromSlash("foo/work/user")}, }}, }}, @@ -394,9 +394,9 @@ func TestTree(t *testing.T) { }, { targets: []string{"./foo/work/user", "foo/work/user"}, - want: Tree{Nodes: map[string]Tree{ - "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]Tree{ - "work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ + "foo": {Root: ".", FileInfoPath: "foo", Nodes: map[string]tree{ + "work": {FileInfoPath: filepath.FromSlash("foo/work"), Nodes: map[string]tree{ "user": {Path: filepath.FromSlash("foo/work/user")}, }}, }}, @@ -405,10 +405,10 @@ func TestTree(t *testing.T) { { win: true, targets: []string{`c:\users\foobar\temp`}, - want: Tree{Nodes: map[string]Tree{ - "c": {Root: `c:\`, FileInfoPath: `c:\`, Nodes: map[string]Tree{ - "users": {FileInfoPath: `c:\users`, Nodes: map[string]Tree{ - "foobar": {FileInfoPath: `c:\users\foobar`, Nodes: map[string]Tree{ + want: tree{Nodes: map[string]tree{ + "c": {Root: `c:\`, FileInfoPath: `c:\`, Nodes: map[string]tree{ + "users": {FileInfoPath: `c:\users`, Nodes: map[string]tree{ + "foobar": {FileInfoPath: `c:\users\foobar`, Nodes: map[string]tree{ "temp": {Path: `c:\users\foobar\temp`}, }}, }}, @@ -445,7 +445,7 @@ func TestTree(t *testing.T) { back := rtest.Chdir(t, tempdir) defer back() - tree, err := NewTree(fs.Local{}, test.targets) + tree, err := newTree(fs.Local{}, test.targets) if test.mustError { if err == nil { t.Fatal("expected error, got nil") diff --git a/internal/filter/exclude.go b/internal/filter/exclude.go new file mode 100644 index 000000000..48ecdfddf --- /dev/null +++ b/internal/filter/exclude.go @@ -0,0 +1,162 @@ +package filter + +import ( + "bufio" + "bytes" + "fmt" + "os" + "strings" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/textfile" + "github.com/spf13/pflag" +) + +// RejectByNameFunc is a function that takes a filename of a +// file that would be included in the backup. The function returns true if it +// should be excluded (rejected) from the backup. +type RejectByNameFunc func(path string) bool + +// RejectByPattern returns a RejectByNameFunc which rejects files that match +// one of the patterns. +func RejectByPattern(patterns []string, warnf func(msg string, args ...interface{})) RejectByNameFunc { + parsedPatterns := ParsePatterns(patterns) + return func(item string) bool { + matched, err := List(parsedPatterns, item) + if err != nil { + warnf("error for exclude pattern: %v", err) + } + + if matched { + debug.Log("path %q excluded by an exclude pattern", item) + return true + } + + return false + } +} + +// RejectByInsensitivePattern is like RejectByPattern but case insensitive. +func RejectByInsensitivePattern(patterns []string, warnf func(msg string, args ...interface{})) RejectByNameFunc { + for index, path := range patterns { + patterns[index] = strings.ToLower(path) + } + + rejFunc := RejectByPattern(patterns, warnf) + return func(item string) bool { + return rejFunc(strings.ToLower(item)) + } +} + +// readPatternsFromFiles reads all files and returns the list of +// patterns. For each line, leading and trailing white space is removed +// and comment lines are ignored. For each remaining pattern, environment +// variables are resolved. For adding a literal dollar sign ($), write $$ to +// the file. +func readPatternsFromFiles(files []string) ([]string, error) { + getenvOrDollar := func(s string) string { + if s == "$" { + return "$" + } + return os.Getenv(s) + } + + var patterns []string + for _, filename := range files { + err := func() (err error) { + data, err := textfile.Read(filename) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + // ignore empty lines + if line == "" { + continue + } + + // strip comments + if strings.HasPrefix(line, "#") { + continue + } + + line = os.Expand(line, getenvOrDollar) + patterns = append(patterns, line) + } + return scanner.Err() + }() + if err != nil { + return nil, fmt.Errorf("failed to read patterns from file %q: %w", filename, err) + } + } + return patterns, nil +} + +type ExcludePatternOptions struct { + Excludes []string + InsensitiveExcludes []string + ExcludeFiles []string + InsensitiveExcludeFiles []string +} + +func (opts *ExcludePatternOptions) Add(f *pflag.FlagSet) { + f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") + f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames") + f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)") + f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns") +} + +func (opts *ExcludePatternOptions) Empty() bool { + return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0 +} + +func (opts ExcludePatternOptions) CollectPatterns(warnf func(msg string, args ...interface{})) ([]RejectByNameFunc, error) { + var fs []RejectByNameFunc + // add patterns from file + if len(opts.ExcludeFiles) > 0 { + excludePatterns, err := readPatternsFromFiles(opts.ExcludeFiles) + if err != nil { + return nil, err + } + + if err := ValidatePatterns(excludePatterns); err != nil { + return nil, errors.Fatalf("--exclude-file: %s", err) + } + + opts.Excludes = append(opts.Excludes, excludePatterns...) + } + + if len(opts.InsensitiveExcludeFiles) > 0 { + excludes, err := readPatternsFromFiles(opts.InsensitiveExcludeFiles) + if err != nil { + return nil, err + } + + if err := ValidatePatterns(excludes); err != nil { + return nil, errors.Fatalf("--iexclude-file: %s", err) + } + + opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...) + } + + if len(opts.InsensitiveExcludes) > 0 { + if err := ValidatePatterns(opts.InsensitiveExcludes); err != nil { + return nil, errors.Fatalf("--iexclude: %s", err) + } + + fs = append(fs, RejectByInsensitivePattern(opts.InsensitiveExcludes, warnf)) + } + + if len(opts.Excludes) > 0 { + if err := ValidatePatterns(opts.Excludes); err != nil { + return nil, errors.Fatalf("--exclude: %s", err) + } + + fs = append(fs, RejectByPattern(opts.Excludes, warnf)) + } + return fs, nil +} diff --git a/internal/filter/exclude_test.go b/internal/filter/exclude_test.go new file mode 100644 index 000000000..738fb216d --- /dev/null +++ b/internal/filter/exclude_test.go @@ -0,0 +1,59 @@ +package filter + +import ( + "testing" +) + +func TestRejectByPattern(t *testing.T) { + var tests = []struct { + filename string + reject bool + }{ + {filename: "/home/user/foo.go", reject: true}, + {filename: "/home/user/foo.c", reject: false}, + {filename: "/home/user/foobar", reject: false}, + {filename: "/home/user/foobar/x", reject: true}, + {filename: "/home/user/README", reject: false}, + {filename: "/home/user/README.md", reject: true}, + } + + patterns := []string{"*.go", "README.md", "/home/user/foobar/*"} + + for _, tc := range tests { + t.Run("", func(t *testing.T) { + reject := RejectByPattern(patterns, nil) + res := reject(tc.filename) + if res != tc.reject { + t.Fatalf("wrong result for filename %v: want %v, got %v", + tc.filename, tc.reject, res) + } + }) + } +} + +func TestRejectByInsensitivePattern(t *testing.T) { + var tests = []struct { + filename string + reject bool + }{ + {filename: "/home/user/foo.GO", reject: true}, + {filename: "/home/user/foo.c", reject: false}, + {filename: "/home/user/foobar", reject: false}, + {filename: "/home/user/FOObar/x", reject: true}, + {filename: "/home/user/README", reject: false}, + {filename: "/home/user/readme.md", reject: true}, + } + + patterns := []string{"*.go", "README.md", "/home/user/foobar/*"} + + for _, tc := range tests { + t.Run("", func(t *testing.T) { + reject := RejectByInsensitivePattern(patterns, nil) + res := reject(tc.filename) + if res != tc.reject { + t.Fatalf("wrong result for filename %v: want %v, got %v", + tc.filename, tc.reject, res) + } + }) + } +} diff --git a/cmd/restic/include.go b/internal/filter/include.go similarity index 65% rename from cmd/restic/include.go rename to internal/filter/include.go index dcc4c7f37..87d5f1207 100644 --- a/cmd/restic/include.go +++ b/internal/filter/include.go @@ -1,10 +1,9 @@ -package main +package filter import ( "strings" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/filter" "github.com/spf13/pflag" ) @@ -12,21 +11,21 @@ import ( // in the restore process and returns whether it should be included. type IncludeByNameFunc func(item string) (matched bool, childMayMatch bool) -type includePatternOptions struct { +type IncludePatternOptions struct { Includes []string InsensitiveIncludes []string IncludeFiles []string InsensitiveIncludeFiles []string } -func initIncludePatternOptions(f *pflag.FlagSet, opts *includePatternOptions) { +func (opts *IncludePatternOptions) Add(f *pflag.FlagSet) { f.StringArrayVarP(&opts.Includes, "include", "i", nil, "include a `pattern` (can be specified multiple times)") f.StringArrayVar(&opts.InsensitiveIncludes, "iinclude", nil, "same as --include `pattern` but ignores the casing of filenames") f.StringArrayVar(&opts.IncludeFiles, "include-file", nil, "read include patterns from a `file` (can be specified multiple times)") f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of `file`names in patterns") } -func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) { +func (opts IncludePatternOptions) CollectPatterns(warnf func(msg string, args ...interface{})) ([]IncludeByNameFunc, error) { var fs []IncludeByNameFunc if len(opts.IncludeFiles) > 0 { includePatterns, err := readPatternsFromFiles(opts.IncludeFiles) @@ -34,7 +33,7 @@ func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) return nil, err } - if err := filter.ValidatePatterns(includePatterns); err != nil { + if err := ValidatePatterns(includePatterns); err != nil { return nil, errors.Fatalf("--include-file: %s", err) } @@ -47,7 +46,7 @@ func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) return nil, err } - if err := filter.ValidatePatterns(includePatterns); err != nil { + if err := ValidatePatterns(includePatterns); err != nil { return nil, errors.Fatalf("--iinclude-file: %s", err) } @@ -55,45 +54,45 @@ func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) } if len(opts.InsensitiveIncludes) > 0 { - if err := filter.ValidatePatterns(opts.InsensitiveIncludes); err != nil { + if err := ValidatePatterns(opts.InsensitiveIncludes); err != nil { return nil, errors.Fatalf("--iinclude: %s", err) } - fs = append(fs, includeByInsensitivePattern(opts.InsensitiveIncludes)) + fs = append(fs, IncludeByInsensitivePattern(opts.InsensitiveIncludes, warnf)) } if len(opts.Includes) > 0 { - if err := filter.ValidatePatterns(opts.Includes); err != nil { + if err := ValidatePatterns(opts.Includes); err != nil { return nil, errors.Fatalf("--include: %s", err) } - fs = append(fs, includeByPattern(opts.Includes)) + fs = append(fs, IncludeByPattern(opts.Includes, warnf)) } return fs, nil } -// includeByPattern returns a IncludeByNameFunc which includes files that match +// IncludeByPattern returns a IncludeByNameFunc which includes files that match // one of the patterns. -func includeByPattern(patterns []string) IncludeByNameFunc { - parsedPatterns := filter.ParsePatterns(patterns) +func IncludeByPattern(patterns []string, warnf func(msg string, args ...interface{})) IncludeByNameFunc { + parsedPatterns := ParsePatterns(patterns) return func(item string) (matched bool, childMayMatch bool) { - matched, childMayMatch, err := filter.ListWithChild(parsedPatterns, item) + matched, childMayMatch, err := ListWithChild(parsedPatterns, item) if err != nil { - Warnf("error for include pattern: %v", err) + warnf("error for include pattern: %v", err) } return matched, childMayMatch } } -// includeByInsensitivePattern returns a IncludeByNameFunc which includes files that match +// IncludeByInsensitivePattern returns a IncludeByNameFunc which includes files that match // one of the patterns, ignoring the casing of the filenames. -func includeByInsensitivePattern(patterns []string) IncludeByNameFunc { +func IncludeByInsensitivePattern(patterns []string, warnf func(msg string, args ...interface{})) IncludeByNameFunc { for index, path := range patterns { patterns[index] = strings.ToLower(path) } - includeFunc := includeByPattern(patterns) + includeFunc := IncludeByPattern(patterns, warnf) return func(item string) (matched bool, childMayMatch bool) { return includeFunc(strings.ToLower(item)) } diff --git a/cmd/restic/include_test.go b/internal/filter/include_test.go similarity index 92% rename from cmd/restic/include_test.go rename to internal/filter/include_test.go index 751bfbb76..2f474622c 100644 --- a/cmd/restic/include_test.go +++ b/internal/filter/include_test.go @@ -1,4 +1,4 @@ -package main +package filter import ( "testing" @@ -21,7 +21,7 @@ func TestIncludeByPattern(t *testing.T) { for _, tc := range tests { t.Run(tc.filename, func(t *testing.T) { - includeFunc := includeByPattern(patterns) + includeFunc := IncludeByPattern(patterns, nil) matched, _ := includeFunc(tc.filename) if matched != tc.include { t.Fatalf("wrong result for filename %v: want %v, got %v", @@ -48,7 +48,7 @@ func TestIncludeByInsensitivePattern(t *testing.T) { for _, tc := range tests { t.Run(tc.filename, func(t *testing.T) { - includeFunc := includeByInsensitivePattern(patterns) + includeFunc := IncludeByInsensitivePattern(patterns, nil) matched, _ := includeFunc(tc.filename) if matched != tc.include { t.Fatalf("wrong result for filename %v: want %v, got %v", diff --git a/internal/fs/deviceid_unix.go b/internal/fs/deviceid_unix.go index c366221ab..4d5593335 100644 --- a/internal/fs/deviceid_unix.go +++ b/internal/fs/deviceid_unix.go @@ -10,9 +10,9 @@ import ( "github.com/restic/restic/internal/errors" ) -// DeviceID extracts the device ID from an os.FileInfo object by casting it +// deviceID extracts the device ID from an os.FileInfo object by casting it // to syscall.Stat_t -func DeviceID(fi os.FileInfo) (deviceID uint64, err error) { +func deviceID(fi os.FileInfo) (deviceID uint64, err error) { if fi == nil { return 0, errors.New("unable to determine device: fi is nil") } diff --git a/internal/fs/deviceid_windows.go b/internal/fs/deviceid_windows.go index 42355817d..bfb22dc9a 100644 --- a/internal/fs/deviceid_windows.go +++ b/internal/fs/deviceid_windows.go @@ -9,8 +9,8 @@ import ( "github.com/restic/restic/internal/errors" ) -// DeviceID extracts the device ID from an os.FileInfo object by casting it +// deviceID extracts the device ID from an os.FileInfo object by casting it // to syscall.Stat_t -func DeviceID(fi os.FileInfo) (deviceID uint64, err error) { +func deviceID(_ os.FileInfo) (deviceID uint64, err error) { return 0, errors.New("Device IDs are not supported on Windows") } diff --git a/internal/fs/fs_local.go b/internal/fs/fs_local.go index 0bcbf7f3a..034d1aa24 100644 --- a/internal/fs/fs_local.go +++ b/internal/fs/fs_local.go @@ -46,6 +46,17 @@ func (fs Local) Lstat(name string) (os.FileInfo, error) { return os.Lstat(fixpath(name)) } +// DeviceID extracts the DeviceID from the given FileInfo. If the fs does +// not support a DeviceID, it returns an error instead +func (fs Local) DeviceID(fi os.FileInfo) (id uint64, err error) { + return deviceID(fi) +} + +// ExtendedStat converts the give FileInfo into ExtendedFileInfo. +func (fs Local) ExtendedStat(fi os.FileInfo) ExtendedFileInfo { + return ExtendedStat(fi) +} + // Join joins any number of path elements into a single path, adding a // Separator if necessary. Join calls Clean on the result; in particular, all // empty strings are ignored. On Windows, the result is a UNC path if and only diff --git a/internal/fs/fs_reader.go b/internal/fs/fs_reader.go index 93a42f9eb..84a79168e 100644 --- a/internal/fs/fs_reader.go +++ b/internal/fs/fs_reader.go @@ -122,6 +122,16 @@ func (fs *Reader) Lstat(name string) (os.FileInfo, error) { return nil, pathError("lstat", name, os.ErrNotExist) } +func (fs *Reader) DeviceID(_ os.FileInfo) (deviceID uint64, err error) { + return 0, errors.New("Device IDs are not supported") +} + +func (fs *Reader) ExtendedStat(fi os.FileInfo) ExtendedFileInfo { + return ExtendedFileInfo{ + FileInfo: fi, + } +} + // Join joins any number of path elements into a single path, adding a // Separator if necessary. Join calls Clean on the result; in particular, all // empty strings are ignored. On Windows, the result is a UNC path if and only diff --git a/internal/fs/interface.go b/internal/fs/interface.go index e1f4ef2d9..bc6aab44a 100644 --- a/internal/fs/interface.go +++ b/internal/fs/interface.go @@ -10,6 +10,8 @@ type FS interface { OpenFile(name string, flag int, perm os.FileMode) (File, error) Stat(name string) (os.FileInfo, error) Lstat(name string) (os.FileInfo, error) + DeviceID(fi os.FileInfo) (deviceID uint64, err error) + ExtendedStat(fi os.FileInfo) ExtendedFileInfo Join(elem ...string) string Separator() string