octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-02 11:46:36 +00:00
Code Issues Releases Wiki Activity

Merge pull request #993 from restic/improve-search-performance

Browse Source 
Improve find
This commit is contained in:
Alexander Neumann 2017-06-04 12:44:29 +02:00
commit a9a2798910
2 changed files with 50 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -14,6 +14,12 @@ Important Changes in 0.X.Y
https://github.com/restic/restic/issues/512 https://github.com/restic/restic/issues/512
https://github.com/restic/restic/pull/978 https://github.com/restic/restic/pull/978
* Improved performance for the `find` command: Restic recognizes paths it has
already checked for the files in question, so the number of backend requests
is reduced a lot.
https://github.com/restic/restic/issues/989
https://github.com/restic/restic/pull/993
Important Changes in 0.6.1 Important Changes in 0.6.1
========================== ==========================

66
src/cmds/restic/cmd_find.go
View File

@ -12,7 +12,6 @@ import (
"restic" "restic"
"restic/debug" "restic/debug"
"restic/errors" "restic/errors"
"restic/repository"
) )
var cmdFind = &cobra.Command{ var cmdFind = &cobra.Command{
@ -172,59 +171,76 @@ func (s *statefulOutput) Finish() {
} }
} }
func findInTree(repo *repository.Repository, pat *findPattern, id restic.ID, prefix string, state *statefulOutput) error { // Finder bundles information needed to find a file or directory.
debug.Log("checking tree %v\n", id) type Finder struct {
repo restic.Repository
pat findPattern
out statefulOutput
notfound restic.IDSet
}
tree, err := repo.LoadTree(id) func (f *Finder) findInTree(treeID restic.ID, prefix string) error {
if f.notfound.Has(treeID) {
debug.Log("%v skipping tree %v, has already been checked", prefix, treeID.Str())
return nil
}
debug.Log("%v checking tree %v\n", prefix, treeID.Str())
tree, err := f.repo.LoadTree(treeID)
if err != nil { if err != nil {
return err return err
} }
var found bool
for _, node := range tree.Nodes { for _, node := range tree.Nodes {
debug.Log(" testing entry %q\n", node.Name) debug.Log(" testing entry %q\n", node.Name)
name := node.Name name := node.Name
if pat.ignoreCase { if f.pat.ignoreCase {
name = strings.ToLower(name) name = strings.ToLower(name)
} }
m, err := filepath.Match(pat.pattern, name) m, err := filepath.Match(f.pat.pattern, name)
if err != nil { if err != nil {
return err return err
} }
if m { if m {
debug.Log(" pattern matches\n") if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) {
if !pat.oldest.IsZero() && node.ModTime.Before(pat.oldest) { debug.Log(" ModTime is older than %s\n", f.pat.oldest)
debug.Log(" ModTime is older than %s\n", pat.oldest)
continue continue
} }
if !pat.newest.IsZero() && node.ModTime.After(pat.newest) { if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) {
debug.Log(" ModTime is newer than %s\n", pat.newest) debug.Log(" ModTime is newer than %s\n", f.pat.newest)
continue continue
} }
state.Print(prefix, node) debug.Log(" found match\n")
} else { found = true
debug.Log(" pattern does not match\n") f.out.Print(prefix, node)
} }
if node.Type == "dir" { if node.Type == "dir" {
if err := findInTree(repo, pat, *node.Subtree, filepath.Join(prefix, node.Name), state); err != nil { if err := f.findInTree(*node.Subtree, filepath.Join(prefix, node.Name)); err != nil {
return err return err
} }
} }
} }
if !found {
f.notfound.Insert(treeID)
}
return nil return nil
} }
func findInSnapshot(repo *repository.Repository, sn *restic.Snapshot, pat findPattern, state *statefulOutput) error { func (f *Finder) findInSnapshot(sn *restic.Snapshot) error {
debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), pat.oldest, pat.newest) debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), f.pat.oldest, f.pat.newest)
state.newsn = sn f.out.newsn = sn
if err := findInTree(repo, &pat, *sn.Tree, string(filepath.Separator), state); err != nil { if err := f.findInTree(*sn.Tree, string(filepath.Separator)); err != nil {
return err return err
} }
return nil return nil
@ -273,13 +289,19 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
ctx, cancel := context.WithCancel(gopts.ctx) ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel() defer cancel()
state := statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON}
f := &Finder{
repo: repo,
pat: pat,
out: statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON},
notfound: restic.NewIDSet(),
}
for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) { for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) {
if err = findInSnapshot(repo, sn, pat, &state); err != nil { if err = f.findInSnapshot(sn); err != nil {
return err return err
} }
} }
state.Finish() f.out.Finish()
return nil return nil
} }