From 82cfd37263fc64a902fd3ad7e0afdd5acf31c322 Mon Sep 17 00:00:00 2001 From: Arthur Axel 'fREW' Schmidt Date: Fri, 6 Jun 2014 22:10:15 -0500 Subject: [PATCH] Allow prioritization of downloads based on name (fixes #174) --- CONTRIBUTORS | 1 + config/config.go | 59 ++++++++++++++++++++++---- config/config_test.go | 96 +++++++++++++++++++++++++++++++++++++++++++ files/sort.go | 34 +++++++++++++++ model/model.go | 6 ++- 5 files changed, 188 insertions(+), 8 deletions(-) create mode 100644 files/sort.go diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 90e2f9257..c35455f7f 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -1,5 +1,6 @@ Aaron Bieber Andrew Dunham +Arthur Axel fREW Schmidt Brandon Philips James Patterson Jens Diemer diff --git a/config/config.go b/config/config.go index 2c8b75551..54d3512fd 100644 --- a/config/config.go +++ b/config/config.go @@ -11,10 +11,12 @@ import ( "io" "os" "reflect" + "regexp" "sort" "strconv" "strings" + "github.com/calmh/syncthing/scanner" "code.google.com/p/go.crypto/bcrypt" "github.com/calmh/syncthing/logger" ) @@ -30,14 +32,42 @@ type Configuration struct { XMLName xml.Name `xml:"configuration" json:"-"` } +// SyncOrderPattern allows a user to prioritize file downloading based on a +// regular expression. If a file matches the Pattern the Priority will be +// assigned to the file. If a file matches more than one Pattern the +// Priorities are summed. This allows a user to, for example, prioritize files +// in a directory, as well as prioritize based on file type. The higher the +// priority the "sooner" a file will be downloaded. Files can be deprioritized +// by giving them a negative priority. While Priority is represented as an +// integer, the expected range is something like -1000 to 1000. +type SyncOrderPattern struct { + Pattern string `xml:"pattern,attr"` + Priority int `xml:"priority,attr"` + compiledPattern *regexp.Regexp +} + +func (s *SyncOrderPattern) CompiledPattern() *regexp.Regexp { + if s.compiledPattern == nil { + re, err := regexp.Compile(s.Pattern) + if err != nil { + l.Warnln("Could not compile regexp (" + s.Pattern + "): " + err.Error()) + s.compiledPattern = regexp.MustCompile("^\\0$") + } else { + s.compiledPattern = re + } + } + return s.compiledPattern +} + type RepositoryConfiguration struct { - ID string `xml:"id,attr"` - Directory string `xml:"directory,attr"` - Nodes []NodeConfiguration `xml:"node"` - ReadOnly bool `xml:"ro,attr"` - IgnorePerms bool `xml:"ignorePerms,attr"` - Invalid string `xml:"-"` // Set at runtime when there is an error, not saved - Versioning VersioningConfiguration `xml:"versioning"` + ID string `xml:"id,attr"` + Directory string `xml:"directory,attr"` + Nodes []NodeConfiguration `xml:"node"` + ReadOnly bool `xml:"ro,attr"` + IgnorePerms bool `xml:"ignorePerms,attr"` + Invalid string `xml:"-"` // Set at runtime when there is an error, not saved + Versioning VersioningConfiguration `xml:"versioning"` + SyncOrderPatterns []SyncOrderPattern `xml:"syncorder>pattern"` nodeIDs []string } @@ -92,6 +122,21 @@ func (r *RepositoryConfiguration) NodeIDs() []string { return r.nodeIDs } +func (r RepositoryConfiguration) FileRanker() func(scanner.File) int { + if len(r.SyncOrderPatterns) <= 0 { + return nil + } + return func(f scanner.File) int { + ret := 0 + for _, v := range r.SyncOrderPatterns { + if v.CompiledPattern().MatchString(f.Name) { + ret += v.Priority + } + } + return ret + } +} + type NodeConfiguration struct { NodeID string `xml:"id,attr"` Name string `xml:"name,attr,omitempty"` diff --git a/config/config_test.go b/config/config_test.go index dd876d525..58f364a2d 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -10,6 +10,9 @@ import ( "os" "reflect" "testing" + + "github.com/calmh/syncthing/files" + "github.com/calmh/syncthing/scanner" ) func TestDefaultValues(t *testing.T) { @@ -281,3 +284,96 @@ func TestStripNodeIs(t *testing.T) { } } } + +func TestSyncOrders(t *testing.T) { + data := []byte(` + + +
dynamic
+ + + + + + + + +`) + + expected := []SyncOrderPattern{ + { + Pattern: "\\.jpg$", + Priority: 1, + }, + } + + cfg, err := Load(bytes.NewReader(data), "n4") + if err != nil { + t.Error(err) + } + + for i := range expected { + if !reflect.DeepEqual(cfg.Repositories[0].SyncOrderPatterns[i], expected[i]) { + t.Errorf("Nodes[%d] differ;\n E: %#v\n A: %#v", i, expected[i], cfg.Repositories[0].SyncOrderPatterns[i]) + } + } +} + +func TestFileSorter(t *testing.T) { + rcfg := RepositoryConfiguration{ + SyncOrderPatterns: []SyncOrderPattern{ + {"\\.jpg$", 10, nil}, + {"\\.mov$", 5, nil}, + {"^camera-uploads", 100, nil}, + }, + } + + f := []scanner.File{ + {Name: "bar.mov"}, + {Name: "baz.txt"}, + {Name: "foo.jpg"}, + {Name: "frew/foo.jpg"}, + {Name: "frew/lol.go"}, + {Name: "frew/rofl.copter"}, + {Name: "frew/bar.mov"}, + {Name: "camera-uploads/foo.jpg"}, + {Name: "camera-uploads/hurr.pl"}, + {Name: "camera-uploads/herp.mov"}, + {Name: "camera-uploads/wee.txt"}, + } + + files.SortBy(rcfg.FileRanker()).Sort(f) + + expected := []scanner.File{ + {Name: "camera-uploads/foo.jpg"}, + {Name: "camera-uploads/herp.mov"}, + {Name: "camera-uploads/hurr.pl"}, + {Name: "camera-uploads/wee.txt"}, + {Name: "foo.jpg"}, + {Name: "frew/foo.jpg"}, + {Name: "bar.mov"}, + {Name: "frew/bar.mov"}, + {Name: "frew/lol.go"}, + {Name: "baz.txt"}, + {Name: "frew/rofl.copter"}, + } + + if !reflect.DeepEqual(f, expected) { + t.Errorf( + "\n\nexpected:\n" + + formatFiles(expected) + "\n" + + "got:\n" + + formatFiles(f) + "\n\n", + ) + } +} + +func formatFiles(f []scanner.File) string { + ret := "" + + for _, v := range f { + ret += " " + v.Name + "\n" + } + + return ret +} diff --git a/files/sort.go b/files/sort.go new file mode 100644 index 000000000..f5bcdfcff --- /dev/null +++ b/files/sort.go @@ -0,0 +1,34 @@ +package files + +import ( + "sort" + + "github.com/calmh/syncthing/scanner" +) + +type SortBy func(p scanner.File) int + +func (by SortBy) Sort(files []scanner.File) { + ps := &fileSorter{ + files: files, + by: by, + } + sort.Sort(ps) +} + +type fileSorter struct { + files []scanner.File + by func(p1 scanner.File) int +} + +func (s *fileSorter) Len() int { + return len(s.files) +} + +func (s *fileSorter) Swap(i, j int) { + s.files[i], s.files[j] = s.files[j], s.files[i] +} + +func (s *fileSorter) Less(i, j int) bool { + return s.by(s.files[i]) > s.by(s.files[j]) +} diff --git a/model/model.go b/model/model.go index 0861ea602..193e64dd2 100644 --- a/model/model.go +++ b/model/model.go @@ -248,7 +248,11 @@ func (m *Model) NeedFilesRepo(repo string) []scanner.File { m.rmut.RLock() defer m.rmut.RUnlock() if rf, ok := m.repoFiles[repo]; ok { - return rf.Need(cid.LocalID) + f := rf.Need(cid.LocalID) + if r := m.repoCfgs[repo].FileRanker(); r != nil { + files.SortBy(r).Sort(f) + } + return f } return nil }