octoleo/syncthing
1
0
Fork 0
mirror of https://github.com/octoleo/syncthing.git synced 2024-11-08 22:31:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

lib/ignore, lib/model: Use an interface to detect file changes, improving tests

Browse Source 
This solves the erratic test failures on model.TestIgnores by ensuring
that the ignore patterns are reloaded even in the face of unchanged
timestamps.

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4208
This commit is contained in:
Jakob Borg 2017-06-11 10:27:12 +00:00 committed by Audrius Butkevicius
parent 2a38d2a3d2
commit 68c1a0b9b4
7 changed files with 195 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lib/ignore/cache.go
View File

@ -8,6 +8,12 @@ package ignore
import "time"
type nower interface {
Now() time.Time
}
var clock = nower(defaultClock{})
type cache struct {
patterns []Pattern
entries map[string]cacheEntry
@ -27,7 +33,7 @@ func newCache(patterns []Pattern) *cache {
func (c *cache) clean(d time.Duration) {
for k, v := range c.entries {
if time.Since(v.access) > d {
if clock.Now().Sub(v.access) > d {
delete(c.entries, k)
}
}
@ -36,7 +42,7 @@ func (c *cache) clean(d time.Duration) {
func (c *cache) get(key string) (Result, bool) {
entry, ok := c.entries[key]
if ok {
entry.access = time.Now()
entry.access = clock.Now()
c.entries[key] = entry
}
return entry.result, ok
@ -50,3 +56,9 @@ func (c *cache) len() int {
l := len(c.entries)
return l
}
type defaultClock struct{}
func (defaultClock) Now() time.Time {
return time.Now()
}

19
lib/ignore/cache_test.go
View File

@ -12,6 +12,13 @@ import (
)
func TestCache(t *testing.T) {
fc := new(fakeClock)
oldClock := clock
clock = fc
defer func() {
clock = oldClock
}()
c := newCache(nil)
res, ok := c.get("nonexistent")
@ -52,11 +59,11 @@ func TestCache(t *testing.T) {
// Sleep and access, to get some data for clean
time.Sleep(500 * time.Millisecond)
*fc += 500 // milliseconds
c.get("true")
time.Sleep(100 * time.Millisecond)
*fc += 100 // milliseconds
// "false" was accessed ~600 ms ago, "true" was accessed ~100 ms ago.
// This should clean out "false" but not "true"
@ -75,3 +82,11 @@ func TestCache(t *testing.T) {
t.Errorf("item should have been cleaned")
}
}
type fakeClock int64 // milliseconds
func (f *fakeClock) Now() time.Time {
t := time.Unix(int64(*f)/1000, (int64(*f)%1000)*int64(time.Millisecond))
*f++
return t
}

123
lib/ignore/ignore.go
View File

@ -64,6 +64,18 @@ func (r Result) IsCaseFolded() bool {
return r&resultFoldCase == resultFoldCase
}
// The ChangeDetector is responsible for determining if files have changed
// on disk. It gets told to Remember() files (name and modtime) and will
// then get asked if a file has been Seen() (i.e., Remember() has been
// called on it) and if any of the files have Changed(). To forget all
// files, call Reset().
type ChangeDetector interface {
Remember(name string, modtime time.Time)
Seen(name string) bool
Changed() bool
Reset()
}
type Matcher struct {
lines []string
patterns []Pattern
@ -71,17 +83,40 @@ type Matcher struct {
matches *cache
curHash string
stop chan struct{}
modtimes map[string]time.Time
changeDetector ChangeDetector
mut sync.Mutex
}
func New(withCache bool) *Matcher {
// An Option can be passed to New()
type Option func(*Matcher)
// WithCache enables or disables lookup caching. The default is disabled.
func WithCache(v bool) Option {
return func(m *Matcher) {
m.withCache = v
}
}
// WithChangeDetector sets a custom ChangeDetector. The default is to simply
// use the on disk modtime for comparison.
func WithChangeDetector(cd ChangeDetector) Option {
return func(m *Matcher) {
m.changeDetector = cd
}
}
func New(opts ...Option) *Matcher {
m := &Matcher{
withCache: withCache,
stop: make(chan struct{}),
mut: sync.NewMutex(),
}
if withCache {
for _, opt := range opts {
opt(m)
}
if m.changeDetector == nil {
m.changeDetector = newModtimeChecker()
}
if m.withCache {
go m.clean(2 * time.Hour)
}
return m
@ -91,7 +126,7 @@ func (m *Matcher) Load(file string) error {
m.mut.Lock()
defer m.mut.Unlock()
if m.patternsUnchanged(file) {
if m.changeDetector.Seen(file) && !m.changeDetector.Changed() {
return nil
}
@ -108,9 +143,8 @@ func (m *Matcher) Load(file string) error {
return err
}
m.modtimes = map[string]time.Time{
file: info.ModTime(),
}
m.changeDetector.Reset()
m.changeDetector.Remember(file, info.ModTime())
return m.parseLocked(fd, file)
}
@ -122,7 +156,7 @@ func (m *Matcher) Parse(r io.Reader, file string) error {
}
func (m *Matcher) parseLocked(r io.Reader, file string) error {
lines, patterns, err := parseIgnoreFile(r, file, m.modtimes)
lines, patterns, err := parseIgnoreFile(r, file, m.changeDetector)
// Error is saved and returned at the end. We process the patterns
// (possibly blank) anyway.
@ -142,26 +176,6 @@ func (m *Matcher) parseLocked(r io.Reader, file string) error {
return err
}
// patternsUnchanged returns true if none of the files making up the loaded
// patterns have changed since last check.
func (m *Matcher) patternsUnchanged(file string) bool {
if _, ok := m.modtimes[file]; !ok {
return false
}
for filename, modtime := range m.modtimes {
info, err := os.Stat(filename)
if err != nil {
return false
}
if !info.ModTime().Equal(modtime) {
return false
}
}
return true
}
func (m *Matcher) Match(file string) (result Result) {
if m == nil || file == "." {
return resultNotMatched
@ -284,8 +298,8 @@ func hashPatterns(patterns []Pattern) string {
return fmt.Sprintf("%x", h.Sum(nil))
}
func loadIgnoreFile(file string, modtimes map[string]time.Time) ([]string, []Pattern, error) {
if _, ok := modtimes[file]; ok {
func loadIgnoreFile(file string, cd ChangeDetector) ([]string, []Pattern, error) {
if cd.Seen(file) {
return nil, nil, fmt.Errorf("multiple include of ignore file %q", file)
}
@ -299,12 +313,13 @@ func loadIgnoreFile(file string, modtimes map[string]time.Time) ([]string, []Pat
if err != nil {
return nil, nil, err
}
modtimes[file] = info.ModTime()
return parseIgnoreFile(fd, file, modtimes)
cd.Remember(file, info.ModTime())
return parseIgnoreFile(fd, file, cd)
}
func parseIgnoreFile(fd io.Reader, currentFile string, modtimes map[string]time.Time) ([]string, []Pattern, error) {
func parseIgnoreFile(fd io.Reader, currentFile string, cd ChangeDetector) ([]string, []Pattern, error) {
var lines []string
var patterns []Pattern
@ -371,7 +386,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, modtimes map[string]time.
} else if strings.HasPrefix(line, "#include ") {
includeRel := line[len("#include "):]
includeFile := filepath.Join(filepath.Dir(currentFile), includeRel)
includeLines, includePatterns, err := loadIgnoreFile(includeFile, modtimes)
includeLines, includePatterns, err := loadIgnoreFile(includeFile, cd)
if err != nil {
return fmt.Errorf("include of %q: %v", includeRel, err)
}
@ -466,3 +481,41 @@ func WriteIgnores(path string, content []string) error {
return nil
}
// modtimeChecker is the default implementation of ChangeDetector
type modtimeChecker struct {
modtimes map[string]time.Time
}
func newModtimeChecker() *modtimeChecker {
return &modtimeChecker{
modtimes: map[string]time.Time{},
}
}
func (c *modtimeChecker) Remember(name string, modtime time.Time) {
c.modtimes[name] = modtime
}
func (c *modtimeChecker) Seen(name string) bool {
_, ok := c.modtimes[name]
return ok
}
func (c *modtimeChecker) Reset() {
c.modtimes = map[string]time.Time{}
}
func (c *modtimeChecker) Changed() bool {
for name, modtime := range c.modtimes {
info, err := os.Stat(name)
if err != nil {
return true
}
if !info.ModTime().Equal(modtime) {
return true
}
}
return false
}

48
lib/ignore/ignore_test.go
View File

@ -18,7 +18,7 @@ import (
)
func TestIgnore(t *testing.T) {
pats := New(true)
pats := New(WithCache(true))
err := pats.Load("testdata/.stignore")
if err != nil {
t.Fatal(err)
@ -68,7 +68,7 @@ func TestExcludes(t *testing.T) {
i*2
!ign2
`
pats := New(true)
pats := New(WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -113,7 +113,7 @@ func TestFlagOrder(t *testing.T) {
(?i)(?d)(?d)!ign9
(?d)(?d)!ign10
`
pats := New(true)
pats := New(WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -148,7 +148,7 @@ func TestDeletables(t *testing.T) {
ign7
(?i)ign8
`
pats := New(true)
pats := New(WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -187,7 +187,7 @@ func TestBadPatterns(t *testing.T) {
}
for _, pat := range badPatterns {
err := New(true).Parse(bytes.NewBufferString(pat), ".stignore")
err := New(WithCache(true)).Parse(bytes.NewBufferString(pat), ".stignore")
if err == nil {
t.Errorf("No error for pattern %q", pat)
}
@ -195,7 +195,7 @@ func TestBadPatterns(t *testing.T) {
}
func TestCaseSensitivity(t *testing.T) {
ign := New(true)
ign := New(WithCache(true))
err := ign.Parse(bytes.NewBufferString("test"), ".stignore")
if err != nil {
t.Error(err)
@ -247,7 +247,7 @@ func TestCaching(t *testing.T) {
fd2.WriteString("/y/\n")
pats := New(true)
pats := New(WithCache(true))
err = pats.Load(fd1.Name())
if err != nil {
t.Fatal(err)
@ -354,7 +354,7 @@ func TestCommentsAndBlankLines(t *testing.T) {
`
pats := New(true)
pats := New(WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Error(err)
@ -382,7 +382,7 @@ flamingo
*.crow
*.crow
`
pats := New(false)
pats := New()
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
b.Error(err)
@ -424,7 +424,7 @@ flamingo
}
// Load the patterns
pats := New(true)
pats := New(WithCache(true))
err = pats.Load(fd.Name())
if err != nil {
b.Fatal(err)
@ -460,7 +460,7 @@ func TestCacheReload(t *testing.T) {
t.Fatal(err)
}
pats := New(true)
pats := New(WithCache(true))
err = pats.Load(fd.Name())
if err != nil {
t.Fatal(err)
@ -515,7 +515,7 @@ func TestCacheReload(t *testing.T) {
}
func TestHash(t *testing.T) {
p1 := New(true)
p1 := New(WithCache(true))
err := p1.Load("testdata/.stignore")
if err != nil {
t.Fatal(err)
@ -531,7 +531,7 @@ func TestHash(t *testing.T) {
/ffile
lost+found
`
p2 := New(true)
p2 := New(WithCache(true))
err = p2.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -546,7 +546,7 @@ func TestHash(t *testing.T) {
/ffile
lost+found
`
p3 := New(true)
p3 := New(WithCache(true))
err = p3.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -570,7 +570,7 @@ func TestHash(t *testing.T) {
}
func TestHashOfEmpty(t *testing.T) {
p1 := New(true)
p1 := New(WithCache(true))
err := p1.Load("testdata/.stignore")
if err != nil {
t.Fatal(err)
@ -608,7 +608,7 @@ func TestWindowsPatterns(t *testing.T) {
a/b
c\d
`
pats := New(true)
pats := New(WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -633,7 +633,7 @@ func TestAutomaticCaseInsensitivity(t *testing.T) {
A/B
c/d
`
pats := New(true)
pats := New(WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -652,7 +652,7 @@ func TestCommas(t *testing.T) {
foo,bar.txt
{baz,quux}.txt
`
pats := New(true)
pats := New(WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -683,7 +683,7 @@ func TestIssue3164(t *testing.T) {
(?d)(?i)/foo
(?d)(?i)**/bar
`
pats := New(true)
pats := New(WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -719,7 +719,7 @@ func TestIssue3174(t *testing.T) {
stignore := `
*ä*
`
pats := New(true)
pats := New(WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -734,7 +734,7 @@ func TestIssue3639(t *testing.T) {
stignore := `
foo/
`
pats := New(true)
pats := New(WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -767,7 +767,7 @@ func TestIssue3674(t *testing.T) {
{"as/dc", true},
}
pats := New(true)
pats := New(WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -799,7 +799,7 @@ func TestGobwasGlobIssue18(t *testing.T) {
{"bbaa", false},
}
pats := New(true)
pats := New(WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -859,7 +859,7 @@ func TestRoot(t *testing.T) {
{"b", true},
}
pats := New(true)
pats := New(WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)

4
lib/model/model.go
View File

@ -314,7 +314,7 @@ func (m *Model) addFolderLocked(cfg config.FolderConfiguration) {
m.deviceFolders[device.DeviceID] = append(m.deviceFolders[device.DeviceID], cfg.ID)
}
ignores := ignore.New(m.cacheIgnoredFiles)
ignores := ignore.New(ignore.WithCache(m.cacheIgnoredFiles))
if err := ignores.Load(filepath.Join(cfg.Path(), ".stignore")); err != nil && !os.IsNotExist(err) {
l.Warnln("Loading ignores:", err)
}
@ -1260,7 +1260,7 @@ func (m *Model) GetIgnores(folder string) ([]string, []string, error) {
}
if cfg, ok := m.cfg.Folders()[folder]; ok {
matcher := ignore.New(false)
matcher := ignore.New()
path := filepath.Join(cfg.Path(), ".stignore")
if err := matcher.Load(path); err != nil {
return nil, nil, err

46
lib/model/model_test.go
View File

@ -953,14 +953,6 @@ func changeIgnores(t *testing.T, m *Model, expected []string) {
ignores = append(ignores, "pox")
if runtime.GOOS == "darwin" {
// Mac has seconds-only timestamp precision, which tricks the ignore
// system into thinking the file has not changed. Work around it in
// an ugly way...
time.Sleep(time.Second)
} else {
time.Sleep(time.Millisecond)
}
err = m.SetIgnores("default", ignores)
if err != nil {
t.Error(err)
@ -1011,6 +1003,14 @@ func TestIgnores(t *testing.T) {
m.AddFolder(defaultFolderConfig)
m.StartFolder("default")
// Reach in and update the ignore matcher to one that always does
// reloads when asked to, instead of checking file mtimes. This is
// because we will be changing the files on disk often enough that the
// mtimes will be unreliable to determine change status.
m.fmut.Lock()
m.folderIgnores["default"] = ignore.New(ignore.WithCache(true), ignore.WithChangeDetector(newAlwaysChanged()))
m.fmut.Unlock()
// Make sure the initial scan has finished (ScanFolders is blocking)
m.ScanFolders()
@ -1843,7 +1843,7 @@ func TestIssue3164(t *testing.T) {
f := protocol.FileInfo{
Name: "issue3164",
}
m := ignore.New(false)
m := ignore.New()
if err := m.Parse(bytes.NewBufferString("(?d)oktodelete"), ""); err != nil {
t.Fatal(err)
}
@ -2490,3 +2490,31 @@ func (fakeAddr) Network() string {
func (fakeAddr) String() string {
return "address"
}
// alwaysChanges is an ignore.ChangeDetector that always returns true on Changed()
type alwaysChanged struct {
seen map[string]struct{}
}
func newAlwaysChanged() *alwaysChanged {
return &alwaysChanged{
seen: make(map[string]struct{}),
}
}
func (c *alwaysChanged) Remember(name string, _ time.Time) {
c.seen[name] = struct{}{}
}
func (c *alwaysChanged) Reset() {
c.seen = make(map[string]struct{})
}
func (c *alwaysChanged) Seen(name string) bool {
_, ok := c.seen[name]
return ok
}
func (c *alwaysChanged) Changed() bool {
return true
}

4
lib/scanner/walk_test.go
View File

@ -54,7 +54,7 @@ func init() {
}
func TestWalkSub(t *testing.T) {
ignores := ignore.New(false)
ignores := ignore.New()
err := ignores.Load("testdata/.stignore")
if err != nil {
t.Fatal(err)
@ -90,7 +90,7 @@ func TestWalkSub(t *testing.T) {
}
func TestWalk(t *testing.T) {
ignores := ignore.New(false)
ignores := ignore.New()
err := ignores.Load("testdata/.stignore")
if err != nil {
t.Fatal(err)