lib/fs: Properly handle case insensitive systems (fixes #1787, fixes #2739, fixes #5708)

With this change we emulate a case sensitive filesystem on top of
insensitive filesystems. This means we correctly pick up case-only renames
and throw a case conflict error when there would be multiple files differing
only in case.

This safety check has a small performance hit (about 20% more filesystem
operations when scanning for changes). The new advanced folder option
`caseSensitiveFS` can be used to disable the safety checks, retaining the
previous behavior on systems known to be fully case sensitive.

Co-authored-by: Jakob Borg <jakob@kastelo.net>
This commit is contained in:
Simon Frei 2020-07-28 11:13:15 +02:00 committed by Jakob Borg
parent 21dd9d6b43
commit 932d8c69de
27 changed files with 1241 additions and 187 deletions

View File

@ -743,6 +743,7 @@ func getReport(db *sql.DB) map[string]interface{} {
inc(features["Folder"]["v3"], "Weak hash, always", rep.FolderUsesV3.AlwaysWeakHash)
inc(features["Folder"]["v3"], "Weak hash, custom threshold", rep.FolderUsesV3.CustomWeakHashThreshold)
inc(features["Folder"]["v3"], "Filesystem watcher", rep.FolderUsesV3.FsWatcherEnabled)
inc(features["Folder"]["v3"], "Case sensitive FS", rep.FolderUsesV3.CaseSensitiveFS)
add(featureGroups["Folder"]["v3"], "Conflicts", "Disabled", rep.FolderUsesV3.ConflictsDisabled)
add(featureGroups["Folder"]["v3"], "Conflicts", "Unlimited", rep.FolderUsesV3.ConflictsUnlimited)

View File

@ -22,7 +22,6 @@ import (
"github.com/d4l3k/messagediff"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
)
@ -129,13 +128,6 @@ func TestDeviceConfig(t *testing.T) {
},
}
// The cachedFilesystem will have been resolved to an absolute path,
// depending on where the tests are running. Zero it out so we don't
// fail based on that.
for i := range cfg.Folders {
cfg.Folders[i].cachedFilesystem = nil
}
expectedDevices := []DeviceConfiguration{
{
DeviceID: device1,
@ -465,6 +457,7 @@ func TestFolderCheckPath(t *testing.T) {
if err != nil {
t.Fatal(err)
}
testFs := fs.NewFilesystem(fs.FilesystemTypeBasic, n)
err = os.MkdirAll(filepath.Join(n, "dir", ".stfolder"), os.FileMode(0777))
if err != nil {
@ -489,7 +482,7 @@ func TestFolderCheckPath(t *testing.T) {
},
}
err = osutil.DebugSymlinkForTestsOnly(filepath.Join(n, "dir"), filepath.Join(n, "link"))
err = fs.DebugSymlinkForTestsOnly(testFs, testFs, "dir", "link")
if err == nil {
t.Log("running with symlink check")
testcases = append(testcases, struct {

View File

@ -61,8 +61,8 @@ type FolderConfiguration struct {
DisableFsync bool `xml:"disableFsync" json:"disableFsync"`
BlockPullOrder BlockPullOrder `xml:"blockPullOrder" json:"blockPullOrder"`
CopyRangeMethod fs.CopyRangeMethod `xml:"copyRangeMethod" json:"copyRangeMethod" default:"standard"`
CaseSensitiveFS bool `xml:"caseSensitiveFS" json:"caseSensitiveFS"`
cachedFilesystem fs.Filesystem
cachedModTimeWindow time.Duration
DeprecatedReadOnly bool `xml:"ro,attr,omitempty" json:"-"`
@ -101,11 +101,11 @@ func (f FolderConfiguration) Copy() FolderConfiguration {
func (f FolderConfiguration) Filesystem() fs.Filesystem {
// This is intentionally not a pointer method, because things like
// cfg.Folders["default"].Filesystem() should be valid.
if f.cachedFilesystem == nil {
l.Infoln("bug: uncached filesystem call (should only happen in tests)")
return fs.NewFilesystem(f.FilesystemType, f.Path)
filesystem := fs.NewFilesystem(f.FilesystemType, f.Path)
if !f.CaseSensitiveFS {
filesystem = fs.NewCaseFilesystem(filesystem)
}
return f.cachedFilesystem
return filesystem
}
func (f FolderConfiguration) ModTimeWindow() time.Duration {
@ -210,8 +210,6 @@ func (f *FolderConfiguration) DeviceIDs() []protocol.DeviceID {
}
func (f *FolderConfiguration) prepare() {
f.cachedFilesystem = fs.NewFilesystem(f.FilesystemType, f.Path)
if f.RescanIntervalS > MaxRescanIntervalS {
f.RescanIntervalS = MaxRescanIntervalS
} else if f.RescanIntervalS < 0 {
@ -263,7 +261,6 @@ func (f FolderConfiguration) RequiresRestartOnly() FolderConfiguration {
// Manual handling for things that are not taken care of by the tag
// copier, yet should not cause a restart.
copy.cachedFilesystem = nil
blank := FolderConfiguration{}
util.CopyMatchingTag(&blank, &copy, "restart", func(v string) bool {

View File

@ -0,0 +1,13 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build !windows
package fs
func newBasicRealCaser(fs Filesystem) realCaser {
return newDefaultRealCaser(fs)
}

View File

@ -0,0 +1,58 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build windows
package fs
import (
"path/filepath"
"strings"
"syscall"
)
type basicRealCaserWindows struct {
uri string
}
func newBasicRealCaser(fs Filesystem) realCaser {
return &basicRealCaserWindows{fs.URI()}
}
// RealCase returns the correct case for the given name, which is a relative
// path below root, as it exists on disk.
func (r *basicRealCaserWindows) realCase(name string) (string, error) {
if name == "." {
return ".", nil
}
path := r.uri
comps := strings.Split(name, string(PathSeparator))
var err error
for i, comp := range comps {
path = filepath.Join(path, comp)
comps[i], err = r.realCaseBase(path)
if err != nil {
return "", err
}
}
return filepath.Join(comps...), nil
}
func (*basicRealCaserWindows) realCaseBase(path string) (string, error) {
p, err := syscall.UTF16PtrFromString(fixLongPath(path))
if err != nil {
return "", err
}
var fd syscall.Win32finddata
h, err := syscall.FindFirstFile(p, &fd)
if err != nil {
return "", err
}
syscall.FindClose(h)
return syscall.UTF16ToString(fd.FileName[:]), nil
}
func (r *basicRealCaserWindows) dropCache() {}

448
lib/fs/casefs.go Normal file
View File

@ -0,0 +1,448 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package fs
import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
"sync"
"time"
)
// Both values were chosen by magic.
const (
caseCacheTimeout = time.Second
// When the number of names (all lengths of []string from DirNames)
// exceeds this, we drop the cache.
caseMaxCachedNames = 1 << 20
)
type ErrCaseConflict struct {
given, real string
}
func (e *ErrCaseConflict) Error() string {
return fmt.Sprintf(`given name "%v" differs from name in filesystem "%v"`, e.given, e.real)
}
func IsErrCaseConflict(err error) bool {
e := &ErrCaseConflict{}
return errors.As(err, &e)
}
type realCaser interface {
realCase(name string) (string, error)
dropCache()
}
type fskey struct {
fstype FilesystemType
uri string
}
var (
caseFilesystems = make(map[fskey]Filesystem)
caseFilesystemsMut sync.Mutex
)
// caseFilesystem is a BasicFilesystem with additional checks to make a
// potentially case insensitive underlying FS behave like it's case-sensitive.
type caseFilesystem struct {
Filesystem
realCaser
}
// NewCaseFilesystem ensures that the given, potentially case-insensitive filesystem
// behaves like a case-sensitive filesystem. Meaning that it takes into account
// the real casing of a path and returns ErrCaseConflict if the given path differs
// from the real path. It is safe to use with any filesystem, i.e. also a
// case-sensitive one. However it will add some overhead and thus shouldn't be
// used if the filesystem is known to already behave case-sensitively.
func NewCaseFilesystem(fs Filesystem) Filesystem {
caseFilesystemsMut.Lock()
defer caseFilesystemsMut.Unlock()
k := fskey{fs.Type(), fs.URI()}
if caseFs, ok := caseFilesystems[k]; ok {
return caseFs
}
caseFs := &caseFilesystem{
Filesystem: fs,
}
switch k.fstype {
case FilesystemTypeBasic:
caseFs.realCaser = newBasicRealCaser(fs)
default:
caseFs.realCaser = newDefaultRealCaser(fs)
}
caseFilesystems[k] = caseFs
return caseFs
}
func (f *caseFilesystem) Chmod(name string, mode FileMode) error {
if err := f.checkCase(name); err != nil {
return err
}
return f.Filesystem.Chmod(name, mode)
}
func (f *caseFilesystem) Lchown(name string, uid, gid int) error {
if err := f.checkCase(name); err != nil {
return err
}
return f.Filesystem.Lchown(name, uid, gid)
}
func (f *caseFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error {
if err := f.checkCase(name); err != nil {
return err
}
return f.Filesystem.Chtimes(name, atime, mtime)
}
func (f *caseFilesystem) Mkdir(name string, perm FileMode) error {
if err := f.checkCase(name); err != nil {
return err
}
if err := f.Filesystem.Mkdir(name, perm); err != nil {
return err
}
f.dropCache()
return nil
}
func (f *caseFilesystem) MkdirAll(path string, perm FileMode) error {
if err := f.checkCase(path); err != nil {
return err
}
if err := f.Filesystem.MkdirAll(path, perm); err != nil {
return err
}
f.dropCache()
return nil
}
func (f *caseFilesystem) Lstat(name string) (FileInfo, error) {
var err error
if name, err = Canonicalize(name); err != nil {
return nil, err
}
stat, err := f.Filesystem.Lstat(name)
if err != nil {
return nil, err
}
if err = f.checkCaseExisting(name); err != nil {
return nil, err
}
return stat, nil
}
func (f *caseFilesystem) Remove(name string) error {
if err := f.checkCase(name); err != nil {
return err
}
if err := f.Filesystem.Remove(name); err != nil {
return err
}
f.dropCache()
return nil
}
func (f *caseFilesystem) RemoveAll(name string) error {
if err := f.checkCase(name); err != nil {
return err
}
if err := f.Filesystem.RemoveAll(name); err != nil {
return err
}
f.dropCache()
return nil
}
func (f *caseFilesystem) Rename(oldpath, newpath string) error {
if err := f.checkCase(oldpath); err != nil {
return err
}
if err := f.Filesystem.Rename(oldpath, newpath); err != nil {
return err
}
f.dropCache()
return nil
}
func (f *caseFilesystem) Stat(name string) (FileInfo, error) {
var err error
if name, err = Canonicalize(name); err != nil {
return nil, err
}
stat, err := f.Filesystem.Stat(name)
if err != nil {
return nil, err
}
if err = f.checkCaseExisting(name); err != nil {
return nil, err
}
return stat, nil
}
func (f *caseFilesystem) DirNames(name string) ([]string, error) {
if err := f.checkCase(name); err != nil {
return nil, err
}
return f.Filesystem.DirNames(name)
}
func (f *caseFilesystem) Open(name string) (File, error) {
if err := f.checkCase(name); err != nil {
return nil, err
}
return f.Filesystem.Open(name)
}
func (f *caseFilesystem) OpenFile(name string, flags int, mode FileMode) (File, error) {
if err := f.checkCase(name); err != nil {
return nil, err
}
file, err := f.Filesystem.OpenFile(name, flags, mode)
if err != nil {
return nil, err
}
f.dropCache()
return file, nil
}
func (f *caseFilesystem) ReadSymlink(name string) (string, error) {
if err := f.checkCase(name); err != nil {
return "", err
}
return f.Filesystem.ReadSymlink(name)
}
func (f *caseFilesystem) Create(name string) (File, error) {
if err := f.checkCase(name); err != nil {
return nil, err
}
file, err := f.Filesystem.Create(name)
if err != nil {
return nil, err
}
f.dropCache()
return file, nil
}
func (f *caseFilesystem) CreateSymlink(target, name string) error {
if err := f.checkCase(name); err != nil {
return err
}
if err := f.Filesystem.CreateSymlink(target, name); err != nil {
return err
}
f.dropCache()
return nil
}
func (f *caseFilesystem) Walk(root string, walkFn WalkFunc) error {
// Walking the filesystem is likely (in Syncthing's case certainly) done
// to pick up external changes, for which caching is undesirable.
f.dropCache()
if err := f.checkCase(root); err != nil {
return err
}
return f.Filesystem.Walk(root, walkFn)
}
func (f *caseFilesystem) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, <-chan error, error) {
if err := f.checkCase(path); err != nil {
return nil, nil, err
}
return f.Filesystem.Watch(path, ignore, ctx, ignorePerms)
}
func (f *caseFilesystem) Hide(name string) error {
if err := f.checkCase(name); err != nil {
return err
}
return f.Filesystem.Hide(name)
}
func (f *caseFilesystem) Unhide(name string) error {
if err := f.checkCase(name); err != nil {
return err
}
return f.Filesystem.Unhide(name)
}
func (f *caseFilesystem) checkCase(name string) error {
var err error
if name, err = Canonicalize(name); err != nil {
return err
}
// Stat is necessary for case sensitive FS, as it's then not a conflict
// if name is e.g. "foo" and on dir there is "Foo".
if _, err := f.Filesystem.Lstat(name); err != nil {
if IsNotExist(err) {
return nil
}
return err
}
return f.checkCaseExisting(name)
}
// checkCaseExisting must only be called after successfully canonicalizing and
// stating the file.
func (f *caseFilesystem) checkCaseExisting(name string) error {
realName, err := f.realCase(name)
if IsNotExist(err) {
// It did exist just before -> cache is outdated, try again
f.dropCache()
realName, err = f.realCase(name)
}
if err != nil {
return err
}
if realName != name {
return &ErrCaseConflict{name, realName}
}
return nil
}
type defaultRealCaser struct {
fs Filesystem
root *caseNode
count int
timer *time.Timer
timerStop chan struct{}
mut sync.RWMutex
}
func newDefaultRealCaser(fs Filesystem) *defaultRealCaser {
caser := &defaultRealCaser{
fs: fs,
root: &caseNode{name: "."},
timer: time.NewTimer(0),
}
<-caser.timer.C
return caser
}
func (r *defaultRealCaser) realCase(name string) (string, error) {
out := "."
if name == out {
return out, nil
}
r.mut.Lock()
defer func() {
if r.count > caseMaxCachedNames {
select {
case r.timerStop <- struct{}{}:
default:
}
r.dropCacheLocked()
}
r.mut.Unlock()
}()
node := r.root
for _, comp := range strings.Split(name, string(PathSeparator)) {
if node.dirNames == nil {
// Haven't called DirNames yet
var err error
node.dirNames, err = r.fs.DirNames(out)
if err != nil {
return "", err
}
node.dirNamesLower = make([]string, len(node.dirNames))
for i, n := range node.dirNames {
node.dirNamesLower[i] = UnicodeLowercase(n)
}
node.children = make(map[string]*caseNode)
node.results = make(map[string]*caseNode)
r.count += len(node.dirNames)
} else if child, ok := node.results[comp]; ok {
// Check if this exact name has been queried before to shortcut
node = child
out = filepath.Join(out, child.name)
continue
}
// Actually loop dirNames to search for a match
n, err := findCaseInsensitiveMatch(comp, node.dirNames, node.dirNamesLower)
if err != nil {
return "", err
}
child, ok := node.children[n]
if !ok {
child = &caseNode{name: n}
}
node.results[comp] = child
node.children[n] = child
node = child
out = filepath.Join(out, n)
}
return out, nil
}
func (r *defaultRealCaser) startCaseResetTimerLocked() {
r.timerStop = make(chan struct{})
r.timer.Reset(caseCacheTimeout)
go func() {
select {
case <-r.timer.C:
r.dropCache()
case <-r.timerStop:
if !r.timer.Stop() {
<-r.timer.C
}
r.mut.Lock()
r.timerStop = nil
r.mut.Unlock()
}
}()
}
func (r *defaultRealCaser) dropCache() {
r.mut.Lock()
r.dropCacheLocked()
r.mut.Unlock()
}
func (r *defaultRealCaser) dropCacheLocked() {
r.root = &caseNode{name: "."}
r.count = 0
}
// Both name and the key to children are "Real", case resolved names of the path
// component this node represents (i.e. containing no path separator).
// The key to results is also a path component, but as given to RealCase, not
// case resolved.
type caseNode struct {
name string
dirNames []string
dirNamesLower []string
children map[string]*caseNode
results map[string]*caseNode
}
func findCaseInsensitiveMatch(name string, names, namesLower []string) (string, error) {
lower := UnicodeLowercase(name)
candidate := ""
for i, n := range names {
if n == name {
return n, nil
}
if candidate == "" && namesLower[i] == lower {
candidate = n
}
}
if candidate == "" {
return "", ErrNotExist
}
return candidate, nil
}

279
lib/fs/casefs_test.go Normal file
View File

@ -0,0 +1,279 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package fs
import (
"errors"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
)
func TestRealCase(t *testing.T) {
// Verify realCase lookups on various underlying filesystems.
t.Run("fake-sensitive", func(t *testing.T) {
testRealCase(t, newFakeFilesystem(t.Name()))
})
t.Run("fake-insensitive", func(t *testing.T) {
testRealCase(t, newFakeFilesystem(t.Name()+"?insens=true"))
})
t.Run("actual", func(t *testing.T) {
fsys, tmpDir := setup(t)
defer os.RemoveAll(tmpDir)
testRealCase(t, fsys)
})
}
func testRealCase(t *testing.T, fsys Filesystem) {
testFs := NewCaseFilesystem(fsys).(*caseFilesystem)
comps := []string{"Foo", "bar", "BAZ", "bAs"}
path := filepath.Join(comps...)
testFs.MkdirAll(filepath.Join(comps[:len(comps)-1]...), 0777)
fd, err := testFs.Create(path)
if err != nil {
t.Fatal(err)
}
fd.Close()
for i, tc := range []struct {
in string
len int
}{
{path, 4},
{strings.ToLower(path), 4},
{strings.ToUpper(path), 4},
{"foo", 1},
{"FOO", 1},
{"foO", 1},
{filepath.Join("Foo", "bar"), 2},
{filepath.Join("Foo", "bAr"), 2},
{filepath.Join("FoO", "bar"), 2},
{filepath.Join("foo", "bar", "BAZ"), 3},
{filepath.Join("Foo", "bar", "bAz"), 3},
{filepath.Join("foo", "bar", "BAZ"), 3}, // Repeat on purpose
} {
out, err := testFs.realCase(tc.in)
if err != nil {
t.Error(err)
} else if exp := filepath.Join(comps[:tc.len]...); out != exp {
t.Errorf("tc %v: Expected %v, got %v", i, exp, out)
}
}
}
func TestRealCaseSensitive(t *testing.T) {
// Verify that realCase returns the best on-disk case for case sensitive
// systems. Test is skipped if the underlying fs is insensitive.
t.Run("fake-sensitive", func(t *testing.T) {
testRealCaseSensitive(t, newFakeFilesystem(t.Name()))
})
t.Run("actual", func(t *testing.T) {
fsys, tmpDir := setup(t)
defer os.RemoveAll(tmpDir)
testRealCaseSensitive(t, fsys)
})
}
func testRealCaseSensitive(t *testing.T, fsys Filesystem) {
testFs := NewCaseFilesystem(fsys).(*caseFilesystem)
names := make([]string, 2)
names[0] = "foo"
names[1] = strings.ToUpper(names[0])
for _, n := range names {
if err := testFs.MkdirAll(n, 0777); err != nil {
if IsErrCaseConflict(err) {
t.Skip("Filesystem is case-insensitive")
}
t.Fatal(err)
}
}
for _, n := range names {
if rn, err := testFs.realCase(n); err != nil {
t.Error(err)
} else if rn != n {
t.Errorf("Got %v, expected %v", rn, n)
}
}
}
func TestCaseFSStat(t *testing.T) {
// Verify that a Stat() lookup behaves in a case sensitive manner
// regardless of the underlying fs.
t.Run("fake-sensitive", func(t *testing.T) {
testCaseFSStat(t, newFakeFilesystem(t.Name()))
})
t.Run("fake-insensitive", func(t *testing.T) {
testCaseFSStat(t, newFakeFilesystem(t.Name()+"?insens=true"))
})
t.Run("actual", func(t *testing.T) {
fsys, tmpDir := setup(t)
defer os.RemoveAll(tmpDir)
testCaseFSStat(t, fsys)
})
}
func testCaseFSStat(t *testing.T, fsys Filesystem) {
fd, err := fsys.Create("foo")
if err != nil {
t.Fatal(err)
}
fd.Close()
// Check if the underlying fs is sensitive or not
sensitive := true
if _, err = fsys.Stat("FOO"); err == nil {
sensitive = false
}
testFs := NewCaseFilesystem(fsys)
_, err = testFs.Stat("FOO")
if sensitive {
if IsNotExist(err) {
t.Log("pass: case sensitive underlying fs")
} else {
t.Error("expected NotExist, not", err, "for sensitive fs")
}
} else if IsErrCaseConflict(err) {
t.Log("pass: case insensitive underlying fs")
} else {
t.Error("expected ErrCaseConflict, not", err, "for insensitive fs")
}
}
func BenchmarkWalkCaseFakeFS10k(b *testing.B) {
fsys, paths, err := fakefsForBenchmark(10_000, 0)
if err != nil {
b.Fatal(err)
}
slowsys, paths, err := fakefsForBenchmark(10_000, 100*time.Microsecond)
if err != nil {
b.Fatal(err)
}
b.Run("raw-fastfs", func(b *testing.B) {
benchmarkWalkFakeFS(b, fsys, paths)
b.ReportAllocs()
})
b.Run("case-fastfs", func(b *testing.B) {
benchmarkWalkFakeFS(b, NewCaseFilesystem(fsys), paths)
b.ReportAllocs()
})
b.Run("raw-slowfs", func(b *testing.B) {
benchmarkWalkFakeFS(b, slowsys, paths)
b.ReportAllocs()
})
b.Run("case-slowfs", func(b *testing.B) {
benchmarkWalkFakeFS(b, NewCaseFilesystem(slowsys), paths)
b.ReportAllocs()
})
}
func benchmarkWalkFakeFS(b *testing.B, fsys Filesystem, paths []string) {
// Simulate a scanner pass over the filesystem. First walk it to
// discover all names, then stat each name individually to check if it's
// been deleted or not (pretending that they all existed in the
// database).
var ms0 runtime.MemStats
runtime.ReadMemStats(&ms0)
t0 := time.Now()
for i := 0; i < b.N; i++ {
if err := doubleWalkFS(fsys, paths); err != nil {
b.Fatal(err)
}
}
t1 := time.Now()
var ms1 runtime.MemStats
runtime.ReadMemStats(&ms1)
// We add metrics per path entry
b.ReportMetric(float64(t1.Sub(t0))/float64(b.N)/float64(len(paths)), "ns/entry")
b.ReportMetric(float64(ms1.Mallocs-ms0.Mallocs)/float64(b.N)/float64(len(paths)), "allocs/entry")
b.ReportMetric(float64(ms1.TotalAlloc-ms0.TotalAlloc)/float64(b.N)/float64(len(paths)), "B/entry")
}
func TestStressCaseFS(t *testing.T) {
// Exercise a bunch of paralell operations for stressing out race
// conditions in the realnamer cache etc.
const limit = 10 * time.Second
if testing.Short() {
t.Skip("long test")
}
fsys, paths, err := fakefsForBenchmark(10_000, 0)
if err != nil {
t.Fatal(err)
}
for i := 0; i < runtime.NumCPU()/2+1; i++ {
t.Run(fmt.Sprintf("walker-%d", i), func(t *testing.T) {
// Walk the filesystem and stat everything
t.Parallel()
t0 := time.Now()
for time.Since(t0) < limit {
if err := doubleWalkFS(fsys, paths); err != nil {
t.Fatal(err)
}
}
})
t.Run(fmt.Sprintf("toucher-%d", i), func(t *testing.T) {
// Touch all the things
t.Parallel()
t0 := time.Now()
for time.Since(t0) < limit {
for _, p := range paths {
now := time.Now()
if err := fsys.Chtimes(p, now, now); err != nil {
t.Fatal(err)
}
}
}
})
}
}
func doubleWalkFS(fsys Filesystem, paths []string) error {
if err := fsys.Walk("/", func(path string, info FileInfo, err error) error {
return err
}); err != nil {
return err
}
for _, p := range paths {
if _, err := fsys.Lstat(p); err != nil {
return err
}
}
return nil
}
func fakefsForBenchmark(nfiles int, latency time.Duration) (Filesystem, []string, error) {
fsys := NewFilesystem(FilesystemTypeFake, fmt.Sprintf("fakefsForBenchmark?files=%d&insens=true&latency=%s", nfiles, latency))
var paths []string
if err := fsys.Walk("/", func(path string, info FileInfo, err error) error {
paths = append(paths, path)
return err
}); err != nil {
return nil, nil, err
}
if len(paths) < nfiles {
return nil, nil, errors.New("didn't find enough stuff")
}
return fsys, paths, nil
}

View File

@ -0,0 +1,47 @@
// Copyright (C) 2017 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build !windows
package fs
import (
"os"
"path/filepath"
)
// DebugSymlinkForTestsOnly is not and should not be used in Syncthing code,
// hence the cumbersome name to make it obvious if this ever leaks. Its
// reason for existence is the Windows version, which allows creating
// symlinks when non-elevated.
func DebugSymlinkForTestsOnly(oldFs, newFs Filesystem, oldname, newname string) error {
if caseFs, ok := unwrapFilesystem(newFs).(*caseFilesystem); ok {
if err := caseFs.checkCase(newname); err != nil {
return err
}
caseFs.dropCache()
}
if err := os.Symlink(filepath.Join(oldFs.URI(), oldname), filepath.Join(newFs.URI(), newname)); err != nil {
return err
}
return nil
}
// unwrapFilesystem removes "wrapping" filesystems to expose the underlying filesystem.
func unwrapFilesystem(fs Filesystem) Filesystem {
for {
switch sfs := fs.(type) {
case *logFilesystem:
fs = sfs.Filesystem
case *walkFilesystem:
fs = sfs.Filesystem
case *MtimeFS:
fs = sfs.Filesystem
default:
return sfs
}
}
}

View File

@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package osutil
package fs
import (
"os"
@ -17,7 +17,10 @@ import (
// This is not and should not be used in Syncthing code, hence the
// cumbersome name to make it obvious if this ever leaks. Nonetheless it's
// useful in tests.
func DebugSymlinkForTestsOnly(oldname, newname string) error {
func DebugSymlinkForTestsOnly(oldFs, newFS Filesystem, oldname, newname string) error {
oldname = filepath.Join(oldFs.URI(), oldname)
newname = filepath.Join(newFS.URI(), newname)
// CreateSymbolicLink is not supported before Windows Vista
if syscall.LoadCreateSymbolicLink() != nil {
return &os.LinkError{"symlink", oldname, newname, syscall.EWINDOWS}

View File

@ -48,14 +48,17 @@ const randomBlockShift = 14 // 128k
// sizeavg=n to set the average size of random files, in bytes (default 1<<20)
// seed=n to set the initial random seed (default 0)
// insens=b "true" makes filesystem case-insensitive Windows- or OSX-style (default false)
// latency=d to set the amount of time each "disk" operation takes, where d is time.ParseDuration format
//
// - Two fakefs:s pointing at the same root path see the same files.
//
type fakefs struct {
uri string
mut sync.Mutex
root *fakeEntry
insens bool
withContent bool
latency time.Duration
}
var (
@ -63,23 +66,25 @@ var (
fakefsFs = make(map[string]*fakefs)
)
func newFakeFilesystem(root string) *fakefs {
func newFakeFilesystem(rootURI string) *fakefs {
fakefsMut.Lock()
defer fakefsMut.Unlock()
root := rootURI
var params url.Values
uri, err := url.Parse(root)
uri, err := url.Parse(rootURI)
if err == nil {
root = uri.Path
params = uri.Query()
}
if fs, ok := fakefsFs[root]; ok {
if fs, ok := fakefsFs[rootURI]; ok {
// Already have an fs at this path
return fs
}
fs := &fakefs{
uri: "fake://" + rootURI,
root: &fakeEntry{
name: "/",
entryType: fakeEntryTypeDir,
@ -129,6 +134,10 @@ func newFakeFilesystem(root string) *fakefs {
// Also create a default folder marker for good measure
fs.Mkdir(".stfolder", 0700)
// We only set the latency after doing the operations required to create
// the filesystem initially.
fs.latency, _ = time.ParseDuration(params.Get("latency"))
fakefsFs[root] = fs
return fs
}
@ -185,6 +194,7 @@ func (fs *fakefs) entryForName(name string) *fakeEntry {
func (fs *fakefs) Chmod(name string, mode FileMode) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
entry := fs.entryForName(name)
if entry == nil {
return os.ErrNotExist
@ -196,6 +206,7 @@ func (fs *fakefs) Chmod(name string, mode FileMode) error {
func (fs *fakefs) Lchown(name string, uid, gid int) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
entry := fs.entryForName(name)
if entry == nil {
return os.ErrNotExist
@ -208,6 +219,7 @@ func (fs *fakefs) Lchown(name string, uid, gid int) error {
func (fs *fakefs) Chtimes(name string, atime time.Time, mtime time.Time) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
entry := fs.entryForName(name)
if entry == nil {
return os.ErrNotExist
@ -219,6 +231,7 @@ func (fs *fakefs) Chtimes(name string, atime time.Time, mtime time.Time) error {
func (fs *fakefs) create(name string) (*fakeEntry, error) {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
if entry := fs.entryForName(name); entry != nil {
if entry.entryType == fakeEntryTypeDir {
@ -284,6 +297,7 @@ func (fs *fakefs) CreateSymlink(target, name string) error {
func (fs *fakefs) DirNames(name string) ([]string, error) {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
entry := fs.entryForName(name)
if entry == nil {
@ -301,6 +315,7 @@ func (fs *fakefs) DirNames(name string) ([]string, error) {
func (fs *fakefs) Lstat(name string) (FileInfo, error) {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
entry := fs.entryForName(name)
if entry == nil {
@ -318,6 +333,7 @@ func (fs *fakefs) Lstat(name string) (FileInfo, error) {
func (fs *fakefs) Mkdir(name string, perm FileMode) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
dir := filepath.Dir(name)
base := filepath.Base(name)
@ -348,6 +364,10 @@ func (fs *fakefs) Mkdir(name string, perm FileMode) error {
}
func (fs *fakefs) MkdirAll(name string, perm FileMode) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
name = filepath.ToSlash(name)
name = strings.Trim(name, "/")
comps := strings.Split(name, "/")
@ -382,6 +402,7 @@ func (fs *fakefs) MkdirAll(name string, perm FileMode) error {
func (fs *fakefs) Open(name string) (File, error) {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
entry := fs.entryForName(name)
if entry == nil || entry.entryType != fakeEntryTypeFile {
@ -401,6 +422,7 @@ func (fs *fakefs) OpenFile(name string, flags int, mode FileMode) (File, error)
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
dir := filepath.Dir(name)
base := filepath.Base(name)
@ -438,6 +460,7 @@ func (fs *fakefs) OpenFile(name string, flags int, mode FileMode) (File, error)
func (fs *fakefs) ReadSymlink(name string) (string, error) {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
entry := fs.entryForName(name)
if entry == nil {
@ -451,6 +474,7 @@ func (fs *fakefs) ReadSymlink(name string) (string, error) {
func (fs *fakefs) Remove(name string) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
if fs.insens {
name = UnicodeLowercase(name)
@ -472,6 +496,7 @@ func (fs *fakefs) Remove(name string) error {
func (fs *fakefs) RemoveAll(name string) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
if fs.insens {
name = UnicodeLowercase(name)
@ -491,6 +516,7 @@ func (fs *fakefs) RemoveAll(name string) error {
func (fs *fakefs) Rename(oldname, newname string) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
oldKey := filepath.Base(oldname)
newKey := filepath.Base(newname)
@ -578,7 +604,7 @@ func (fs *fakefs) Type() FilesystemType {
}
func (fs *fakefs) URI() string {
return "fake://" + fs.root.name
return fs.uri
}
func (fs *fakefs) SameFile(fi1, fi2 FileInfo) bool {

View File

@ -129,8 +129,8 @@ type sendReceiveFolder struct {
blockPullReorderer blockPullReorderer
writeLimiter *byteSemaphore
pullErrors map[string]string // errors for most recent/current iteration
oldPullErrors map[string]string // errors from previous iterations for log filtering only
pullErrors map[string]string // actual exposed pull errors
tempPullErrors map[string]string // pull errors that might be just transient
pullErrorsMut sync.Mutex
}
@ -192,6 +192,10 @@ func (f *sendReceiveFolder) pull() bool {
changed := 0
f.pullErrorsMut.Lock()
f.pullErrors = nil
f.pullErrorsMut.Unlock()
for tries := 0; tries < maxPullerIterations; tries++ {
select {
case <-f.ctx.Done():
@ -216,8 +220,14 @@ func (f *sendReceiveFolder) pull() bool {
}
f.pullErrorsMut.Lock()
f.pullErrors = f.tempPullErrors
f.tempPullErrors = nil
for path, err := range f.pullErrors {
l.Infof("Puller (folder %s, item %q): %v", f.Description(), path, err)
}
pullErrNum := len(f.pullErrors)
f.pullErrorsMut.Unlock()
if pullErrNum > 0 {
l.Infof("%v: Failed to sync %v items", f.Description(), pullErrNum)
f.evLogger.Log(events.FolderErrors, map[string]interface{}{
@ -235,8 +245,7 @@ func (f *sendReceiveFolder) pull() bool {
// flagged as needed in the folder.
func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
f.pullErrorsMut.Lock()
f.oldPullErrors = f.pullErrors
f.pullErrors = make(map[string]string)
f.tempPullErrors = make(map[string]string)
f.pullErrorsMut.Unlock()
snap := f.fset.Snapshot()
@ -306,10 +315,6 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
close(dbUpdateChan)
updateWg.Wait()
f.pullErrorsMut.Lock()
f.oldPullErrors = nil
f.pullErrorsMut.Unlock()
f.queue.Reset()
return changed
@ -739,7 +744,11 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, snap *db.Snaps
}
// There is already something under that name, we need to handle that.
if info, err := f.fs.Lstat(file.Name); err == nil {
switch info, err := f.fs.Lstat(file.Name); {
case err != nil && !fs.IsNotExist(err):
f.newPullError(file.Name, errors.Wrap(err, "checking for existing symlink"))
return
case err == nil:
// Check that it is what we have in the database.
curFile, hasCurFile := f.model.CurrentFolderFile(f.folderID, file.Name)
if err := f.scanIfItemChanged(file.Name, info, curFile, hasCurFile, scanChan); err != nil {
@ -1783,7 +1792,7 @@ func (f *sendReceiveFolder) newPullError(path string, err error) {
// We might get more than one error report for a file (i.e. error on
// Write() followed by Close()); we keep the first error as that is
// probably closer to the root cause.
if _, ok := f.pullErrors[path]; ok {
if _, ok := f.tempPullErrors[path]; ok {
return
}
@ -1791,15 +1800,9 @@ func (f *sendReceiveFolder) newPullError(path string, err error) {
// Use "syncing" as opposed to "pulling" as the latter might be used
// for errors occurring specificly in the puller routine.
errStr := fmt.Sprintln("syncing:", err)
f.pullErrors[path] = errStr
f.tempPullErrors[path] = errStr
if oldErr, ok := f.oldPullErrors[path]; ok && oldErr == errStr {
l.Debugf("Repeat error on puller (folder %s, item %q): %v", f.Description(), path, err)
delete(f.oldPullErrors, path) // Potential repeats are now caught by f.pullErrors itself
return
}
l.Infof("Puller (folder %s, item %q): %v", f.Description(), path, err)
l.Debugf("%v new error for %v: %v", f, path, err)
}
func (f *sendReceiveFolder) Errors() []FileError {

View File

@ -10,11 +10,13 @@ import (
"bytes"
"context"
"crypto/rand"
"errors"
"io"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
@ -95,6 +97,7 @@ func setupSendReceiveFolder(files ...protocol.FileInfo) (*model, *sendReceiveFol
model.Supervisor.Stop()
f := model.folderRunners[fcfg.ID].(*sendReceiveFolder)
f.pullErrors = make(map[string]string)
f.tempPullErrors = make(map[string]string)
f.ctx = context.Background()
// Update index
@ -983,7 +986,7 @@ func TestDeleteBehindSymlink(t *testing.T) {
must(t, osutil.RenameOrCopy(fs.CopyRangeMethodStandard, ffs, destFs, file, "file"))
must(t, ffs.RemoveAll(link))
if err := osutil.DebugSymlinkForTestsOnly(destFs.URI(), filepath.Join(ffs.URI(), link)); err != nil {
if err := fs.DebugSymlinkForTestsOnly(destFs, ffs, "", link); err != nil {
if runtime.GOOS == "windows" {
// Probably we require permissions we don't have.
t.Skip("Need admin permissions or developer mode to run symlink test on Windows: " + err.Error())
@ -1087,6 +1090,122 @@ func TestPullDeleteUnscannedDir(t *testing.T) {
}
}
func TestPullCaseOnlyPerformFinish(t *testing.T) {
m, f := setupSendReceiveFolder()
defer cleanupSRFolder(f, m)
ffs := f.Filesystem()
name := "foo"
contents := []byte("contents")
must(t, writeFile(ffs, name, contents, 0644))
must(t, f.scanSubdirs(nil))
var cur protocol.FileInfo
hasCur := false
snap := dbSnapshot(t, m, f.ID)
defer snap.Release()
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
if hasCur {
t.Fatal("got more than one file")
}
cur = i.(protocol.FileInfo)
hasCur = true
return true
})
if !hasCur {
t.Fatal("file is missing")
}
remote := *(&cur)
remote.Version = protocol.Vector{}.Update(device1.Short())
remote.Name = strings.ToUpper(cur.Name)
temp := fs.TempName(remote.Name)
must(t, writeFile(ffs, temp, contents, 0644))
scanChan := make(chan string, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
err := f.performFinish(remote, cur, hasCur, temp, snap, dbUpdateChan, scanChan)
select {
case <-dbUpdateChan: // boring case sensitive filesystem
return
case <-scanChan:
t.Error("no need to scan anything here")
default:
}
var caseErr *fs.ErrCaseConflict
if !errors.As(err, &caseErr) {
t.Error("Expected case conflict error, got", err)
}
}
func TestPullCaseOnlyDir(t *testing.T) {
testPullCaseOnlyDirOrSymlink(t, true)
}
func TestPullCaseOnlySymlink(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("symlinks not supported on windows")
}
testPullCaseOnlyDirOrSymlink(t, false)
}
func testPullCaseOnlyDirOrSymlink(t *testing.T, dir bool) {
m, f := setupSendReceiveFolder()
defer cleanupSRFolder(f, m)
ffs := f.Filesystem()
name := "foo"
if dir {
must(t, ffs.Mkdir(name, 0777))
} else {
must(t, ffs.CreateSymlink("target", name))
}
must(t, f.scanSubdirs(nil))
var cur protocol.FileInfo
hasCur := false
snap := dbSnapshot(t, m, f.ID)
defer snap.Release()
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
if hasCur {
t.Fatal("got more than one file")
}
cur = i.(protocol.FileInfo)
hasCur = true
return true
})
if !hasCur {
t.Fatal("file is missing")
}
scanChan := make(chan string, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
remote := *(&cur)
remote.Version = protocol.Vector{}.Update(device1.Short())
remote.Name = strings.ToUpper(cur.Name)
if dir {
f.handleDir(remote, snap, dbUpdateChan, scanChan)
} else {
f.handleSymlink(remote, snap, dbUpdateChan, scanChan)
}
select {
case <-dbUpdateChan: // boring case sensitive filesystem
return
case <-scanChan:
t.Error("no need to scan anything here")
default:
}
if errStr, ok := f.tempPullErrors[remote.Name]; !ok {
t.Error("missing error for", remote.Name)
} else if !strings.Contains(errStr, "differs from name") {
t.Error("unexpected error", errStr, "for", remote.Name)
}
}
func cleanupSharedPullerState(s *sharedPullerState) {
s.mut.Lock()
defer s.mut.Unlock()

View File

@ -12,6 +12,7 @@ import (
"testing"
"github.com/d4l3k/messagediff"
"github.com/syncthing/syncthing/lib/config"
)

View File

@ -270,17 +270,15 @@ func BenchmarkRequestOut(b *testing.B) {
}
func BenchmarkRequestInSingleFile(b *testing.B) {
testOs := &fatalOs{b}
m := setupModel(defaultCfgWrapper)
defer cleanupModel(m)
buf := make([]byte, 128<<10)
rand.Read(buf)
testOs.RemoveAll("testdata/request")
defer testOs.RemoveAll("testdata/request")
testOs.MkdirAll("testdata/request/for/a/file/in/a/couple/of/dirs", 0755)
ioutil.WriteFile("testdata/request/for/a/file/in/a/couple/of/dirs/128k", buf, 0644)
mustRemove(b, defaultFs.RemoveAll("request"))
defer func() { mustRemove(b, defaultFs.RemoveAll("request")) }()
must(b, defaultFs.MkdirAll("request/for/a/file/in/a/couple/of/dirs", 0755))
writeFile(defaultFs, "request/for/a/file/in/a/couple/of/dirs/128k", buf, 0644)
b.ResetTimer()
@ -294,13 +292,11 @@ func BenchmarkRequestInSingleFile(b *testing.B) {
}
func TestDeviceRename(t *testing.T) {
testOs := &fatalOs{t}
hello := protocol.HelloResult{
ClientName: "syncthing",
ClientVersion: "v0.9.4",
}
defer testOs.Remove("testdata/tmpconfig.xml")
defer func() { mustRemove(t, defaultFs.Remove("tmpconfig.xml")) }()
rawCfg := config.New(device1)
rawCfg.Devices = []config.DeviceConfiguration{
@ -1447,12 +1443,10 @@ func changeIgnores(t *testing.T, m *model, expected []string) {
}
func TestIgnores(t *testing.T) {
testOs := &fatalOs{t}
// Assure a clean start state
testOs.RemoveAll(filepath.Join("testdata", config.DefaultMarkerName))
testOs.MkdirAll(filepath.Join("testdata", config.DefaultMarkerName), 0644)
ioutil.WriteFile("testdata/.stignore", []byte(".*\nquux\n"), 0644)
mustRemove(t, defaultFs.RemoveAll(config.DefaultMarkerName))
mustRemove(t, defaultFs.MkdirAll(config.DefaultMarkerName, 0644))
writeFile(defaultFs, ".stignore", []byte(".*\nquux\n"), 0644)
m := setupModel(defaultCfgWrapper)
defer cleanupModel(m)
@ -1504,18 +1498,16 @@ func TestIgnores(t *testing.T) {
// Make sure no .stignore file is considered valid
defer func() {
testOs.Rename("testdata/.stignore.bak", "testdata/.stignore")
must(t, defaultFs.Rename(".stignore.bak", ".stignore"))
}()
testOs.Rename("testdata/.stignore", "testdata/.stignore.bak")
must(t, defaultFs.Rename(".stignore", ".stignore.bak"))
changeIgnores(t, m, []string{})
}
func TestEmptyIgnores(t *testing.T) {
testOs := &fatalOs{t}
// Assure a clean start state
testOs.RemoveAll(filepath.Join("testdata", config.DefaultMarkerName))
testOs.MkdirAll(filepath.Join("testdata", config.DefaultMarkerName), 0644)
mustRemove(t, defaultFs.RemoveAll(config.DefaultMarkerName))
must(t, defaultFs.MkdirAll(config.DefaultMarkerName, 0644))
m := setupModel(defaultCfgWrapper)
defer cleanupModel(m)
@ -2117,14 +2109,14 @@ func benchmarkTree(b *testing.B, n1, n2 int) {
}
func TestIssue3028(t *testing.T) {
testOs := &fatalOs{t}
// Create two files that we'll delete, one with a name that is a prefix of the other.
must(t, ioutil.WriteFile("testdata/testrm", []byte("Hello"), 0644))
defer testOs.Remove("testdata/testrm")
must(t, ioutil.WriteFile("testdata/testrm2", []byte("Hello"), 0644))
defer testOs.Remove("testdata/testrm2")
must(t, writeFile(defaultFs, "testrm", []byte("Hello"), 0644))
must(t, writeFile(defaultFs, "testrm2", []byte("Hello"), 0644))
defer func() {
mustRemove(t, defaultFs.Remove("testrm"))
mustRemove(t, defaultFs.Remove("testrm2"))
}()
// Create a model and default folder
@ -2138,8 +2130,8 @@ func TestIssue3028(t *testing.T) {
// Delete and rescan specifically these two
testOs.Remove("testdata/testrm")
testOs.Remove("testdata/testrm2")
must(t, defaultFs.Remove("testrm"))
must(t, defaultFs.Remove("testrm2"))
m.ScanFolderSubdirs("default", []string{"testrm", "testrm2"})
// Verify that the number of files decreased by two and the number of
@ -2601,7 +2593,7 @@ func TestIssue2571(t *testing.T) {
must(t, testFs.RemoveAll("toLink"))
must(t, osutil.DebugSymlinkForTestsOnly(filepath.Join(testFs.URI(), "linkTarget"), filepath.Join(testFs.URI(), "toLink")))
must(t, fs.DebugSymlinkForTestsOnly(testFs, testFs, "linkTarget", "toLink"))
m.ScanFolder("default")
@ -2718,13 +2710,10 @@ func TestCustomMarkerName(t *testing.T) {
{Name: "dummyfile"},
})
fcfg := config.FolderConfiguration{
ID: "default",
Path: "rwtestfolder",
Type: config.FolderTypeSendReceive,
RescanIntervalS: 1,
MarkerName: "myfile",
}
fcfg := testFolderConfigTmp()
fcfg.ID = "default"
fcfg.RescanIntervalS = 1
fcfg.MarkerName = "myfile"
cfg := createTmpWrapper(config.Configuration{
Folders: []config.FolderConfiguration{fcfg},
Devices: []config.DeviceConfiguration{
@ -2735,13 +2724,12 @@ func TestCustomMarkerName(t *testing.T) {
})
testOs.RemoveAll(fcfg.Path)
defer testOs.RemoveAll(fcfg.Path)
m := newModel(cfg, myID, "syncthing", "dev", ldb, nil)
sub := m.evLogger.Subscribe(events.StateChanged)
defer sub.Unsubscribe()
m.ServeBackground()
defer cleanupModel(m)
defer cleanupModelAndRemoveDir(m, fcfg.Path)
waitForState(t, sub, "default", "folder path missing")
@ -3806,6 +3794,57 @@ func TestBlockListMap(t *testing.T) {
}
}
func TestScanRenameCaseOnly(t *testing.T) {
wcfg, fcfg := tmpDefaultWrapper()
m := setupModel(wcfg)
defer cleanupModel(m)
ffs := fcfg.Filesystem()
name := "foo"
must(t, writeFile(ffs, name, []byte("contents"), 0644))
m.ScanFolders()
snap := dbSnapshot(t, m, fcfg.ID)
defer snap.Release()
found := false
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
if found {
t.Fatal("got more than one file")
}
if i.FileName() != name {
t.Fatalf("got file %v, expected %v", i.FileName(), name)
}
found = true
return true
})
snap.Release()
upper := strings.ToUpper(name)
must(t, ffs.Rename(name, upper))
m.ScanFolders()
snap = dbSnapshot(t, m, fcfg.ID)
defer snap.Release()
found = false
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
if i.FileName() == name {
if i.IsDeleted() {
return true
}
t.Fatal("renamed file not deleted")
}
if i.FileName() != upper {
t.Fatalf("got file %v, expected %v", i.FileName(), upper)
}
if found {
t.Fatal("got more than the expected files")
}
found = true
return true
})
}
func TestConnectionTerminationOnFolderAdd(t *testing.T) {
testConfigChangeClosesConnections(t, false, true, nil, func(cfg config.Wrapper) {
fcfg := testFolderConfigTmp()

View File

@ -323,7 +323,7 @@ func pullInvalidIgnored(t *testing.T, ft config.FolderType) {
fc.deleteFile(invDel)
fc.addFile(ign, 0644, protocol.FileInfoTypeFile, contents)
fc.addFile(ignExisting, 0644, protocol.FileInfoTypeFile, contents)
if err := ioutil.WriteFile(filepath.Join(fss.URI(), ignExisting), otherContents, 0644); err != nil {
if err := writeFile(fss, ignExisting, otherContents, 0644); err != nil {
panic(err)
}
@ -465,12 +465,12 @@ func TestIssue4841(t *testing.T) {
func TestRescanIfHaveInvalidContent(t *testing.T) {
m, fc, fcfg := setupModelWithConnection()
tmpDir := fcfg.Filesystem().URI()
defer cleanupModelAndRemoveDir(m, tmpDir)
tfs := fcfg.Filesystem()
defer cleanupModelAndRemoveDir(m, tfs.URI())
payload := []byte("hello")
must(t, ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777))
must(t, writeFile(tfs, "foo", payload, 0777))
received := make(chan []protocol.FileInfo)
fc.mut.Lock()
@ -511,7 +511,7 @@ func TestRescanIfHaveInvalidContent(t *testing.T) {
payload = []byte("bye")
buf = make([]byte, len(payload))
must(t, ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777))
must(t, writeFile(tfs, "foo", payload, 0777))
_, err = m.Request(device1, "default", "foo", int32(len(payload)), 0, f.Blocks[0].Hash, f.Blocks[0].WeakHash, false)
if err == nil {
@ -1051,7 +1051,7 @@ func TestIgnoreDeleteUnignore(t *testing.T) {
}
fc.mut.Unlock()
if err := ioutil.WriteFile(filepath.Join(fss.URI(), file), contents, 0644); err != nil {
if err := writeFile(fss, file, contents, 0644); err != nil {
panic(err)
}
m.ScanFolders()

View File

@ -9,6 +9,8 @@ package model
import (
"os"
"time"
"github.com/syncthing/syncthing/lib/fs"
)
// fatal is the required common interface between *testing.B and *testing.T
@ -28,6 +30,13 @@ func must(f fatal, err error) {
}
}
func mustRemove(f fatal, err error) {
f.Helper()
if err != nil && !fs.IsNotExist(err) {
f.Fatal(err)
}
}
func (f *fatalOs) Chmod(name string, mode os.FileMode) {
f.Helper()
must(f, os.Chmod(name, mode))

View File

@ -36,9 +36,8 @@ func init() {
device1, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
device2, _ = protocol.DeviceIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
defaultFs = fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata")
defaultFolderConfig = testFolderConfig("testdata")
defaultFs = defaultFolderConfig.Filesystem()
defaultCfgWrapper = createTmpWrapper(config.New(myID))
_, _ = defaultCfgWrapper.SetDevice(config.NewDeviceConfiguration(device1, "device1"))

View File

@ -131,9 +131,11 @@ func copyFileContents(method fs.CopyRangeMethod, srcFs, dstFs fs.Filesystem, src
}
func IsDeleted(ffs fs.Filesystem, name string) bool {
if _, err := ffs.Lstat(name); fs.IsNotExist(err) {
if _, err := ffs.Lstat(name); err != nil {
if fs.IsNotExist(err) || fs.IsErrCaseConflict(err) {
return true
}
}
switch TraversesSymlink(ffs, filepath.Dir(name)).(type) {
case *NotADirectoryError, *TraversesSymlinkError:
return true

View File

@ -62,7 +62,7 @@ func TestIsDeleted(t *testing.T) {
}
}
for _, n := range []string{"Dir", "File", "Del"} {
if err := osutil.DebugSymlinkForTestsOnly(filepath.Join(testFs.URI(), strings.ToLower(n)), filepath.Join(testFs.URI(), "linkTo"+n)); err != nil {
if err := fs.DebugSymlinkForTestsOnly(testFs, testFs, strings.ToLower(n), "linkTo"+n); err != nil {
if runtime.GOOS == "windows" {
t.Skip("Symlinks aren't working")
}

View File

@ -1,21 +0,0 @@
// Copyright (C) 2017 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build !windows
package osutil
import (
"os"
)
// DebugSymlinkForTestsOnly is not and should not be used in Syncthing code,
// hence the cumbersome name to make it obvious if this ever leaks. Its
// reason for existence is the Windows version, which allows creating
// symlinks when non-elevated.
func DebugSymlinkForTestsOnly(oldname, newname string) error {
return os.Symlink(oldname, newname)
}

View File

@ -24,9 +24,9 @@ func TestTraversesSymlink(t *testing.T) {
}
defer os.RemoveAll(tmpDir)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir)
fs.MkdirAll("a/b/c", 0755)
if err = osutil.DebugSymlinkForTestsOnly(filepath.Join(fs.URI(), "a", "b"), filepath.Join(fs.URI(), "a", "l")); err != nil {
testFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir)
testFs.MkdirAll("a/b/c", 0755)
if err = fs.DebugSymlinkForTestsOnly(testFs, testFs, filepath.Join("a", "b"), filepath.Join("a", "l")); err != nil {
if runtime.GOOS == "windows" {
t.Skip("Symlinks aren't working")
}
@ -34,7 +34,7 @@ func TestTraversesSymlink(t *testing.T) {
}
// a/l -> b, so a/l/c should resolve by normal stat
info, err := fs.Lstat("a/l/c")
info, err := testFs.Lstat("a/l/c")
if err != nil {
t.Fatal("unexpected error", err)
}
@ -64,7 +64,7 @@ func TestTraversesSymlink(t *testing.T) {
}
for _, tc := range cases {
if res := osutil.TraversesSymlink(fs, tc.name); tc.traverses == (res == nil) {
if res := osutil.TraversesSymlink(testFs, tc.name); tc.traverses == (res == nil) {
t.Errorf("TraversesSymlink(%q) = %v, should be %v", tc.name, res, tc.traverses)
}
}
@ -78,8 +78,8 @@ func TestIssue4875(t *testing.T) {
defer os.RemoveAll(tmpDir)
testFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir)
testFs.MkdirAll("a/b/c", 0755)
if err = osutil.DebugSymlinkForTestsOnly(filepath.Join(testFs.URI(), "a", "b"), filepath.Join(testFs.URI(), "a", "l")); err != nil {
testFs.MkdirAll(filepath.Join("a", "b", "c"), 0755)
if err = fs.DebugSymlinkForTestsOnly(testFs, testFs, filepath.Join("a", "b"), filepath.Join("a", "l")); err != nil {
if runtime.GOOS == "windows" {
t.Skip("Symlinks aren't working")
}

View File

@ -540,6 +540,10 @@ func (w *walker) handleError(ctx context.Context, context, path string, err erro
}
}
func (w *walker) String() string {
return fmt.Sprintf("walker/%s@%p", w.Folder, w)
}
// A byteCounter gets bytes added to it via Update() and then provides the
// Total() and one minute moving average Rate() in bytes per second.
type byteCounter struct {

View File

@ -26,7 +26,6 @@ import (
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sha256"
"golang.org/x/text/unicode/norm"
@ -40,9 +39,10 @@ type testfile struct {
type testfileList []testfile
var testFs fs.Filesystem
var testdata = testfileList{
var (
testFs fs.Filesystem
testFsType = fs.FilesystemTypeBasic
testdata = testfileList{
{"afile", 4, "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"},
{"dir1", 128, ""},
{filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"},
@ -51,6 +51,7 @@ var testdata = testfileList{
{"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"},
{"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"},
}
)
func init() {
// This test runs the risk of entering infinite recursion if it fails.
@ -270,7 +271,7 @@ func TestWalkSymlinkUnix(t *testing.T) {
defer os.RemoveAll("_symlinks")
os.Symlink("../testdata", "_symlinks/link")
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks")
fs := fs.NewFilesystem(testFsType, "_symlinks")
for _, path := range []string{".", "link"} {
// Scan it
files := walkDir(fs, path, nil, nil, 0)
@ -298,15 +299,15 @@ func TestWalkSymlinkWindows(t *testing.T) {
os.RemoveAll(name)
os.Mkdir(name, 0755)
defer os.RemoveAll(name)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, name)
if err := osutil.DebugSymlinkForTestsOnly("../testdata", "_symlinks/link"); err != nil {
testFs := fs.NewFilesystem(testFsType, name)
if err := fs.DebugSymlinkForTestsOnly(testFs, testFs, "../testdata", "link"); err != nil {
// Probably we require permissions we don't have.
t.Skip(err)
}
for _, path := range []string{".", "link"} {
// Scan it
files := walkDir(fs, path, nil, nil, 0)
files := walkDir(testFs, path, nil, nil, 0)
// Verify that we got zero symlinks
if len(files) != 0 {
@ -322,10 +323,12 @@ func TestWalkRootSymlink(t *testing.T) {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
testFs := fs.NewFilesystem(testFsType, tmp)
link := filepath.Join(tmp, "link")
link := "link"
dest, _ := filepath.Abs("testdata/dir1")
if err := osutil.DebugSymlinkForTestsOnly(dest, link); err != nil {
destFs := fs.NewFilesystem(testFsType, dest)
if err := fs.DebugSymlinkForTestsOnly(destFs, testFs, ".", "link"); err != nil {
if runtime.GOOS == "windows" {
// Probably we require permissions we don't have.
t.Skip("Need admin permissions or developer mode to run symlink test on Windows: " + err.Error())
@ -335,15 +338,15 @@ func TestWalkRootSymlink(t *testing.T) {
}
// Scan root with symlink at FS root
files := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil, nil, 0)
files := walkDir(fs.NewFilesystem(testFsType, filepath.Join(testFs.URI(), link)), ".", nil, nil, 0)
// Verify that we got two files
if len(files) != 2 {
t.Errorf("expected two files, not %d", len(files))
t.Fatalf("expected two files, not %d", len(files))
}
// Scan symlink below FS root
files = walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, tmp), "link", nil, nil, 0)
files = walkDir(testFs, "link", nil, nil, 0)
// Verify that we got the one symlink, except on windows
if runtime.GOOS == "windows" {
@ -355,7 +358,7 @@ func TestWalkRootSymlink(t *testing.T) {
}
// Scan path below symlink
files = walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, tmp), filepath.Join("link", "cfile"), nil, nil, 0)
files = walkDir(fs.NewFilesystem(testFsType, tmp), filepath.Join("link", "cfile"), nil, nil, 0)
// Verify that we get nothing
if len(files) != 0 {
@ -554,7 +557,7 @@ func BenchmarkHashFile(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.MinBlockSize, nil, true); err != nil {
if _, err := HashFile(context.TODO(), fs.NewFilesystem(testFsType, ""), testdataName, protocol.MinBlockSize, nil, true); err != nil {
b.Fatal(err)
}
}
@ -652,7 +655,7 @@ func TestIssue4799(t *testing.T) {
}
defer os.RemoveAll(tmp)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmp)
fs := fs.NewFilesystem(testFsType, tmp)
fd, err := fs.Create("foo")
if err != nil {
@ -714,7 +717,7 @@ func TestIssue4841(t *testing.T) {
}
defer os.RemoveAll(tmp)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmp)
fs := fs.NewFilesystem(testFsType, tmp)
fd, err := fs.Create("foo")
if err != nil {

View File

@ -128,6 +128,7 @@ type Report struct {
DisableFsync int `json:"disableFsync,omitempty" since:"3"`
BlockPullOrder map[string]int `json:"blockPullOrder,omitempty" since:"3"`
CopyRangeMethod map[string]int `json:"copyRangeMethod,omitempty" since:"3"`
CaseSensitiveFS int `json:"caseSensitiveFS,omitempty" since:"3"`
} `json:"folderUsesV3,omitempty" since:"3"`
GUIStats struct {

View File

@ -269,6 +269,9 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (
}
report.FolderUsesV3.BlockPullOrder[cfg.BlockPullOrder.String()]++
report.FolderUsesV3.CopyRangeMethod[cfg.CopyRangeMethod.String()]++
if cfg.CaseSensitiveFS {
report.FolderUsesV3.CaseSensitiveFS++
}
}
sort.Ints(report.FolderUsesV3.FsWatcherDelays)

View File

@ -10,42 +10,42 @@ package integration
import (
"log"
"math/rand"
"os"
"testing"
"time"
"github.com/syncthing/syncthing/lib/rc"
)
func TestBenchmarkTransferManyFiles(t *testing.T) {
benchmarkTransfer(t, 10000, 15)
setupAndBenchmarkTransfer(t, 10000, 15)
}
func TestBenchmarkTransferLargeFile1G(t *testing.T) {
benchmarkTransfer(t, 1, 30)
setupAndBenchmarkTransfer(t, 1, 30)
}
func TestBenchmarkTransferLargeFile2G(t *testing.T) {
benchmarkTransfer(t, 1, 31)
setupAndBenchmarkTransfer(t, 1, 31)
}
func TestBenchmarkTransferLargeFile4G(t *testing.T) {
benchmarkTransfer(t, 1, 32)
setupAndBenchmarkTransfer(t, 1, 32)
}
func TestBenchmarkTransferLargeFile8G(t *testing.T) {
benchmarkTransfer(t, 1, 33)
setupAndBenchmarkTransfer(t, 1, 33)
}
func TestBenchmarkTransferLargeFile16G(t *testing.T) {
benchmarkTransfer(t, 1, 34)
setupAndBenchmarkTransfer(t, 1, 34)
}
func TestBenchmarkTransferLargeFile32G(t *testing.T) {
benchmarkTransfer(t, 1, 35)
setupAndBenchmarkTransfer(t, 1, 35)
}
func benchmarkTransfer(t *testing.T, files, sizeExp int) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index*", "h2/index*")
if err != nil {
t.Fatal(err)
}
func setupAndBenchmarkTransfer(t *testing.T, files, sizeExp int) {
cleanBenchmarkTransfer(t)
log.Println("Generating files...")
var err error
if files == 1 {
// Special case. Generate one file with the specified size exactly.
var fd *os.File
@ -57,13 +57,39 @@ func benchmarkTransfer(t *testing.T, files, sizeExp int) {
if err != nil {
t.Fatal(err)
}
err = generateOneFile(fd, "s1/onefile", 1<<uint(sizeExp))
err = generateOneFile(fd, "s1/onefile", 1<<uint(sizeExp), time.Now())
} else {
err = generateFiles("s1", files, sizeExp, "../LICENSE")
}
if err != nil {
t.Fatal(err)
}
benchmarkTransfer(t)
}
// TestBenchmarkTransferSameFiles doesn't actually transfer anything, but tests
// how fast two devicees get in sync if they have the same data locally.
func TestBenchmarkTransferSameFiles(t *testing.T) {
cleanBenchmarkTransfer(t)
t0 := time.Now()
rand.Seed(0)
log.Println("Generating files in s1...")
if err := generateFilesWithTime("s1", 10000, 10, "../LICENSE", t0); err != nil {
t.Fatal(err)
}
rand.Seed(0)
log.Println("Generating same files in s2...")
if err := generateFilesWithTime("s2", 10000, 10, "../LICENSE", t0); err != nil {
t.Fatal(err)
}
benchmarkTransfer(t)
}
func benchmarkTransfer(t *testing.T) {
expected, err := directoryContents("s1")
if err != nil {
t.Fatal(err)
@ -86,9 +112,9 @@ func benchmarkTransfer(t *testing.T, files, sizeExp int) {
sender.ResumeAll()
receiver.ResumeAll()
var t0, t1 time.Time
t0 := time.Now()
var t1 time.Time
lastEvent := 0
oneItemFinished := false
loop:
for {
@ -105,32 +131,19 @@ loop:
switch ev.Type {
case "ItemFinished":
oneItemFinished = true
continue
case "StateChanged":
data := ev.Data.(map[string]interface{})
if data["folder"].(string) != "default" {
continue
}
switch data["to"].(string) {
case "syncing":
t0 = ev.Time
continue
case "idle":
if !oneItemFinished {
continue
}
if !t0.IsZero() {
t1 = ev.Time
break loop
}
}
}
time.Sleep(250 * time.Millisecond)
}
processes := []*rc.Process{sender, receiver}
for {
if rc.InSync("default", processes...) {
t1 = time.Now()
break
}
time.Sleep(250 * time.Millisecond)
}
@ -159,4 +172,14 @@ loop:
printUsage("Receiver", recvProc, total)
printUsage("Sender", sendProc, total)
cleanBenchmarkTransfer(t)
}
func cleanBenchmarkTransfer(t *testing.T) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index*", "h2/index*")
if err != nil {
t.Fatal(err)
}
}

View File

@ -41,6 +41,10 @@ const (
)
func generateFiles(dir string, files, maxexp int, srcname string) error {
return generateFilesWithTime(dir, files, maxexp, srcname, time.Now())
}
func generateFilesWithTime(dir string, files, maxexp int, srcname string, t0 time.Time) error {
fd, err := os.Open(srcname)
if err != nil {
return err
@ -69,7 +73,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
}
s += rand.Int63n(a)
if err := generateOneFile(fd, p1, s); err != nil {
if err := generateOneFile(fd, p1, s, t0); err != nil {
return err
}
}
@ -77,7 +81,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
return nil
}
func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
func generateOneFile(fd io.ReadSeeker, p1 string, s int64, t0 time.Time) error {
src := io.LimitReader(&inifiteReader{fd}, int64(s))
dst, err := os.Create(p1)
if err != nil {
@ -96,7 +100,7 @@ func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
t := t0.Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
err = os.Chtimes(p1, t, t)
if err != nil {
return err