lib/fs: Properly handle case insensitive systems (fixes #1787, fixes #2739, fixes #5708)

With this change we emulate a case sensitive filesystem on top of
insensitive filesystems. This means we correctly pick up case-only renames
and throw a case conflict error when there would be multiple files differing
only in case.

This safety check has a small performance hit (about 20% more filesystem
operations when scanning for changes). The new advanced folder option
`caseSensitiveFS` can be used to disable the safety checks, retaining the
previous behavior on systems known to be fully case sensitive.

Co-authored-by: Jakob Borg <jakob@kastelo.net>

cmd/ursrv/main.go

@@ -743,6 +743,7 @@ func getReport(db *sql.DB) map[string]interface{} {
 			inc(features["Folder"]["v3"], "Weak hash, always", rep.FolderUsesV3.AlwaysWeakHash)
 			inc(features["Folder"]["v3"], "Weak hash, custom threshold", rep.FolderUsesV3.CustomWeakHashThreshold)
 			inc(features["Folder"]["v3"], "Filesystem watcher", rep.FolderUsesV3.FsWatcherEnabled)
+			inc(features["Folder"]["v3"], "Case sensitive FS", rep.FolderUsesV3.CaseSensitiveFS)
 
 			add(featureGroups["Folder"]["v3"], "Conflicts", "Disabled", rep.FolderUsesV3.ConflictsDisabled)
 			add(featureGroups["Folder"]["v3"], "Conflicts", "Unlimited", rep.FolderUsesV3.ConflictsUnlimited)

lib/config/config_test.go

@@ -22,7 +22,6 @@ import (
 	"github.com/d4l3k/messagediff"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
-	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -129,13 +128,6 @@ func TestDeviceConfig(t *testing.T) {
 			},
 		}
 
-		// The cachedFilesystem will have been resolved to an absolute path,
-		// depending on where the tests are running. Zero it out so we don't
-		// fail based on that.
-		for i := range cfg.Folders {
-			cfg.Folders[i].cachedFilesystem = nil
-		}
-
 		expectedDevices := []DeviceConfiguration{
 			{
 				DeviceID:        device1,
@@ -465,6 +457,7 @@ func TestFolderCheckPath(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	testFs := fs.NewFilesystem(fs.FilesystemTypeBasic, n)
 
 	err = os.MkdirAll(filepath.Join(n, "dir", ".stfolder"), os.FileMode(0777))
 	if err != nil {
@@ -489,7 +482,7 @@ func TestFolderCheckPath(t *testing.T) {
 		},
 	}
 
-	err = osutil.DebugSymlinkForTestsOnly(filepath.Join(n, "dir"), filepath.Join(n, "link"))
+	err = fs.DebugSymlinkForTestsOnly(testFs, testFs, "dir", "link")
 	if err == nil {
 		t.Log("running with symlink check")
 		testcases = append(testcases, struct {

lib/config/folderconfiguration.go

@@ -61,8 +61,8 @@ type FolderConfiguration struct {
 	DisableFsync            bool                        `xml:"disableFsync" json:"disableFsync"`
 	BlockPullOrder          BlockPullOrder              `xml:"blockPullOrder" json:"blockPullOrder"`
 	CopyRangeMethod         fs.CopyRangeMethod          `xml:"copyRangeMethod" json:"copyRangeMethod" default:"standard"`
+	CaseSensitiveFS         bool                        `xml:"caseSensitiveFS" json:"caseSensitiveFS"`
 
-	cachedFilesystem    fs.Filesystem
 	cachedModTimeWindow time.Duration
 
 	DeprecatedReadOnly       bool    `xml:"ro,attr,omitempty" json:"-"`
@@ -101,11 +101,11 @@ func (f FolderConfiguration) Copy() FolderConfiguration {
 func (f FolderConfiguration) Filesystem() fs.Filesystem {
 	// This is intentionally not a pointer method, because things like
 	// cfg.Folders["default"].Filesystem() should be valid.
-	if f.cachedFilesystem == nil {
-		l.Infoln("bug: uncached filesystem call (should only happen in tests)")
-		return fs.NewFilesystem(f.FilesystemType, f.Path)
+	filesystem := fs.NewFilesystem(f.FilesystemType, f.Path)
+	if !f.CaseSensitiveFS {
+		filesystem = fs.NewCaseFilesystem(filesystem)
 	}
-	return f.cachedFilesystem
+	return filesystem
 }
 
 func (f FolderConfiguration) ModTimeWindow() time.Duration {
@@ -210,8 +210,6 @@ func (f *FolderConfiguration) DeviceIDs() []protocol.DeviceID {
 }
 
 func (f *FolderConfiguration) prepare() {
-	f.cachedFilesystem = fs.NewFilesystem(f.FilesystemType, f.Path)
-
 	if f.RescanIntervalS > MaxRescanIntervalS {
 		f.RescanIntervalS = MaxRescanIntervalS
 	} else if f.RescanIntervalS < 0 {
@@ -263,7 +261,6 @@ func (f FolderConfiguration) RequiresRestartOnly() FolderConfiguration {
 
 	// Manual handling for things that are not taken care of by the tag
 	// copier, yet should not cause a restart.
-	copy.cachedFilesystem = nil
 
 	blank := FolderConfiguration{}
 	util.CopyMatchingTag(&blank, &copy, "restart", func(v string) bool {

lib/fs/basicfs_realcaser_unix.go

@@ -0,0 +1,13 @@
+// Copyright (C) 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// +build !windows
+
+package fs
+
+func newBasicRealCaser(fs Filesystem) realCaser {
+	return newDefaultRealCaser(fs)
+}

lib/fs/basicfs_realcaser_windows.go

@@ -0,0 +1,58 @@
+// Copyright (C) 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// +build windows
+
+package fs
+
+import (
+	"path/filepath"
+	"strings"
+	"syscall"
+)
+
+type basicRealCaserWindows struct {
+	uri string
+}
+
+func newBasicRealCaser(fs Filesystem) realCaser {
+	return &basicRealCaserWindows{fs.URI()}
+}
+
+// RealCase returns the correct case for the given name, which is a relative
+// path below root, as it exists on disk.
+func (r *basicRealCaserWindows) realCase(name string) (string, error) {
+	if name == "." {
+		return ".", nil
+	}
+	path := r.uri
+	comps := strings.Split(name, string(PathSeparator))
+	var err error
+	for i, comp := range comps {
+		path = filepath.Join(path, comp)
+		comps[i], err = r.realCaseBase(path)
+		if err != nil {
+			return "", err
+		}
+	}
+	return filepath.Join(comps...), nil
+}
+
+func (*basicRealCaserWindows) realCaseBase(path string) (string, error) {
+	p, err := syscall.UTF16PtrFromString(fixLongPath(path))
+	if err != nil {
+		return "", err
+	}
+	var fd syscall.Win32finddata
+	h, err := syscall.FindFirstFile(p, &fd)
+	if err != nil {
+		return "", err
+	}
+	syscall.FindClose(h)
+	return syscall.UTF16ToString(fd.FileName[:]), nil
+}
+
+func (r *basicRealCaserWindows) dropCache() {}

lib/fs/casefs.go

@@ -0,0 +1,448 @@
+// Copyright (C) 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package fs
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Both values were chosen by magic.
+const (
+	caseCacheTimeout = time.Second
+	// When the number of names (all lengths of []string from DirNames)
+	// exceeds this, we drop the cache.
+	caseMaxCachedNames = 1 << 20
+)
+
+type ErrCaseConflict struct {
+	given, real string
+}
+
+func (e *ErrCaseConflict) Error() string {
+	return fmt.Sprintf(`given name "%v" differs from name in filesystem "%v"`, e.given, e.real)
+}
+
+func IsErrCaseConflict(err error) bool {
+	e := &ErrCaseConflict{}
+	return errors.As(err, &e)
+}
+
+type realCaser interface {
+	realCase(name string) (string, error)
+	dropCache()
+}
+
+type fskey struct {
+	fstype FilesystemType
+	uri    string
+}
+
+var (
+	caseFilesystems    = make(map[fskey]Filesystem)
+	caseFilesystemsMut sync.Mutex
+)
+
+// caseFilesystem is a BasicFilesystem with additional checks to make a
+// potentially case insensitive underlying FS behave like it's case-sensitive.
+type caseFilesystem struct {
+	Filesystem
+	realCaser
+}
+
+// NewCaseFilesystem ensures that the given, potentially case-insensitive filesystem
+// behaves like a case-sensitive filesystem. Meaning that it takes into account
+// the real casing of a path and returns ErrCaseConflict if the given path differs
+// from the real path. It is safe to use with any filesystem, i.e. also a
+// case-sensitive one. However it will add some overhead and thus shouldn't be
+// used if the filesystem is known to already behave case-sensitively.
+func NewCaseFilesystem(fs Filesystem) Filesystem {
+	caseFilesystemsMut.Lock()
+	defer caseFilesystemsMut.Unlock()
+	k := fskey{fs.Type(), fs.URI()}
+	if caseFs, ok := caseFilesystems[k]; ok {
+		return caseFs
+	}
+	caseFs := &caseFilesystem{
+		Filesystem: fs,
+	}
+	switch k.fstype {
+	case FilesystemTypeBasic:
+		caseFs.realCaser = newBasicRealCaser(fs)
+	default:
+		caseFs.realCaser = newDefaultRealCaser(fs)
+	}
+	caseFilesystems[k] = caseFs
+	return caseFs
+}
+
+func (f *caseFilesystem) Chmod(name string, mode FileMode) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	return f.Filesystem.Chmod(name, mode)
+}
+
+func (f *caseFilesystem) Lchown(name string, uid, gid int) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	return f.Filesystem.Lchown(name, uid, gid)
+}
+
+func (f *caseFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	return f.Filesystem.Chtimes(name, atime, mtime)
+}
+
+func (f *caseFilesystem) Mkdir(name string, perm FileMode) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	if err := f.Filesystem.Mkdir(name, perm); err != nil {
+		return err
+	}
+	f.dropCache()
+	return nil
+}
+
+func (f *caseFilesystem) MkdirAll(path string, perm FileMode) error {
+	if err := f.checkCase(path); err != nil {
+		return err
+	}
+	if err := f.Filesystem.MkdirAll(path, perm); err != nil {
+		return err
+	}
+	f.dropCache()
+	return nil
+}
+
+func (f *caseFilesystem) Lstat(name string) (FileInfo, error) {
+	var err error
+	if name, err = Canonicalize(name); err != nil {
+		return nil, err
+	}
+	stat, err := f.Filesystem.Lstat(name)
+	if err != nil {
+		return nil, err
+	}
+	if err = f.checkCaseExisting(name); err != nil {
+		return nil, err
+	}
+	return stat, nil
+}
+
+func (f *caseFilesystem) Remove(name string) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	if err := f.Filesystem.Remove(name); err != nil {
+		return err
+	}
+	f.dropCache()
+	return nil
+}
+
+func (f *caseFilesystem) RemoveAll(name string) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	if err := f.Filesystem.RemoveAll(name); err != nil {
+		return err
+	}
+	f.dropCache()
+	return nil
+}
+
+func (f *caseFilesystem) Rename(oldpath, newpath string) error {
+	if err := f.checkCase(oldpath); err != nil {
+		return err
+	}
+	if err := f.Filesystem.Rename(oldpath, newpath); err != nil {
+		return err
+	}
+	f.dropCache()
+	return nil
+}
+
+func (f *caseFilesystem) Stat(name string) (FileInfo, error) {
+	var err error
+	if name, err = Canonicalize(name); err != nil {
+		return nil, err
+	}
+	stat, err := f.Filesystem.Stat(name)
+	if err != nil {
+		return nil, err
+	}
+	if err = f.checkCaseExisting(name); err != nil {
+		return nil, err
+	}
+	return stat, nil
+}
+
+func (f *caseFilesystem) DirNames(name string) ([]string, error) {
+	if err := f.checkCase(name); err != nil {
+		return nil, err
+	}
+	return f.Filesystem.DirNames(name)
+}
+
+func (f *caseFilesystem) Open(name string) (File, error) {
+	if err := f.checkCase(name); err != nil {
+		return nil, err
+	}
+	return f.Filesystem.Open(name)
+}
+
+func (f *caseFilesystem) OpenFile(name string, flags int, mode FileMode) (File, error) {
+	if err := f.checkCase(name); err != nil {
+		return nil, err
+	}
+	file, err := f.Filesystem.OpenFile(name, flags, mode)
+	if err != nil {
+		return nil, err
+	}
+	f.dropCache()
+	return file, nil
+}
+
+func (f *caseFilesystem) ReadSymlink(name string) (string, error) {
+	if err := f.checkCase(name); err != nil {
+		return "", err
+	}
+	return f.Filesystem.ReadSymlink(name)
+}
+
+func (f *caseFilesystem) Create(name string) (File, error) {
+	if err := f.checkCase(name); err != nil {
+		return nil, err
+	}
+	file, err := f.Filesystem.Create(name)
+	if err != nil {
+		return nil, err
+	}
+	f.dropCache()
+	return file, nil
+}
+
+func (f *caseFilesystem) CreateSymlink(target, name string) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	if err := f.Filesystem.CreateSymlink(target, name); err != nil {
+		return err
+	}
+	f.dropCache()
+	return nil
+}
+
+func (f *caseFilesystem) Walk(root string, walkFn WalkFunc) error {
+	// Walking the filesystem is likely (in Syncthing's case certainly) done
+	// to pick up external changes, for which caching is undesirable.
+	f.dropCache()
+	if err := f.checkCase(root); err != nil {
+		return err
+	}
+	return f.Filesystem.Walk(root, walkFn)
+}
+
+func (f *caseFilesystem) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, <-chan error, error) {
+	if err := f.checkCase(path); err != nil {
+		return nil, nil, err
+	}
+	return f.Filesystem.Watch(path, ignore, ctx, ignorePerms)
+}
+
+func (f *caseFilesystem) Hide(name string) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	return f.Filesystem.Hide(name)
+}
+
+func (f *caseFilesystem) Unhide(name string) error {
+	if err := f.checkCase(name); err != nil {
+		return err
+	}
+	return f.Filesystem.Unhide(name)
+}
+
+func (f *caseFilesystem) checkCase(name string) error {
+	var err error
+	if name, err = Canonicalize(name); err != nil {
+		return err
+	}
+	// Stat is necessary for case sensitive FS, as it's then not a conflict
+	// if name is e.g. "foo" and on dir there is "Foo".
+	if _, err := f.Filesystem.Lstat(name); err != nil {
+		if IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+	return f.checkCaseExisting(name)
+}
+
+// checkCaseExisting must only be called after successfully canonicalizing and
+// stating the file.
+func (f *caseFilesystem) checkCaseExisting(name string) error {
+	realName, err := f.realCase(name)
+	if IsNotExist(err) {
+		// It did exist just before -> cache is outdated, try again
+		f.dropCache()
+		realName, err = f.realCase(name)
+	}
+	if err != nil {
+		return err
+	}
+	if realName != name {
+		return &ErrCaseConflict{name, realName}
+	}
+	return nil
+}
+
+type defaultRealCaser struct {
+	fs        Filesystem
+	root      *caseNode
+	count     int
+	timer     *time.Timer
+	timerStop chan struct{}
+	mut       sync.RWMutex
+}
+
+func newDefaultRealCaser(fs Filesystem) *defaultRealCaser {
+	caser := &defaultRealCaser{
+		fs:    fs,
+		root:  &caseNode{name: "."},
+		timer: time.NewTimer(0),
+	}
+	<-caser.timer.C
+	return caser
+}
+
+func (r *defaultRealCaser) realCase(name string) (string, error) {
+	out := "."
+	if name == out {
+		return out, nil
+	}
+
+	r.mut.Lock()
+	defer func() {
+		if r.count > caseMaxCachedNames {
+			select {
+			case r.timerStop <- struct{}{}:
+			default:
+			}
+			r.dropCacheLocked()
+		}
+		r.mut.Unlock()
+	}()
+
+	node := r.root
+	for _, comp := range strings.Split(name, string(PathSeparator)) {
+		if node.dirNames == nil {
+			// Haven't called DirNames yet
+			var err error
+			node.dirNames, err = r.fs.DirNames(out)
+			if err != nil {
+				return "", err
+			}
+			node.dirNamesLower = make([]string, len(node.dirNames))
+			for i, n := range node.dirNames {
+				node.dirNamesLower[i] = UnicodeLowercase(n)
+			}
+			node.children = make(map[string]*caseNode)
+			node.results = make(map[string]*caseNode)
+			r.count += len(node.dirNames)
+		} else if child, ok := node.results[comp]; ok {
+			// Check if this exact name has been queried before to shortcut
+			node = child
+			out = filepath.Join(out, child.name)
+			continue
+		}
+		// Actually loop dirNames to search for a match
+		n, err := findCaseInsensitiveMatch(comp, node.dirNames, node.dirNamesLower)
+		if err != nil {
+			return "", err
+		}
+		child, ok := node.children[n]
+		if !ok {
+			child = &caseNode{name: n}
+		}
+		node.results[comp] = child
+		node.children[n] = child
+		node = child
+		out = filepath.Join(out, n)
+	}
+
+	return out, nil
+}
+
+func (r *defaultRealCaser) startCaseResetTimerLocked() {
+	r.timerStop = make(chan struct{})
+	r.timer.Reset(caseCacheTimeout)
+	go func() {
+		select {
+		case <-r.timer.C:
+			r.dropCache()
+		case <-r.timerStop:
+			if !r.timer.Stop() {
+				<-r.timer.C
+			}
+			r.mut.Lock()
+			r.timerStop = nil
+			r.mut.Unlock()
+		}
+	}()
+}
+
+func (r *defaultRealCaser) dropCache() {
+	r.mut.Lock()
+	r.dropCacheLocked()
+	r.mut.Unlock()
+}
+
+func (r *defaultRealCaser) dropCacheLocked() {
+	r.root = &caseNode{name: "."}
+	r.count = 0
+}
+
+// Both name and the key to children are "Real", case resolved names of the path
+// component this node represents (i.e. containing no path separator).
+// The key to results is also a path component, but as given to RealCase, not
+// case resolved.
+type caseNode struct {
+	name          string
+	dirNames      []string
+	dirNamesLower []string
+	children      map[string]*caseNode
+	results       map[string]*caseNode
+}
+
+func findCaseInsensitiveMatch(name string, names, namesLower []string) (string, error) {
+	lower := UnicodeLowercase(name)
+	candidate := ""
+	for i, n := range names {
+		if n == name {
+			return n, nil
+		}
+		if candidate == "" && namesLower[i] == lower {
+			candidate = n
+		}
+	}
+	if candidate == "" {
+		return "", ErrNotExist
+	}
+	return candidate, nil
+}

lib/fs/casefs_test.go

@@ -0,0 +1,279 @@
+// Copyright (C) 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package fs
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestRealCase(t *testing.T) {
+	// Verify realCase lookups on various underlying filesystems.
+
+	t.Run("fake-sensitive", func(t *testing.T) {
+		testRealCase(t, newFakeFilesystem(t.Name()))
+	})
+	t.Run("fake-insensitive", func(t *testing.T) {
+		testRealCase(t, newFakeFilesystem(t.Name()+"?insens=true"))
+	})
+	t.Run("actual", func(t *testing.T) {
+		fsys, tmpDir := setup(t)
+		defer os.RemoveAll(tmpDir)
+		testRealCase(t, fsys)
+	})
+}
+
+func testRealCase(t *testing.T, fsys Filesystem) {
+	testFs := NewCaseFilesystem(fsys).(*caseFilesystem)
+	comps := []string{"Foo", "bar", "BAZ", "bAs"}
+	path := filepath.Join(comps...)
+	testFs.MkdirAll(filepath.Join(comps[:len(comps)-1]...), 0777)
+	fd, err := testFs.Create(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fd.Close()
+
+	for i, tc := range []struct {
+		in  string
+		len int
+	}{
+		{path, 4},
+		{strings.ToLower(path), 4},
+		{strings.ToUpper(path), 4},
+		{"foo", 1},
+		{"FOO", 1},
+		{"foO", 1},
+		{filepath.Join("Foo", "bar"), 2},
+		{filepath.Join("Foo", "bAr"), 2},
+		{filepath.Join("FoO", "bar"), 2},
+		{filepath.Join("foo", "bar", "BAZ"), 3},
+		{filepath.Join("Foo", "bar", "bAz"), 3},
+		{filepath.Join("foo", "bar", "BAZ"), 3}, // Repeat on purpose
+	} {
+		out, err := testFs.realCase(tc.in)
+		if err != nil {
+			t.Error(err)
+		} else if exp := filepath.Join(comps[:tc.len]...); out != exp {
+			t.Errorf("tc %v: Expected %v, got %v", i, exp, out)
+		}
+	}
+}
+
+func TestRealCaseSensitive(t *testing.T) {
+	// Verify that realCase returns the best on-disk case for case sensitive
+	// systems. Test is skipped if the underlying fs is insensitive.
+
+	t.Run("fake-sensitive", func(t *testing.T) {
+		testRealCaseSensitive(t, newFakeFilesystem(t.Name()))
+	})
+	t.Run("actual", func(t *testing.T) {
+		fsys, tmpDir := setup(t)
+		defer os.RemoveAll(tmpDir)
+		testRealCaseSensitive(t, fsys)
+	})
+}
+
+func testRealCaseSensitive(t *testing.T, fsys Filesystem) {
+	testFs := NewCaseFilesystem(fsys).(*caseFilesystem)
+
+	names := make([]string, 2)
+	names[0] = "foo"
+	names[1] = strings.ToUpper(names[0])
+	for _, n := range names {
+		if err := testFs.MkdirAll(n, 0777); err != nil {
+			if IsErrCaseConflict(err) {
+				t.Skip("Filesystem is case-insensitive")
+			}
+			t.Fatal(err)
+		}
+	}
+
+	for _, n := range names {
+		if rn, err := testFs.realCase(n); err != nil {
+			t.Error(err)
+		} else if rn != n {
+			t.Errorf("Got %v, expected %v", rn, n)
+		}
+	}
+}
+
+func TestCaseFSStat(t *testing.T) {
+	// Verify that a Stat() lookup behaves in a case sensitive manner
+	// regardless of the underlying fs.
+
+	t.Run("fake-sensitive", func(t *testing.T) {
+		testCaseFSStat(t, newFakeFilesystem(t.Name()))
+	})
+	t.Run("fake-insensitive", func(t *testing.T) {
+		testCaseFSStat(t, newFakeFilesystem(t.Name()+"?insens=true"))
+	})
+	t.Run("actual", func(t *testing.T) {
+		fsys, tmpDir := setup(t)
+		defer os.RemoveAll(tmpDir)
+		testCaseFSStat(t, fsys)
+	})
+}
+
+func testCaseFSStat(t *testing.T, fsys Filesystem) {
+	fd, err := fsys.Create("foo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fd.Close()
+
+	// Check if the underlying fs is sensitive or not
+	sensitive := true
+	if _, err = fsys.Stat("FOO"); err == nil {
+		sensitive = false
+	}
+
+	testFs := NewCaseFilesystem(fsys)
+	_, err = testFs.Stat("FOO")
+	if sensitive {
+		if IsNotExist(err) {
+			t.Log("pass: case sensitive underlying fs")
+		} else {
+			t.Error("expected NotExist, not", err, "for sensitive fs")
+		}
+	} else if IsErrCaseConflict(err) {
+		t.Log("pass: case insensitive underlying fs")
+	} else {
+		t.Error("expected ErrCaseConflict, not", err, "for insensitive fs")
+	}
+}
+
+func BenchmarkWalkCaseFakeFS10k(b *testing.B) {
+	fsys, paths, err := fakefsForBenchmark(10_000, 0)
+	if err != nil {
+		b.Fatal(err)
+	}
+	slowsys, paths, err := fakefsForBenchmark(10_000, 100*time.Microsecond)
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.Run("raw-fastfs", func(b *testing.B) {
+		benchmarkWalkFakeFS(b, fsys, paths)
+		b.ReportAllocs()
+	})
+	b.Run("case-fastfs", func(b *testing.B) {
+		benchmarkWalkFakeFS(b, NewCaseFilesystem(fsys), paths)
+		b.ReportAllocs()
+	})
+	b.Run("raw-slowfs", func(b *testing.B) {
+		benchmarkWalkFakeFS(b, slowsys, paths)
+		b.ReportAllocs()
+	})
+	b.Run("case-slowfs", func(b *testing.B) {
+		benchmarkWalkFakeFS(b, NewCaseFilesystem(slowsys), paths)
+		b.ReportAllocs()
+	})
+}
+
+func benchmarkWalkFakeFS(b *testing.B, fsys Filesystem, paths []string) {
+	// Simulate a scanner pass over the filesystem. First walk it to
+	// discover all names, then stat each name individually to check if it's
+	// been deleted or not (pretending that they all existed in the
+	// database).
+
+	var ms0 runtime.MemStats
+	runtime.ReadMemStats(&ms0)
+	t0 := time.Now()
+
+	for i := 0; i < b.N; i++ {
+		if err := doubleWalkFS(fsys, paths); err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	t1 := time.Now()
+	var ms1 runtime.MemStats
+	runtime.ReadMemStats(&ms1)
+
+	// We add metrics per path entry
+	b.ReportMetric(float64(t1.Sub(t0))/float64(b.N)/float64(len(paths)), "ns/entry")
+	b.ReportMetric(float64(ms1.Mallocs-ms0.Mallocs)/float64(b.N)/float64(len(paths)), "allocs/entry")
+	b.ReportMetric(float64(ms1.TotalAlloc-ms0.TotalAlloc)/float64(b.N)/float64(len(paths)), "B/entry")
+}
+
+func TestStressCaseFS(t *testing.T) {
+	// Exercise a bunch of paralell operations for stressing out race
+	// conditions in the realnamer cache etc.
+
+	const limit = 10 * time.Second
+	if testing.Short() {
+		t.Skip("long test")
+	}
+
+	fsys, paths, err := fakefsForBenchmark(10_000, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < runtime.NumCPU()/2+1; i++ {
+		t.Run(fmt.Sprintf("walker-%d", i), func(t *testing.T) {
+			// Walk the filesystem and stat everything
+			t.Parallel()
+			t0 := time.Now()
+			for time.Since(t0) < limit {
+				if err := doubleWalkFS(fsys, paths); err != nil {
+					t.Fatal(err)
+				}
+			}
+		})
+		t.Run(fmt.Sprintf("toucher-%d", i), func(t *testing.T) {
+			// Touch all the things
+			t.Parallel()
+			t0 := time.Now()
+			for time.Since(t0) < limit {
+				for _, p := range paths {
+					now := time.Now()
+					if err := fsys.Chtimes(p, now, now); err != nil {
+						t.Fatal(err)
+					}
+				}
+			}
+		})
+	}
+}
+
+func doubleWalkFS(fsys Filesystem, paths []string) error {
+	if err := fsys.Walk("/", func(path string, info FileInfo, err error) error {
+		return err
+	}); err != nil {
+		return err
+	}
+
+	for _, p := range paths {
+		if _, err := fsys.Lstat(p); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func fakefsForBenchmark(nfiles int, latency time.Duration) (Filesystem, []string, error) {
+	fsys := NewFilesystem(FilesystemTypeFake, fmt.Sprintf("fakefsForBenchmark?files=%d&insens=true&latency=%s", nfiles, latency))
+
+	var paths []string
+	if err := fsys.Walk("/", func(path string, info FileInfo, err error) error {
+		paths = append(paths, path)
+		return err
+	}); err != nil {
+		return nil, nil, err
+	}
+	if len(paths) < nfiles {
+		return nil, nil, errors.New("didn't find enough stuff")
+	}
+
+	return fsys, paths, nil
+}

lib/fs/debug_symlink_unix.go

@@ -0,0 +1,47 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// +build !windows
+
+package fs
+
+import (
+	"os"
+	"path/filepath"
+)
+
+// DebugSymlinkForTestsOnly is not and should not be used in Syncthing code,
+// hence the cumbersome name to make it obvious if this ever leaks. Its
+// reason for existence is the Windows version, which allows creating
+// symlinks when non-elevated.
+func DebugSymlinkForTestsOnly(oldFs, newFs Filesystem, oldname, newname string) error {
+	if caseFs, ok := unwrapFilesystem(newFs).(*caseFilesystem); ok {
+		if err := caseFs.checkCase(newname); err != nil {
+			return err
+		}
+		caseFs.dropCache()
+	}
+	if err := os.Symlink(filepath.Join(oldFs.URI(), oldname), filepath.Join(newFs.URI(), newname)); err != nil {
+		return err
+	}
+	return nil
+}
+
+// unwrapFilesystem removes "wrapping" filesystems to expose the underlying filesystem.
+func unwrapFilesystem(fs Filesystem) Filesystem {
+	for {
+		switch sfs := fs.(type) {
+		case *logFilesystem:
+			fs = sfs.Filesystem
+		case *walkFilesystem:
+			fs = sfs.Filesystem
+		case *MtimeFS:
+			fs = sfs.Filesystem
+		default:
+			return sfs
+		}
+	}
+}

lib/fs/debug_symlink_windows.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package osutil
+package fs
 
 import (
 	"os"
@@ -17,7 +17,10 @@ import (
 // This is not and should not be used in Syncthing code, hence the
 // cumbersome name to make it obvious if this ever leaks. Nonetheless it's
 // useful in tests.
-func DebugSymlinkForTestsOnly(oldname, newname string) error {
+func DebugSymlinkForTestsOnly(oldFs, newFS Filesystem, oldname, newname string) error {
+	oldname = filepath.Join(oldFs.URI(), oldname)
+	newname = filepath.Join(newFS.URI(), newname)
+
 	// CreateSymbolicLink is not supported before Windows Vista
 	if syscall.LoadCreateSymbolicLink() != nil {
 		return &os.LinkError{"symlink", oldname, newname, syscall.EWINDOWS}

lib/fs/fakefs.go

@@ -48,14 +48,17 @@ const randomBlockShift = 14 // 128k
 //     sizeavg=n  to set the average size of random files, in bytes (default 1<<20)
 //     seed=n     to set the initial random seed (default 0)
 //     insens=b   "true" makes filesystem case-insensitive Windows- or OSX-style (default false)
+//     latency=d  to set the amount of time each "disk" operation takes, where d is time.ParseDuration format
 //
 // - Two fakefs:s pointing at the same root path see the same files.
 //
 type fakefs struct {
+	uri         string
 	mut         sync.Mutex
 	root        *fakeEntry
 	insens      bool
 	withContent bool
+	latency     time.Duration
 }
 
 var (
@@ -63,23 +66,25 @@ var (
 	fakefsFs  = make(map[string]*fakefs)
 )
 
-func newFakeFilesystem(root string) *fakefs {
+func newFakeFilesystem(rootURI string) *fakefs {
 	fakefsMut.Lock()
 	defer fakefsMut.Unlock()
 
+	root := rootURI
 	var params url.Values
-	uri, err := url.Parse(root)
+	uri, err := url.Parse(rootURI)
 	if err == nil {
 		root = uri.Path
 		params = uri.Query()
 	}
 
-	if fs, ok := fakefsFs[root]; ok {
+	if fs, ok := fakefsFs[rootURI]; ok {
 		// Already have an fs at this path
 		return fs
 	}
 
 	fs := &fakefs{
+		uri: "fake://" + rootURI,
 		root: &fakeEntry{
 			name:      "/",
 			entryType: fakeEntryTypeDir,
@@ -129,6 +134,10 @@ func newFakeFilesystem(root string) *fakefs {
 	// Also create a default folder marker for good measure
 	fs.Mkdir(".stfolder", 0700)
 
+	// We only set the latency after doing the operations required to create
+	// the filesystem initially.
+	fs.latency, _ = time.ParseDuration(params.Get("latency"))
+
 	fakefsFs[root] = fs
 	return fs
 }
@@ -185,6 +194,7 @@ func (fs *fakefs) entryForName(name string) *fakeEntry {
 func (fs *fakefs) Chmod(name string, mode FileMode) error {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 	entry := fs.entryForName(name)
 	if entry == nil {
 		return os.ErrNotExist
@@ -196,6 +206,7 @@ func (fs *fakefs) Chmod(name string, mode FileMode) error {
 func (fs *fakefs) Lchown(name string, uid, gid int) error {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 	entry := fs.entryForName(name)
 	if entry == nil {
 		return os.ErrNotExist
@@ -208,6 +219,7 @@ func (fs *fakefs) Lchown(name string, uid, gid int) error {
 func (fs *fakefs) Chtimes(name string, atime time.Time, mtime time.Time) error {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 	entry := fs.entryForName(name)
 	if entry == nil {
 		return os.ErrNotExist
@@ -219,6 +231,7 @@ func (fs *fakefs) Chtimes(name string, atime time.Time, mtime time.Time) error {
 func (fs *fakefs) create(name string) (*fakeEntry, error) {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	if entry := fs.entryForName(name); entry != nil {
 		if entry.entryType == fakeEntryTypeDir {
@@ -284,6 +297,7 @@ func (fs *fakefs) CreateSymlink(target, name string) error {
 func (fs *fakefs) DirNames(name string) ([]string, error) {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	entry := fs.entryForName(name)
 	if entry == nil {
@@ -301,6 +315,7 @@ func (fs *fakefs) DirNames(name string) ([]string, error) {
 func (fs *fakefs) Lstat(name string) (FileInfo, error) {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	entry := fs.entryForName(name)
 	if entry == nil {
@@ -318,6 +333,7 @@ func (fs *fakefs) Lstat(name string) (FileInfo, error) {
 func (fs *fakefs) Mkdir(name string, perm FileMode) error {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	dir := filepath.Dir(name)
 	base := filepath.Base(name)
@@ -348,6 +364,10 @@ func (fs *fakefs) Mkdir(name string, perm FileMode) error {
 }
 
 func (fs *fakefs) MkdirAll(name string, perm FileMode) error {
+	fs.mut.Lock()
+	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
+
 	name = filepath.ToSlash(name)
 	name = strings.Trim(name, "/")
 	comps := strings.Split(name, "/")
@@ -382,6 +402,7 @@ func (fs *fakefs) MkdirAll(name string, perm FileMode) error {
 func (fs *fakefs) Open(name string) (File, error) {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	entry := fs.entryForName(name)
 	if entry == nil || entry.entryType != fakeEntryTypeFile {
@@ -401,6 +422,7 @@ func (fs *fakefs) OpenFile(name string, flags int, mode FileMode) (File, error)
 
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	dir := filepath.Dir(name)
 	base := filepath.Base(name)
@@ -438,6 +460,7 @@ func (fs *fakefs) OpenFile(name string, flags int, mode FileMode) (File, error)
 func (fs *fakefs) ReadSymlink(name string) (string, error) {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	entry := fs.entryForName(name)
 	if entry == nil {
@@ -451,6 +474,7 @@ func (fs *fakefs) ReadSymlink(name string) (string, error) {
 func (fs *fakefs) Remove(name string) error {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	if fs.insens {
 		name = UnicodeLowercase(name)
@@ -472,6 +496,7 @@ func (fs *fakefs) Remove(name string) error {
 func (fs *fakefs) RemoveAll(name string) error {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	if fs.insens {
 		name = UnicodeLowercase(name)
@@ -491,6 +516,7 @@ func (fs *fakefs) RemoveAll(name string) error {
 func (fs *fakefs) Rename(oldname, newname string) error {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
+	time.Sleep(fs.latency)
 
 	oldKey := filepath.Base(oldname)
 	newKey := filepath.Base(newname)
@@ -578,7 +604,7 @@ func (fs *fakefs) Type() FilesystemType {
 }
 
 func (fs *fakefs) URI() string {
-	return "fake://" + fs.root.name
+	return fs.uri
 }
 
 func (fs *fakefs) SameFile(fi1, fi2 FileInfo) bool {

lib/model/folder_sendrecv.go

@@ -129,9 +129,9 @@ type sendReceiveFolder struct {
 	blockPullReorderer blockPullReorderer
 	writeLimiter       *byteSemaphore
 
-	pullErrors    map[string]string // errors for most recent/current iteration
-	oldPullErrors map[string]string // errors from previous iterations for log filtering only
-	pullErrorsMut sync.Mutex
+	pullErrors     map[string]string // actual exposed pull errors
+	tempPullErrors map[string]string // pull errors that might be just transient
+	pullErrorsMut  sync.Mutex
 }
 
 func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem, evLogger events.Logger, ioLimiter *byteSemaphore) service {
@@ -192,6 +192,10 @@ func (f *sendReceiveFolder) pull() bool {
 
 	changed := 0
 
+	f.pullErrorsMut.Lock()
+	f.pullErrors = nil
+	f.pullErrorsMut.Unlock()
+
 	for tries := 0; tries < maxPullerIterations; tries++ {
 		select {
 		case <-f.ctx.Done():
@@ -216,8 +220,14 @@ func (f *sendReceiveFolder) pull() bool {
 	}
 
 	f.pullErrorsMut.Lock()
+	f.pullErrors = f.tempPullErrors
+	f.tempPullErrors = nil
+	for path, err := range f.pullErrors {
+		l.Infof("Puller (folder %s, item %q): %v", f.Description(), path, err)
+	}
 	pullErrNum := len(f.pullErrors)
 	f.pullErrorsMut.Unlock()
+
 	if pullErrNum > 0 {
 		l.Infof("%v: Failed to sync %v items", f.Description(), pullErrNum)
 		f.evLogger.Log(events.FolderErrors, map[string]interface{}{
@@ -235,8 +245,7 @@ func (f *sendReceiveFolder) pull() bool {
 // flagged as needed in the folder.
 func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
 	f.pullErrorsMut.Lock()
-	f.oldPullErrors = f.pullErrors
-	f.pullErrors = make(map[string]string)
+	f.tempPullErrors = make(map[string]string)
 	f.pullErrorsMut.Unlock()
 
 	snap := f.fset.Snapshot()
@@ -306,10 +315,6 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
 	close(dbUpdateChan)
 	updateWg.Wait()
 
-	f.pullErrorsMut.Lock()
-	f.oldPullErrors = nil
-	f.pullErrorsMut.Unlock()
-
 	f.queue.Reset()
 
 	return changed
@@ -739,7 +744,11 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, snap *db.Snaps
 	}
 
 	// There is already something under that name, we need to handle that.
-	if info, err := f.fs.Lstat(file.Name); err == nil {
+	switch info, err := f.fs.Lstat(file.Name); {
+	case err != nil && !fs.IsNotExist(err):
+		f.newPullError(file.Name, errors.Wrap(err, "checking for existing symlink"))
+		return
+	case err == nil:
 		// Check that it is what we have in the database.
 		curFile, hasCurFile := f.model.CurrentFolderFile(f.folderID, file.Name)
 		if err := f.scanIfItemChanged(file.Name, info, curFile, hasCurFile, scanChan); err != nil {
@@ -1783,7 +1792,7 @@ func (f *sendReceiveFolder) newPullError(path string, err error) {
 	// We might get more than one error report for a file (i.e. error on
 	// Write() followed by Close()); we keep the first error as that is
 	// probably closer to the root cause.
-	if _, ok := f.pullErrors[path]; ok {
+	if _, ok := f.tempPullErrors[path]; ok {
 		return
 	}
 
@@ -1791,15 +1800,9 @@ func (f *sendReceiveFolder) newPullError(path string, err error) {
 	// Use "syncing" as opposed to "pulling" as the latter might be used
 	// for errors occurring specificly in the puller routine.
 	errStr := fmt.Sprintln("syncing:", err)
-	f.pullErrors[path] = errStr
+	f.tempPullErrors[path] = errStr
 
-	if oldErr, ok := f.oldPullErrors[path]; ok && oldErr == errStr {
-		l.Debugf("Repeat error on puller (folder %s, item %q): %v", f.Description(), path, err)
-		delete(f.oldPullErrors, path) // Potential repeats are now caught by f.pullErrors itself
-		return
-	}
-
-	l.Infof("Puller (folder %s, item %q): %v", f.Description(), path, err)
+	l.Debugf("%v new error for %v: %v", f, path, err)
 }
 
 func (f *sendReceiveFolder) Errors() []FileError {

lib/model/folder_sendrecv_test.go

@@ -10,11 +10,13 @@ import (
 	"bytes"
 	"context"
 	"crypto/rand"
+	"errors"
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
 	"time"
 
@@ -95,6 +97,7 @@ func setupSendReceiveFolder(files ...protocol.FileInfo) (*model, *sendReceiveFol
 	model.Supervisor.Stop()
 	f := model.folderRunners[fcfg.ID].(*sendReceiveFolder)
 	f.pullErrors = make(map[string]string)
+	f.tempPullErrors = make(map[string]string)
 	f.ctx = context.Background()
 
 	// Update index
@@ -983,7 +986,7 @@ func TestDeleteBehindSymlink(t *testing.T) {
 	must(t, osutil.RenameOrCopy(fs.CopyRangeMethodStandard, ffs, destFs, file, "file"))
 	must(t, ffs.RemoveAll(link))
 
-	if err := osutil.DebugSymlinkForTestsOnly(destFs.URI(), filepath.Join(ffs.URI(), link)); err != nil {
+	if err := fs.DebugSymlinkForTestsOnly(destFs, ffs, "", link); err != nil {
 		if runtime.GOOS == "windows" {
 			// Probably we require permissions we don't have.
 			t.Skip("Need admin permissions or developer mode to run symlink test on Windows: " + err.Error())
@@ -1087,6 +1090,122 @@ func TestPullDeleteUnscannedDir(t *testing.T) {
 	}
 }
 
+func TestPullCaseOnlyPerformFinish(t *testing.T) {
+	m, f := setupSendReceiveFolder()
+	defer cleanupSRFolder(f, m)
+	ffs := f.Filesystem()
+
+	name := "foo"
+	contents := []byte("contents")
+	must(t, writeFile(ffs, name, contents, 0644))
+	must(t, f.scanSubdirs(nil))
+
+	var cur protocol.FileInfo
+	hasCur := false
+	snap := dbSnapshot(t, m, f.ID)
+	defer snap.Release()
+	snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
+		if hasCur {
+			t.Fatal("got more than one file")
+		}
+		cur = i.(protocol.FileInfo)
+		hasCur = true
+		return true
+	})
+	if !hasCur {
+		t.Fatal("file is missing")
+	}
+
+	remote := *(&cur)
+	remote.Version = protocol.Vector{}.Update(device1.Short())
+	remote.Name = strings.ToUpper(cur.Name)
+	temp := fs.TempName(remote.Name)
+	must(t, writeFile(ffs, temp, contents, 0644))
+	scanChan := make(chan string, 1)
+	dbUpdateChan := make(chan dbUpdateJob, 1)
+
+	err := f.performFinish(remote, cur, hasCur, temp, snap, dbUpdateChan, scanChan)
+
+	select {
+	case <-dbUpdateChan: // boring case sensitive filesystem
+		return
+	case <-scanChan:
+		t.Error("no need to scan anything here")
+	default:
+	}
+
+	var caseErr *fs.ErrCaseConflict
+	if !errors.As(err, &caseErr) {
+		t.Error("Expected case conflict error, got", err)
+	}
+}
+
+func TestPullCaseOnlyDir(t *testing.T) {
+	testPullCaseOnlyDirOrSymlink(t, true)
+}
+
+func TestPullCaseOnlySymlink(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("symlinks not supported on windows")
+	}
+	testPullCaseOnlyDirOrSymlink(t, false)
+}
+
+func testPullCaseOnlyDirOrSymlink(t *testing.T, dir bool) {
+	m, f := setupSendReceiveFolder()
+	defer cleanupSRFolder(f, m)
+	ffs := f.Filesystem()
+
+	name := "foo"
+	if dir {
+		must(t, ffs.Mkdir(name, 0777))
+	} else {
+		must(t, ffs.CreateSymlink("target", name))
+	}
+
+	must(t, f.scanSubdirs(nil))
+	var cur protocol.FileInfo
+	hasCur := false
+	snap := dbSnapshot(t, m, f.ID)
+	defer snap.Release()
+	snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
+		if hasCur {
+			t.Fatal("got more than one file")
+		}
+		cur = i.(protocol.FileInfo)
+		hasCur = true
+		return true
+	})
+	if !hasCur {
+		t.Fatal("file is missing")
+	}
+
+	scanChan := make(chan string, 1)
+	dbUpdateChan := make(chan dbUpdateJob, 1)
+	remote := *(&cur)
+	remote.Version = protocol.Vector{}.Update(device1.Short())
+	remote.Name = strings.ToUpper(cur.Name)
+
+	if dir {
+		f.handleDir(remote, snap, dbUpdateChan, scanChan)
+	} else {
+		f.handleSymlink(remote, snap, dbUpdateChan, scanChan)
+	}
+
+	select {
+	case <-dbUpdateChan: // boring case sensitive filesystem
+		return
+	case <-scanChan:
+		t.Error("no need to scan anything here")
+	default:
+	}
+	if errStr, ok := f.tempPullErrors[remote.Name]; !ok {
+		t.Error("missing error for", remote.Name)
+	} else if !strings.Contains(errStr, "differs from name") {
+		t.Error("unexpected error", errStr, "for", remote.Name)
+	}
+}
+
 func cleanupSharedPullerState(s *sharedPullerState) {
 	s.mut.Lock()
 	defer s.mut.Unlock()

lib/model/folder_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 
 	"github.com/d4l3k/messagediff"
+
 	"github.com/syncthing/syncthing/lib/config"
 )
 

lib/model/model_test.go

@@ -270,17 +270,15 @@ func BenchmarkRequestOut(b *testing.B) {
 }
 
 func BenchmarkRequestInSingleFile(b *testing.B) {
-	testOs := &fatalOs{b}
-
 	m := setupModel(defaultCfgWrapper)
 	defer cleanupModel(m)
 
 	buf := make([]byte, 128<<10)
 	rand.Read(buf)
-	testOs.RemoveAll("testdata/request")
-	defer testOs.RemoveAll("testdata/request")
-	testOs.MkdirAll("testdata/request/for/a/file/in/a/couple/of/dirs", 0755)
-	ioutil.WriteFile("testdata/request/for/a/file/in/a/couple/of/dirs/128k", buf, 0644)
+	mustRemove(b, defaultFs.RemoveAll("request"))
+	defer func() { mustRemove(b, defaultFs.RemoveAll("request")) }()
+	must(b, defaultFs.MkdirAll("request/for/a/file/in/a/couple/of/dirs", 0755))
+	writeFile(defaultFs, "request/for/a/file/in/a/couple/of/dirs/128k", buf, 0644)
 
 	b.ResetTimer()
 
@@ -294,13 +292,11 @@ func BenchmarkRequestInSingleFile(b *testing.B) {
 }
 
 func TestDeviceRename(t *testing.T) {
-	testOs := &fatalOs{t}
-
 	hello := protocol.HelloResult{
 		ClientName:    "syncthing",
 		ClientVersion: "v0.9.4",
 	}
-	defer testOs.Remove("testdata/tmpconfig.xml")
+	defer func() { mustRemove(t, defaultFs.Remove("tmpconfig.xml")) }()
 
 	rawCfg := config.New(device1)
 	rawCfg.Devices = []config.DeviceConfiguration{
@@ -1447,12 +1443,10 @@ func changeIgnores(t *testing.T, m *model, expected []string) {
 }
 
 func TestIgnores(t *testing.T) {
-	testOs := &fatalOs{t}
-
 	// Assure a clean start state
-	testOs.RemoveAll(filepath.Join("testdata", config.DefaultMarkerName))
-	testOs.MkdirAll(filepath.Join("testdata", config.DefaultMarkerName), 0644)
-	ioutil.WriteFile("testdata/.stignore", []byte(".*\nquux\n"), 0644)
+	mustRemove(t, defaultFs.RemoveAll(config.DefaultMarkerName))
+	mustRemove(t, defaultFs.MkdirAll(config.DefaultMarkerName, 0644))
+	writeFile(defaultFs, ".stignore", []byte(".*\nquux\n"), 0644)
 
 	m := setupModel(defaultCfgWrapper)
 	defer cleanupModel(m)
@@ -1504,18 +1498,16 @@ func TestIgnores(t *testing.T) {
 
 	// Make sure no .stignore file is considered valid
 	defer func() {
-		testOs.Rename("testdata/.stignore.bak", "testdata/.stignore")
+		must(t, defaultFs.Rename(".stignore.bak", ".stignore"))
 	}()
-	testOs.Rename("testdata/.stignore", "testdata/.stignore.bak")
+	must(t, defaultFs.Rename(".stignore", ".stignore.bak"))
 	changeIgnores(t, m, []string{})
 }
 
 func TestEmptyIgnores(t *testing.T) {
-	testOs := &fatalOs{t}
-
 	// Assure a clean start state
-	testOs.RemoveAll(filepath.Join("testdata", config.DefaultMarkerName))
-	testOs.MkdirAll(filepath.Join("testdata", config.DefaultMarkerName), 0644)
+	mustRemove(t, defaultFs.RemoveAll(config.DefaultMarkerName))
+	must(t, defaultFs.MkdirAll(config.DefaultMarkerName, 0644))
 
 	m := setupModel(defaultCfgWrapper)
 	defer cleanupModel(m)
@@ -2117,14 +2109,14 @@ func benchmarkTree(b *testing.B, n1, n2 int) {
 }
 
 func TestIssue3028(t *testing.T) {
-	testOs := &fatalOs{t}
-
 	// Create two files that we'll delete, one with a name that is a prefix of the other.
 
-	must(t, ioutil.WriteFile("testdata/testrm", []byte("Hello"), 0644))
-	defer testOs.Remove("testdata/testrm")
-	must(t, ioutil.WriteFile("testdata/testrm2", []byte("Hello"), 0644))
-	defer testOs.Remove("testdata/testrm2")
+	must(t, writeFile(defaultFs, "testrm", []byte("Hello"), 0644))
+	must(t, writeFile(defaultFs, "testrm2", []byte("Hello"), 0644))
+	defer func() {
+		mustRemove(t, defaultFs.Remove("testrm"))
+		mustRemove(t, defaultFs.Remove("testrm2"))
+	}()
 
 	// Create a model and default folder
 
@@ -2138,8 +2130,8 @@ func TestIssue3028(t *testing.T) {
 
 	// Delete and rescan specifically these two
 
-	testOs.Remove("testdata/testrm")
-	testOs.Remove("testdata/testrm2")
+	must(t, defaultFs.Remove("testrm"))
+	must(t, defaultFs.Remove("testrm2"))
 	m.ScanFolderSubdirs("default", []string{"testrm", "testrm2"})
 
 	// Verify that the number of files decreased by two and the number of
@@ -2601,7 +2593,7 @@ func TestIssue2571(t *testing.T) {
 
 	must(t, testFs.RemoveAll("toLink"))
 
-	must(t, osutil.DebugSymlinkForTestsOnly(filepath.Join(testFs.URI(), "linkTarget"), filepath.Join(testFs.URI(), "toLink")))
+	must(t, fs.DebugSymlinkForTestsOnly(testFs, testFs, "linkTarget", "toLink"))
 
 	m.ScanFolder("default")
 
@@ -2718,13 +2710,10 @@ func TestCustomMarkerName(t *testing.T) {
 		{Name: "dummyfile"},
 	})
 
-	fcfg := config.FolderConfiguration{
-		ID:              "default",
-		Path:            "rwtestfolder",
-		Type:            config.FolderTypeSendReceive,
-		RescanIntervalS: 1,
-		MarkerName:      "myfile",
-	}
+	fcfg := testFolderConfigTmp()
+	fcfg.ID = "default"
+	fcfg.RescanIntervalS = 1
+	fcfg.MarkerName = "myfile"
 	cfg := createTmpWrapper(config.Configuration{
 		Folders: []config.FolderConfiguration{fcfg},
 		Devices: []config.DeviceConfiguration{
@@ -2735,13 +2724,12 @@ func TestCustomMarkerName(t *testing.T) {
 	})
 
 	testOs.RemoveAll(fcfg.Path)
-	defer testOs.RemoveAll(fcfg.Path)
 
 	m := newModel(cfg, myID, "syncthing", "dev", ldb, nil)
 	sub := m.evLogger.Subscribe(events.StateChanged)
 	defer sub.Unsubscribe()
 	m.ServeBackground()
-	defer cleanupModel(m)
+	defer cleanupModelAndRemoveDir(m, fcfg.Path)
 
 	waitForState(t, sub, "default", "folder path missing")
 
@@ -3806,6 +3794,57 @@ func TestBlockListMap(t *testing.T) {
 	}
 }
 
+func TestScanRenameCaseOnly(t *testing.T) {
+	wcfg, fcfg := tmpDefaultWrapper()
+	m := setupModel(wcfg)
+	defer cleanupModel(m)
+
+	ffs := fcfg.Filesystem()
+	name := "foo"
+	must(t, writeFile(ffs, name, []byte("contents"), 0644))
+
+	m.ScanFolders()
+
+	snap := dbSnapshot(t, m, fcfg.ID)
+	defer snap.Release()
+	found := false
+	snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
+		if found {
+			t.Fatal("got more than one file")
+		}
+		if i.FileName() != name {
+			t.Fatalf("got file %v, expected %v", i.FileName(), name)
+		}
+		found = true
+		return true
+	})
+	snap.Release()
+
+	upper := strings.ToUpper(name)
+	must(t, ffs.Rename(name, upper))
+	m.ScanFolders()
+
+	snap = dbSnapshot(t, m, fcfg.ID)
+	defer snap.Release()
+	found = false
+	snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
+		if i.FileName() == name {
+			if i.IsDeleted() {
+				return true
+			}
+			t.Fatal("renamed file not deleted")
+		}
+		if i.FileName() != upper {
+			t.Fatalf("got file %v, expected %v", i.FileName(), upper)
+		}
+		if found {
+			t.Fatal("got more than the expected files")
+		}
+		found = true
+		return true
+	})
+}
+
 func TestConnectionTerminationOnFolderAdd(t *testing.T) {
 	testConfigChangeClosesConnections(t, false, true, nil, func(cfg config.Wrapper) {
 		fcfg := testFolderConfigTmp()

lib/model/requests_test.go

@@ -323,7 +323,7 @@ func pullInvalidIgnored(t *testing.T, ft config.FolderType) {
 	fc.deleteFile(invDel)
 	fc.addFile(ign, 0644, protocol.FileInfoTypeFile, contents)
 	fc.addFile(ignExisting, 0644, protocol.FileInfoTypeFile, contents)
-	if err := ioutil.WriteFile(filepath.Join(fss.URI(), ignExisting), otherContents, 0644); err != nil {
+	if err := writeFile(fss, ignExisting, otherContents, 0644); err != nil {
 		panic(err)
 	}
 
@@ -465,12 +465,12 @@ func TestIssue4841(t *testing.T) {
 
 func TestRescanIfHaveInvalidContent(t *testing.T) {
 	m, fc, fcfg := setupModelWithConnection()
-	tmpDir := fcfg.Filesystem().URI()
-	defer cleanupModelAndRemoveDir(m, tmpDir)
+	tfs := fcfg.Filesystem()
+	defer cleanupModelAndRemoveDir(m, tfs.URI())
 
 	payload := []byte("hello")
 
-	must(t, ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777))
+	must(t, writeFile(tfs, "foo", payload, 0777))
 
 	received := make(chan []protocol.FileInfo)
 	fc.mut.Lock()
@@ -511,7 +511,7 @@ func TestRescanIfHaveInvalidContent(t *testing.T) {
 	payload = []byte("bye")
 	buf = make([]byte, len(payload))
 
-	must(t, ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777))
+	must(t, writeFile(tfs, "foo", payload, 0777))
 
 	_, err = m.Request(device1, "default", "foo", int32(len(payload)), 0, f.Blocks[0].Hash, f.Blocks[0].WeakHash, false)
 	if err == nil {
@@ -1051,7 +1051,7 @@ func TestIgnoreDeleteUnignore(t *testing.T) {
 	}
 	fc.mut.Unlock()
 
-	if err := ioutil.WriteFile(filepath.Join(fss.URI(), file), contents, 0644); err != nil {
+	if err := writeFile(fss, file, contents, 0644); err != nil {
 		panic(err)
 	}
 	m.ScanFolders()

lib/model/testos_test.go

@@ -9,6 +9,8 @@ package model
 import (
 	"os"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/fs"
 )
 
 // fatal is the required common interface between *testing.B and *testing.T
@@ -28,6 +30,13 @@ func must(f fatal, err error) {
 	}
 }
 
+func mustRemove(f fatal, err error) {
+	f.Helper()
+	if err != nil && !fs.IsNotExist(err) {
+		f.Fatal(err)
+	}
+}
+
 func (f *fatalOs) Chmod(name string, mode os.FileMode) {
 	f.Helper()
 	must(f, os.Chmod(name, mode))

lib/model/testutils_test.go

@@ -36,9 +36,8 @@ func init() {
 	device1, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
 	device2, _ = protocol.DeviceIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
 
-	defaultFs = fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata")
-
 	defaultFolderConfig = testFolderConfig("testdata")
+	defaultFs = defaultFolderConfig.Filesystem()
 
 	defaultCfgWrapper = createTmpWrapper(config.New(myID))
 	_, _ = defaultCfgWrapper.SetDevice(config.NewDeviceConfiguration(device1, "device1"))

lib/osutil/osutil.go

@@ -131,8 +131,10 @@ func copyFileContents(method fs.CopyRangeMethod, srcFs, dstFs fs.Filesystem, src
 }
 
 func IsDeleted(ffs fs.Filesystem, name string) bool {
-	if _, err := ffs.Lstat(name); fs.IsNotExist(err) {
-		return true
+	if _, err := ffs.Lstat(name); err != nil {
+		if fs.IsNotExist(err) || fs.IsErrCaseConflict(err) {
+			return true
+		}
 	}
 	switch TraversesSymlink(ffs, filepath.Dir(name)).(type) {
 	case *NotADirectoryError, *TraversesSymlinkError:

lib/osutil/osutil_test.go

@@ -62,7 +62,7 @@ func TestIsDeleted(t *testing.T) {
 		}
 	}
 	for _, n := range []string{"Dir", "File", "Del"} {
-		if err := osutil.DebugSymlinkForTestsOnly(filepath.Join(testFs.URI(), strings.ToLower(n)), filepath.Join(testFs.URI(), "linkTo"+n)); err != nil {
+		if err := fs.DebugSymlinkForTestsOnly(testFs, testFs, strings.ToLower(n), "linkTo"+n); err != nil {
 			if runtime.GOOS == "windows" {
 				t.Skip("Symlinks aren't working")
 			}

lib/osutil/symlink.go

@@ -1,21 +0,0 @@
-// Copyright (C) 2017 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build !windows
-
-package osutil
-
-import (
-	"os"
-)
-
-// DebugSymlinkForTestsOnly is not and should not be used in Syncthing code,
-// hence the cumbersome name to make it obvious if this ever leaks. Its
-// reason for existence is the Windows version, which allows creating
-// symlinks when non-elevated.
-func DebugSymlinkForTestsOnly(oldname, newname string) error {
-	return os.Symlink(oldname, newname)
-}

lib/osutil/traversessymlink_test.go

@@ -24,9 +24,9 @@ func TestTraversesSymlink(t *testing.T) {
 	}
 	defer os.RemoveAll(tmpDir)
 
-	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir)
-	fs.MkdirAll("a/b/c", 0755)
-	if err = osutil.DebugSymlinkForTestsOnly(filepath.Join(fs.URI(), "a", "b"), filepath.Join(fs.URI(), "a", "l")); err != nil {
+	testFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir)
+	testFs.MkdirAll("a/b/c", 0755)
+	if err = fs.DebugSymlinkForTestsOnly(testFs, testFs, filepath.Join("a", "b"), filepath.Join("a", "l")); err != nil {
 		if runtime.GOOS == "windows" {
 			t.Skip("Symlinks aren't working")
 		}
@@ -34,7 +34,7 @@ func TestTraversesSymlink(t *testing.T) {
 	}
 
 	// a/l -> b, so a/l/c should resolve by normal stat
-	info, err := fs.Lstat("a/l/c")
+	info, err := testFs.Lstat("a/l/c")
 	if err != nil {
 		t.Fatal("unexpected error", err)
 	}
@@ -64,7 +64,7 @@ func TestTraversesSymlink(t *testing.T) {
 	}
 
 	for _, tc := range cases {
-		if res := osutil.TraversesSymlink(fs, tc.name); tc.traverses == (res == nil) {
+		if res := osutil.TraversesSymlink(testFs, tc.name); tc.traverses == (res == nil) {
 			t.Errorf("TraversesSymlink(%q) = %v, should be %v", tc.name, res, tc.traverses)
 		}
 	}
@@ -78,8 +78,8 @@ func TestIssue4875(t *testing.T) {
 	defer os.RemoveAll(tmpDir)
 
 	testFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir)
-	testFs.MkdirAll("a/b/c", 0755)
-	if err = osutil.DebugSymlinkForTestsOnly(filepath.Join(testFs.URI(), "a", "b"), filepath.Join(testFs.URI(), "a", "l")); err != nil {
+	testFs.MkdirAll(filepath.Join("a", "b", "c"), 0755)
+	if err = fs.DebugSymlinkForTestsOnly(testFs, testFs, filepath.Join("a", "b"), filepath.Join("a", "l")); err != nil {
 		if runtime.GOOS == "windows" {
 			t.Skip("Symlinks aren't working")
 		}

lib/scanner/walk.go

@@ -540,6 +540,10 @@ func (w *walker) handleError(ctx context.Context, context, path string, err erro
 	}
 }
 
+func (w *walker) String() string {
+	return fmt.Sprintf("walker/%s@%p", w.Folder, w)
+}
+
 // A byteCounter gets bytes added to it via Update() and then provides the
 // Total() and one minute moving average Rate() in bytes per second.
 type byteCounter struct {

lib/scanner/walk_test.go

@@ -26,7 +26,6 @@ import (
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/ignore"
-	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sha256"
 	"golang.org/x/text/unicode/norm"
@@ -40,17 +39,19 @@ type testfile struct {
 
 type testfileList []testfile
 
-var testFs fs.Filesystem
-
-var testdata = testfileList{
-	{"afile", 4, "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"},
-	{"dir1", 128, ""},
-	{filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"},
-	{"dir2", 128, ""},
-	{filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"},
-	{"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"},
-	{"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"},
-}
+var (
+	testFs     fs.Filesystem
+	testFsType = fs.FilesystemTypeBasic
+	testdata   = testfileList{
+		{"afile", 4, "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"},
+		{"dir1", 128, ""},
+		{filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"},
+		{"dir2", 128, ""},
+		{filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"},
+		{"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"},
+		{"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"},
+	}
+)
 
 func init() {
 	// This test runs the risk of entering infinite recursion if it fails.
@@ -270,7 +271,7 @@ func TestWalkSymlinkUnix(t *testing.T) {
 	defer os.RemoveAll("_symlinks")
 	os.Symlink("../testdata", "_symlinks/link")
 
-	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks")
+	fs := fs.NewFilesystem(testFsType, "_symlinks")
 	for _, path := range []string{".", "link"} {
 		// Scan it
 		files := walkDir(fs, path, nil, nil, 0)
@@ -298,15 +299,15 @@ func TestWalkSymlinkWindows(t *testing.T) {
 	os.RemoveAll(name)
 	os.Mkdir(name, 0755)
 	defer os.RemoveAll(name)
-	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, name)
-	if err := osutil.DebugSymlinkForTestsOnly("../testdata", "_symlinks/link"); err != nil {
+	testFs := fs.NewFilesystem(testFsType, name)
+	if err := fs.DebugSymlinkForTestsOnly(testFs, testFs, "../testdata", "link"); err != nil {
 		// Probably we require permissions we don't have.
 		t.Skip(err)
 	}
 
 	for _, path := range []string{".", "link"} {
 		// Scan it
-		files := walkDir(fs, path, nil, nil, 0)
+		files := walkDir(testFs, path, nil, nil, 0)
 
 		// Verify that we got zero symlinks
 		if len(files) != 0 {
@@ -322,10 +323,12 @@ func TestWalkRootSymlink(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(tmp)
+	testFs := fs.NewFilesystem(testFsType, tmp)
 
-	link := filepath.Join(tmp, "link")
+	link := "link"
 	dest, _ := filepath.Abs("testdata/dir1")
-	if err := osutil.DebugSymlinkForTestsOnly(dest, link); err != nil {
+	destFs := fs.NewFilesystem(testFsType, dest)
+	if err := fs.DebugSymlinkForTestsOnly(destFs, testFs, ".", "link"); err != nil {
 		if runtime.GOOS == "windows" {
 			// Probably we require permissions we don't have.
 			t.Skip("Need admin permissions or developer mode to run symlink test on Windows: " + err.Error())
@@ -335,15 +338,15 @@ func TestWalkRootSymlink(t *testing.T) {
 	}
 
 	// Scan root with symlink at FS root
-	files := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil, nil, 0)
+	files := walkDir(fs.NewFilesystem(testFsType, filepath.Join(testFs.URI(), link)), ".", nil, nil, 0)
 
 	// Verify that we got two files
 	if len(files) != 2 {
-		t.Errorf("expected two files, not %d", len(files))
+		t.Fatalf("expected two files, not %d", len(files))
 	}
 
 	// Scan symlink below FS root
-	files = walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, tmp), "link", nil, nil, 0)
+	files = walkDir(testFs, "link", nil, nil, 0)
 
 	// Verify that we got the one symlink, except on windows
 	if runtime.GOOS == "windows" {
@@ -355,7 +358,7 @@ func TestWalkRootSymlink(t *testing.T) {
 	}
 
 	// Scan path below symlink
-	files = walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, tmp), filepath.Join("link", "cfile"), nil, nil, 0)
+	files = walkDir(fs.NewFilesystem(testFsType, tmp), filepath.Join("link", "cfile"), nil, nil, 0)
 
 	// Verify that we get nothing
 	if len(files) != 0 {
@@ -554,7 +557,7 @@ func BenchmarkHashFile(b *testing.B) {
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
-		if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.MinBlockSize, nil, true); err != nil {
+		if _, err := HashFile(context.TODO(), fs.NewFilesystem(testFsType, ""), testdataName, protocol.MinBlockSize, nil, true); err != nil {
 			b.Fatal(err)
 		}
 	}
@@ -652,7 +655,7 @@ func TestIssue4799(t *testing.T) {
 	}
 	defer os.RemoveAll(tmp)
 
-	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmp)
+	fs := fs.NewFilesystem(testFsType, tmp)
 
 	fd, err := fs.Create("foo")
 	if err != nil {
@@ -714,7 +717,7 @@ func TestIssue4841(t *testing.T) {
 	}
 	defer os.RemoveAll(tmp)
 
-	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmp)
+	fs := fs.NewFilesystem(testFsType, tmp)
 
 	fd, err := fs.Create("foo")
 	if err != nil {

lib/ur/contract/contract.go

@@ -128,6 +128,7 @@ type Report struct {
 		DisableFsync            int            `json:"disableFsync,omitempty" since:"3"`
 		BlockPullOrder          map[string]int `json:"blockPullOrder,omitempty" since:"3"`
 		CopyRangeMethod         map[string]int `json:"copyRangeMethod,omitempty" since:"3"`
+		CaseSensitiveFS         int            `json:"caseSensitiveFS,omitempty" since:"3"`
 	} `json:"folderUsesV3,omitempty" since:"3"`
 
 	GUIStats struct {

lib/ur/usage_report.go

@@ -269,6 +269,9 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (
 			}
 			report.FolderUsesV3.BlockPullOrder[cfg.BlockPullOrder.String()]++
 			report.FolderUsesV3.CopyRangeMethod[cfg.CopyRangeMethod.String()]++
+			if cfg.CaseSensitiveFS {
+				report.FolderUsesV3.CaseSensitiveFS++
+			}
 		}
 		sort.Ints(report.FolderUsesV3.FsWatcherDelays)
 

test/transfer-bench_test.go

@@ -10,42 +10,42 @@ package integration
 
 import (
 	"log"
+	"math/rand"
 	"os"
 	"testing"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/rc"
 )
 
 func TestBenchmarkTransferManyFiles(t *testing.T) {
-	benchmarkTransfer(t, 10000, 15)
+	setupAndBenchmarkTransfer(t, 10000, 15)
 }
 
 func TestBenchmarkTransferLargeFile1G(t *testing.T) {
-	benchmarkTransfer(t, 1, 30)
+	setupAndBenchmarkTransfer(t, 1, 30)
 }
 func TestBenchmarkTransferLargeFile2G(t *testing.T) {
-	benchmarkTransfer(t, 1, 31)
+	setupAndBenchmarkTransfer(t, 1, 31)
 }
 func TestBenchmarkTransferLargeFile4G(t *testing.T) {
-	benchmarkTransfer(t, 1, 32)
+	setupAndBenchmarkTransfer(t, 1, 32)
 }
 func TestBenchmarkTransferLargeFile8G(t *testing.T) {
-	benchmarkTransfer(t, 1, 33)
+	setupAndBenchmarkTransfer(t, 1, 33)
 }
 func TestBenchmarkTransferLargeFile16G(t *testing.T) {
-	benchmarkTransfer(t, 1, 34)
+	setupAndBenchmarkTransfer(t, 1, 34)
 }
 func TestBenchmarkTransferLargeFile32G(t *testing.T) {
-	benchmarkTransfer(t, 1, 35)
+	setupAndBenchmarkTransfer(t, 1, 35)
 }
 
-func benchmarkTransfer(t *testing.T, files, sizeExp int) {
-	log.Println("Cleaning...")
-	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
-	if err != nil {
-		t.Fatal(err)
-	}
+func setupAndBenchmarkTransfer(t *testing.T, files, sizeExp int) {
+	cleanBenchmarkTransfer(t)
 
 	log.Println("Generating files...")
+	var err error
 	if files == 1 {
 		// Special case. Generate one file with the specified size exactly.
 		var fd *os.File
@@ -57,13 +57,39 @@ func benchmarkTransfer(t *testing.T, files, sizeExp int) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		err = generateOneFile(fd, "s1/onefile", 1<<uint(sizeExp))
+		err = generateOneFile(fd, "s1/onefile", 1<<uint(sizeExp), time.Now())
 	} else {
 		err = generateFiles("s1", files, sizeExp, "../LICENSE")
 	}
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	benchmarkTransfer(t)
+}
+
+// TestBenchmarkTransferSameFiles doesn't actually transfer anything, but tests
+// how fast two devicees get in sync if they have the same data locally.
+func TestBenchmarkTransferSameFiles(t *testing.T) {
+	cleanBenchmarkTransfer(t)
+
+	t0 := time.Now()
+	rand.Seed(0)
+	log.Println("Generating files in s1...")
+	if err := generateFilesWithTime("s1", 10000, 10, "../LICENSE", t0); err != nil {
+		t.Fatal(err)
+	}
+
+	rand.Seed(0)
+	log.Println("Generating same files in s2...")
+	if err := generateFilesWithTime("s2", 10000, 10, "../LICENSE", t0); err != nil {
+		t.Fatal(err)
+	}
+
+	benchmarkTransfer(t)
+}
+
+func benchmarkTransfer(t *testing.T) {
 	expected, err := directoryContents("s1")
 	if err != nil {
 		t.Fatal(err)
@@ -86,9 +112,9 @@ func benchmarkTransfer(t *testing.T, files, sizeExp int) {
 	sender.ResumeAll()
 	receiver.ResumeAll()
 
-	var t0, t1 time.Time
+	t0 := time.Now()
+	var t1 time.Time
 	lastEvent := 0
-	oneItemFinished := false
 
 loop:
 	for {
@@ -105,35 +131,22 @@ loop:
 
 			switch ev.Type {
 			case "ItemFinished":
-				oneItemFinished = true
-				continue
-
-			case "StateChanged":
-				data := ev.Data.(map[string]interface{})
-				if data["folder"].(string) != "default" {
-					continue
-				}
-
-				switch data["to"].(string) {
-				case "syncing":
-					t0 = ev.Time
-					continue
-
-				case "idle":
-					if !oneItemFinished {
-						continue
-					}
-					if !t0.IsZero() {
-						t1 = ev.Time
-						break loop
-					}
-				}
+				break loop
 			}
 		}
 
 		time.Sleep(250 * time.Millisecond)
 	}
 
+	processes := []*rc.Process{sender, receiver}
+	for {
+		if rc.InSync("default", processes...) {
+			t1 = time.Now()
+			break
+		}
+		time.Sleep(250 * time.Millisecond)
+	}
+
 	sendProc, err := sender.Stop()
 	if err != nil {
 		t.Fatal(err)
@@ -159,4 +172,14 @@ loop:
 
 	printUsage("Receiver", recvProc, total)
 	printUsage("Sender", sendProc, total)
+
+	cleanBenchmarkTransfer(t)
+}
+
+func cleanBenchmarkTransfer(t *testing.T) {
+	log.Println("Cleaning...")
+	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
+	if err != nil {
+		t.Fatal(err)
+	}
 }

test/util.go

@@ -41,6 +41,10 @@ const (
 )
 
 func generateFiles(dir string, files, maxexp int, srcname string) error {
+	return generateFilesWithTime(dir, files, maxexp, srcname, time.Now())
+}
+
+func generateFilesWithTime(dir string, files, maxexp int, srcname string, t0 time.Time) error {
 	fd, err := os.Open(srcname)
 	if err != nil {
 		return err
@@ -69,7 +73,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
 		}
 		s += rand.Int63n(a)
 
-		if err := generateOneFile(fd, p1, s); err != nil {
+		if err := generateOneFile(fd, p1, s, t0); err != nil {
 			return err
 		}
 	}
@@ -77,7 +81,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
 	return nil
 }
 
-func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
+func generateOneFile(fd io.ReadSeeker, p1 string, s int64, t0 time.Time) error {
 	src := io.LimitReader(&inifiteReader{fd}, int64(s))
 	dst, err := os.Create(p1)
 	if err != nil {
@@ -96,7 +100,7 @@ func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
 
 	os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
 
-	t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
+	t := t0.Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
 	err = os.Chtimes(p1, t, t)
 	if err != nil {
 		return err