all: Convert folders to use filesystem abstraction

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4228

cmd/stcli/cmd_folders.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/AudriusButkevicius/cli"
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/fs"
 )
 
 func init() {
@@ -102,8 +103,10 @@ func foldersList(c *cli.Context) {
 		if !first {
 			fmt.Fprintln(writer)
 		}
+		fs := folder.Filesystem()
 		fmt.Fprintln(writer, "ID:\t", folder.ID, "\t")
-		fmt.Fprintln(writer, "Path:\t", folder.RawPath, "\t(directory)")
+		fmt.Fprintln(writer, "Path:\t", fs.URI(), "\t(directory)")
+		fmt.Fprintln(writer, "Path type:\t", fs.Type(), "\t(directory-type)")
 		fmt.Fprintln(writer, "Folder type:\t", folder.Type, "\t(type)")
 		fmt.Fprintln(writer, "Ignore permissions:\t", folder.IgnorePerms, "\t(permissions)")
 		fmt.Fprintln(writer, "Rescan interval in seconds:\t", folder.RescanIntervalS, "\t(rescan)")
@@ -124,8 +127,9 @@ func foldersAdd(c *cli.Context) {
 	abs, err := filepath.Abs(c.Args()[1])
 	die(err)
 	folder := config.FolderConfiguration{
-		ID:      c.Args()[0],
+		ID:             c.Args()[0],
-		RawPath: filepath.Clean(abs),
+		Path:           filepath.Clean(abs),
+		FilesystemType: fs.FilesystemTypeBasic,
 	}
 	cfg.Folders = append(cfg.Folders, folder)
 	setConfig(c, cfg)
@@ -185,7 +189,9 @@ func foldersGet(c *cli.Context) {
 		}
 		switch arg {
 		case "directory":
-			fmt.Println(folder.RawPath)
+			fmt.Println(folder.Filesystem().URI())
+		case "directory-type":
+			fmt.Println(folder.Filesystem().Type())
 		case "type":
 			fmt.Println(folder.Type)
 		case "permissions":
@@ -197,7 +203,7 @@ func foldersGet(c *cli.Context) {
 				fmt.Println(folder.Versioning.Type)
 			}
 		default:
-			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, type, permissions, versioning, versioning-<key>")
+			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, directory-type, type, permissions, versioning, versioning-<key>")
 		}
 		return
 	}
@@ -220,7 +226,11 @@ func foldersSet(c *cli.Context) {
 		}
 		switch arg {
 		case "directory":
-			cfg.Folders[i].RawPath = val
+			cfg.Folders[i].Path = val
+		case "directory-type":
+			var fsType fs.FilesystemType
+			fsType.UnmarshalText([]byte(val))
+			cfg.Folders[i].FilesystemType = fsType
 		case "type":
 			var t config.FolderType
 			if err := t.UnmarshalText([]byte(val)); err != nil {

cmd/stindex/util.go

@@ -12,7 +12,7 @@ import (
 	"path/filepath"
 	"runtime"
 
-	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/fs"
 )
 
 func nulString(bs []byte) string {
@@ -33,7 +33,7 @@ func defaultConfigDir() string {
 		return filepath.Join(os.Getenv("AppData"), "Syncthing")
 
 	case "darwin":
-		dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing")
+		dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing")
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -43,7 +43,7 @@ func defaultConfigDir() string {
 		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
 			return filepath.Join(xdgCfg, "syncthing")
 		}
-		dir, err := osutil.ExpandTilde("~/.config/syncthing")
+		dir, err := fs.ExpandTilde("~/.config/syncthing")
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/syncthing/gui.go

@@ -28,9 +28,9 @@ import (
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/discover"
 	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/logger"
 	"github.com/syncthing/syncthing/lib/model"
-	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/stats"
@@ -856,7 +856,7 @@ func (s *apiService) getSystemStatus(w http.ResponseWriter, r *http.Request) {
 	var m runtime.MemStats
 	runtime.ReadMemStats(&m)
 
-	tilde, _ := osutil.ExpandTilde("~")
+	tilde, _ := fs.ExpandTilde("~")
 	res := make(map[string]interface{})
 	res["myID"] = myID.String()
 	res["goroutines"] = runtime.NumGoroutine()
@@ -1259,23 +1259,35 @@ func (s *apiService) getPeerCompletion(w http.ResponseWriter, r *http.Request) {
 func (s *apiService) getSystemBrowse(w http.ResponseWriter, r *http.Request) {
 	qs := r.URL.Query()
 	current := qs.Get("current")
+	// Default value or in case of error unmarshalling ends up being basic fs.
+	var fsType fs.FilesystemType
+	fsType.UnmarshalText([]byte(qs.Get("filesystem")))
+
 	if current == "" {
-		if roots, err := osutil.GetFilesystemRoots(); err == nil {
+		filesystem := fs.NewFilesystem(fsType, "")
+		if roots, err := filesystem.Roots(); err == nil {
 			sendJSON(w, roots)
 		} else {
 			http.Error(w, err.Error(), 500)
 		}
 		return
 	}
-	search, _ := osutil.ExpandTilde(current)
+	search, _ := fs.ExpandTilde(current)
-	pathSeparator := string(os.PathSeparator)
+	pathSeparator := string(fs.PathSeparator)
+
 	if strings.HasSuffix(current, pathSeparator) && !strings.HasSuffix(search, pathSeparator) {
 		search = search + pathSeparator
 	}
-	subdirectories, _ := osutil.Glob(search + "*")
+	searchDir := filepath.Dir(search)
+	searchFile := filepath.Base(search)
+
+	fs := fs.NewFilesystem(fsType, searchDir)
+
+	subdirectories, _ := fs.Glob(searchFile + "*")
+
 	ret := make([]string, 0, len(subdirectories))
 	for _, subdirectory := range subdirectories {
-		info, err := os.Stat(subdirectory)
+		info, err := fs.Stat(subdirectory)
 		if err == nil && info.IsDir() {
 			ret = append(ret, subdirectory+pathSeparator)
 		}

cmd/syncthing/locations.go

@@ -13,7 +13,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/fs"
 )
 
 type locationEnum string
@@ -65,7 +65,7 @@ func expandLocations() error {
 			dir = strings.Replace(dir, "${"+varName+"}", value, -1)
 		}
 		var err error
-		dir, err = osutil.ExpandTilde(dir)
+		dir, err = fs.ExpandTilde(dir)
 		if err != nil {
 			return err
 		}
@@ -86,7 +86,7 @@ func defaultConfigDir() string {
 		return filepath.Join(os.Getenv("AppData"), "Syncthing")
 
 	case "darwin":
-		dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing")
+		dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing")
 		if err != nil {
 			l.Fatalln(err)
 		}
@@ -96,7 +96,7 @@ func defaultConfigDir() string {
 		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
 			return filepath.Join(xdgCfg, "syncthing")
 		}
-		dir, err := osutil.ExpandTilde("~/.config/syncthing")
+		dir, err := fs.ExpandTilde("~/.config/syncthing")
 		if err != nil {
 			l.Fatalln(err)
 		}
@@ -106,7 +106,7 @@ func defaultConfigDir() string {
 
 // homeDir returns the user's home directory, or dies trying.
 func homeDir() string {
-	home, err := osutil.ExpandTilde("~")
+	home, err := fs.ExpandTilde("~")
 	if err != nil {
 		l.Fatalln(err)
 	}

cmd/syncthing/main.go

@@ -37,6 +37,7 @@ import (
 	"github.com/syncthing/syncthing/lib/dialer"
 	"github.com/syncthing/syncthing/lib/discover"
 	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/logger"
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/osutil"
@@ -444,7 +445,7 @@ func openGUI() {
 }
 
 func generate(generateDir string) {
-	dir, err := osutil.ExpandTilde(generateDir)
+	dir, err := fs.ExpandTilde(generateDir)
 	if err != nil {
 		l.Fatalln("generate:", err)
 	}
@@ -1085,7 +1086,7 @@ func defaultConfig(myName string) config.Configuration {
 
 	if !noDefaultFolder {
 		l.Infoln("Default folder created and/or linked to new config")
-		defaultFolder = config.NewFolderConfiguration("default", locations[locDefFolder])
+		defaultFolder = config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, locations[locDefFolder])
 		defaultFolder.Label = "Default Folder"
 		defaultFolder.RescanIntervalS = 60
 		defaultFolder.MinDiskFree = config.Size{Value: 1, Unit: "%"}
@@ -1141,19 +1142,20 @@ func shutdown() {
 	stop <- exitSuccess
 }
 
-func ensureDir(dir string, mode os.FileMode) {
+func ensureDir(dir string, mode fs.FileMode) {
-	err := osutil.MkdirAll(dir, mode)
+	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
+	err := fs.MkdirAll(".", mode)
 	if err != nil {
 		l.Fatalln(err)
 	}
 
-	if fi, err := os.Stat(dir); err == nil {
+	if fi, err := fs.Stat("."); err == nil {
 		// Apprently the stat may fail even though the mkdirall passed. If it
 		// does, we'll just assume things are in order and let other things
 		// fail (like loading or creating the config...).
 		currentMode := fi.Mode() & 0777
 		if currentMode != mode {
-			err := os.Chmod(dir, mode)
+			err := fs.Chmod(".", mode)
 			// This can fail on crappy filesystems, nothing we can do about it.
 			if err != nil {
 				l.Warnln(err)
@@ -1276,22 +1278,22 @@ func cleanConfigDirectory() {
 	}
 
 	for pat, dur := range patterns {
-		pat = filepath.Join(baseDirs["config"], pat)
+		fs := fs.NewFilesystem(fs.FilesystemTypeBasic, baseDirs["config"])
-		files, err := osutil.Glob(pat)
+		files, err := fs.Glob(pat)
 		if err != nil {
 			l.Infoln("Cleaning:", err)
 			continue
 		}
 
 		for _, file := range files {
-			info, err := osutil.Lstat(file)
+			info, err := fs.Lstat(file)
 			if err != nil {
 				l.Infoln("Cleaning:", err)
 				continue
 			}
 
 			if time.Since(info.ModTime()) > dur {
-				if err = os.RemoveAll(file); err != nil {
+				if err = fs.RemoveAll(file); err != nil {
 					l.Infoln("Cleaning:", err)
 				} else {
 					l.Infoln("Cleaned away old file", filepath.Base(file))

gui/default/syncthing/core/notifications.html

@@ -40,4 +40,4 @@
       <div class="clearfix"></div>
     </div>
   </div>
-</notification>
+</notification>

gui/default/syncthing/folder/editFolderModalView.html

@@ -180,7 +180,7 @@
               <label translate for="externalCommand">Command</label>
               <input name="externalCommand" id="externalCommand" class="form-control" type="text" ng-model="currentFolder.externalCommand" required="" aria-required="true" />
               <p class="help-block">
-                <span translate ng-if="folderEditor.externalCommand.$valid || folderEditor.externalCommand.$pristine">The first command line parameter is the folder path and the second parameter is the relative path in the folder.</span>
+                <span translate ng-if="folderEditor.externalCommand.$valid || folderEditor.externalCommand.$pristine">See external versioner help for supported templated command line parameters.</span>
                 <span translate ng-if="folderEditor.externalCommand.$error.required && folderEditor.externalCommand.$dirty">The path cannot be blank.</span>
               </p>
             </div>

lib/config/config.go

@@ -23,6 +23,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/upgrade"
@@ -31,7 +32,7 @@ import (
 
 const (
 	OldestHandledVersion = 10
-	CurrentVersion       = 21
+	CurrentVersion       = 22
 	MaxRescanIntervalS   = 365 * 24 * 60 * 60
 )
 
@@ -319,6 +320,9 @@ func (cfg *Configuration) clean() error {
 	if cfg.Version == 20 {
 		convertV20V21(cfg)
 	}
+	if cfg.Version == 21 {
+		convertV21V22(cfg)
+	}
 
 	// Build a list of available devices
 	existingDevices := make(map[protocol.DeviceID]bool)
@@ -368,23 +372,38 @@ func (cfg *Configuration) clean() error {
 	return nil
 }
 
+func convertV21V22(cfg *Configuration) {
+	for i := range cfg.Folders {
+		cfg.Folders[i].FilesystemType = fs.FilesystemTypeBasic
+		// Migrate to templated external versioner commands
+		if cfg.Folders[i].Versioning.Type == "external" {
+			cfg.Folders[i].Versioning.Params["command"] += " %FOLDER_PATH% %FILE_PATH%"
+		}
+	}
+
+	cfg.Version = 22
+}
+
 func convertV20V21(cfg *Configuration) {
 	for _, folder := range cfg.Folders {
+		if folder.FilesystemType != fs.FilesystemTypeBasic {
+			continue
+		}
 		switch folder.Versioning.Type {
 		case "simple", "trashcan":
 			// Clean out symlinks in the known place
-			cleanSymlinks(filepath.Join(folder.Path(), ".stversions"))
+			cleanSymlinks(folder.Filesystem(), ".stversions")
 		case "staggered":
 			versionDir := folder.Versioning.Params["versionsPath"]
 			if versionDir == "" {
 				// default place
-				cleanSymlinks(filepath.Join(folder.Path(), ".stversions"))
+				cleanSymlinks(folder.Filesystem(), ".stversions")
 			} else if filepath.IsAbs(versionDir) {
 				// absolute
-				cleanSymlinks(versionDir)
+				cleanSymlinks(fs.NewFilesystem(fs.FilesystemTypeBasic, versionDir), ".")
 			} else {
 				// relative to folder
-				cleanSymlinks(filepath.Join(folder.Path(), versionDir))
+				cleanSymlinks(folder.Filesystem(), versionDir)
 			}
 		}
 	}
@@ -428,9 +447,7 @@ func convertV17V18(cfg *Configuration) {
 }
 
 func convertV16V17(cfg *Configuration) {
-	for i := range cfg.Folders {
+	// Fsync = true removed
-		cfg.Folders[i].Fsync = true
-	}
 
 	cfg.Version = 17
 }
@@ -670,21 +687,21 @@ loop:
 	return devices[0:count]
 }
 
-func cleanSymlinks(dir string) {
+func cleanSymlinks(filesystem fs.Filesystem, dir string) {
 	if runtime.GOOS == "windows" {
 		// We don't do symlinks on Windows. Additionally, there may
 		// be things that look like symlinks that are not, which we
 		// should leave alone. Deduplicated files, for example.
 		return
 	}
-	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+	filesystem.Walk(dir, func(path string, info fs.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
-		if info.Mode()&os.ModeSymlink != 0 {
+		if info.IsSymlink() {
 			l.Infoln("Removing incorrectly versioned symlink", path)
-			os.Remove(path)
+			filesystem.Remove(path)
-			return filepath.SkipDir
+			return fs.SkipDir
 		}
 		return nil
 	})

lib/config/config_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/d4l3k/messagediff"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -103,7 +104,8 @@ func TestDeviceConfig(t *testing.T) {
 		expectedFolders := []FolderConfiguration{
 			{
 				ID:              "test",
-				RawPath:         "testdata",
+				FilesystemType:  fs.FilesystemTypeBasic,
+				Path:            "testdata",
 				Devices:         []FolderDeviceConfiguration{{DeviceID: device1}, {DeviceID: device4}},
 				Type:            FolderTypeSendOnly,
 				RescanIntervalS: 600,
@@ -113,7 +115,6 @@ func TestDeviceConfig(t *testing.T) {
 				AutoNormalize:   true,
 				MinDiskFree:     Size{1, "%"},
 				MaxConflicts:    -1,
-				Fsync:           true,
 				Versioning: VersioningConfiguration{
 					Params: map[string]string{},
 				},
@@ -121,15 +122,11 @@ func TestDeviceConfig(t *testing.T) {
 			},
 		}
 
-		// The cachedPath will have been resolved to an absolute path,
+		// The cachedFilesystem will have been resolved to an absolute path,
 		// depending on where the tests are running. Zero it out so we don't
 		// fail based on that.
 		for i := range cfg.Folders {
-			cfg.Folders[i].cachedPath = ""
+			cfg.Folders[i].cachedFilesystem = nil
-		}
-
-		if runtime.GOOS != "windows" {
-			expectedFolders[0].RawPath += string(filepath.Separator)
 		}
 
 		expectedDevices := []DeviceConfiguration{
@@ -377,16 +374,17 @@ func TestVersioningConfig(t *testing.T) {
 }
 
 func TestIssue1262(t *testing.T) {
+	if runtime.GOOS != "windows" {
+		t.Skipf("path gets converted to absolute as part of the filesystem initialization on linux")
+	}
+
 	cfg, err := Load("testdata/issue-1262.xml", device4)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	actual := cfg.Folders()["test"].RawPath
+	actual := cfg.Folders()["test"].Filesystem().URI()
-	expected := "e:/"
+	expected := `e:\`
-	if runtime.GOOS == "windows" {
-		expected = `e:\`
-	}
 
 	if actual != expected {
 		t.Errorf("%q != %q", actual, expected)
@@ -416,43 +414,12 @@ func TestIssue1750(t *testing.T) {
 	}
 }
 
-func TestWindowsPaths(t *testing.T) {
-	if runtime.GOOS != "windows" {
-		t.Skip("Not useful on non-Windows")
-		return
-	}
-
-	folder := FolderConfiguration{
-		RawPath: `e:\`,
-	}
-
-	expected := `\\?\e:\`
-	actual := folder.Path()
-	if actual != expected {
-		t.Errorf("%q != %q", actual, expected)
-	}
-
-	folder.RawPath = `\\192.0.2.22\network\share`
-	expected = folder.RawPath
-	actual = folder.Path()
-	if actual != expected {
-		t.Errorf("%q != %q", actual, expected)
-	}
-
-	folder.RawPath = `relative\path`
-	expected = folder.RawPath
-	actual = folder.Path()
-	if actual == expected || !strings.HasPrefix(actual, "\\\\?\\") {
-		t.Errorf("%q == %q, expected absolutification", actual, expected)
-	}
-}
-
 func TestFolderPath(t *testing.T) {
 	folder := FolderConfiguration{
-		RawPath: "~/tmp",
+		Path: "~/tmp",
 	}
 
-	realPath := folder.Path()
+	realPath := folder.Filesystem().URI()
 	if !filepath.IsAbs(realPath) {
 		t.Error(realPath, "should be absolute")
 	}
@@ -677,8 +644,8 @@ func TestEmptyFolderPaths(t *testing.T) {
 		t.Fatal(err)
 	}
 	folder := wrapper.Folders()["f1"]
-	if folder.Path() != "" {
+	if folder.cachedFilesystem != nil {
-		t.Errorf("Expected %q to be empty", folder.Path())
+		t.Errorf("Expected %q to be empty", folder.cachedFilesystem)
 	}
 }
 

lib/config/folderconfiguration.go

@@ -8,19 +8,17 @@ package config
 
 import (
 	"fmt"
-	"os"
-	"path/filepath"
 	"runtime"
-	"strings"
 
-	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
 type FolderConfiguration struct {
 	ID                    string                      `xml:"id,attr" json:"id"`
 	Label                 string                      `xml:"label,attr" json:"label"`
-	RawPath               string                      `xml:"path,attr" json:"path"`
+	FilesystemType        fs.FilesystemType           `xml:"filesystemType" json:"filesystemType"`
+	Path                  string                      `xml:"path,attr" json:"path"`
 	Type                  FolderType                  `xml:"type,attr" json:"type"`
 	Devices               []FolderDeviceConfiguration `xml:"device" json:"devices"`
 	RescanIntervalS       int                         `xml:"rescanIntervalS,attr" json:"rescanIntervalS"`
@@ -39,11 +37,10 @@ type FolderConfiguration struct {
 	MaxConflicts          int                         `xml:"maxConflicts" json:"maxConflicts"`
 	DisableSparseFiles    bool                        `xml:"disableSparseFiles" json:"disableSparseFiles"`
 	DisableTempIndexes    bool                        `xml:"disableTempIndexes" json:"disableTempIndexes"`
-	Fsync                 bool                        `xml:"fsync" json:"fsync"`
 	Paused                bool                        `xml:"paused" json:"paused"`
 	WeakHashThresholdPct  int                         `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash.
 
-	cachedPath string
+	cachedFilesystem fs.Filesystem
 
 	DeprecatedReadOnly       bool    `xml:"ro,attr,omitempty" json:"-"`
 	DeprecatedMinDiskFreePct float64 `xml:"minDiskFreePct,omitempty" json:"-"`
@@ -54,10 +51,11 @@ type FolderDeviceConfiguration struct {
 	IntroducedBy protocol.DeviceID `xml:"introducedBy,attr" json:"introducedBy"`
 }
 
-func NewFolderConfiguration(id, path string) FolderConfiguration {
+func NewFolderConfiguration(id string, fsType fs.FilesystemType, path string) FolderConfiguration {
 	f := FolderConfiguration{
-		ID:      id,
+		ID:             id,
-		RawPath: path,
+		FilesystemType: fsType,
+		Path:           path,
 	}
 	f.prepare()
 	return f
@@ -71,53 +69,57 @@ func (f FolderConfiguration) Copy() FolderConfiguration {
 	return c
 }
 
-func (f FolderConfiguration) Path() string {
+func (f FolderConfiguration) Filesystem() fs.Filesystem {
 	// This is intentionally not a pointer method, because things like
-	// cfg.Folders["default"].Path() should be valid.
+	// cfg.Folders["default"].Filesystem() should be valid.
-
+	if f.cachedFilesystem == nil && f.Path != "" {
-	if f.cachedPath == "" && f.RawPath != "" {
+		l.Infoln("bug: uncached filesystem call (should only happen in tests)")
-		l.Infoln("bug: uncached path call (should only happen in tests)")
+		return fs.NewFilesystem(f.FilesystemType, f.Path)
-		return f.cleanedPath()
 	}
-	return f.cachedPath
+	return f.cachedFilesystem
 }
 
 func (f *FolderConfiguration) CreateMarker() error {
 	if !f.HasMarker() {
-		marker := filepath.Join(f.Path(), ".stfolder")
+		fs := f.Filesystem()
-		fd, err := os.Create(marker)
+		fd, err := fs.Create(".stfolder")
 		if err != nil {
 			return err
 		}
 		fd.Close()
-		if err := osutil.SyncDir(filepath.Dir(marker)); err != nil {
+		if dir, err := fs.Open("."); err == nil {
-			l.Infof("fsync %q failed: %v", filepath.Dir(marker), err)
+			if serr := dir.Sync(); err != nil {
+				l.Infof("fsync %q failed: %v", ".", serr)
+			}
+		} else {
+			l.Infof("fsync %q failed: %v", ".", err)
 		}
-		osutil.HideFile(marker)
+		fs.Hide(".stfolder")
 	}
 
 	return nil
 }
 
 func (f *FolderConfiguration) HasMarker() bool {
-	_, err := os.Stat(filepath.Join(f.Path(), ".stfolder"))
+	_, err := f.Filesystem().Stat(".stfolder")
 	return err == nil
 }
 
 func (f *FolderConfiguration) CreateRoot() (err error) {
 	// Directory permission bits. Will be filtered down to something
 	// sane by umask on Unixes.
-	permBits := os.FileMode(0777)
+	permBits := fs.FileMode(0777)
 	if runtime.GOOS == "windows" {
 		// Windows has no umask so we must chose a safer set of bits to
 		// begin with.
 		permBits = 0700
 	}
 
-	if _, err = os.Stat(f.Path()); os.IsNotExist(err) {
+	filesystem := f.Filesystem()
-		if err = osutil.MkdirAll(f.Path(), permBits); err != nil {
+
-			l.Warnf("Creating directory for %v: %v",
+	if _, err = filesystem.Stat("."); fs.IsNotExist(err) {
-				f.Description(), err)
+		if err = filesystem.MkdirAll(".", permBits); err != nil {
+			l.Warnf("Creating directory for %v: %v", f.Description(), err)
 		}
 	}
 
@@ -140,24 +142,10 @@ func (f *FolderConfiguration) DeviceIDs() []protocol.DeviceID {
 }
 
 func (f *FolderConfiguration) prepare() {
-	if f.RawPath != "" {
+	if f.Path != "" {
-		// The reason it's done like this:
+		f.cachedFilesystem = fs.NewFilesystem(f.FilesystemType, f.Path)
-		// C:          ->  C:\            ->  C:\        (issue that this is trying to fix)
-		// C:\somedir  ->  C:\somedir\    ->  C:\somedir
-		// C:\somedir\ ->  C:\somedir\\   ->  C:\somedir
-		// This way in the tests, we get away without OS specific separators
-		// in the test configs.
-		f.RawPath = filepath.Dir(f.RawPath + string(filepath.Separator))
-
-		// If we're not on Windows, we want the path to end with a slash to
-		// penetrate symlinks. On Windows, paths must not end with a slash.
-		if runtime.GOOS != "windows" && f.RawPath[len(f.RawPath)-1] != filepath.Separator {
-			f.RawPath = f.RawPath + string(filepath.Separator)
-		}
 	}
 
-	f.cachedPath = f.cleanedPath()
-
 	if f.RescanIntervalS > MaxRescanIntervalS {
 		f.RescanIntervalS = MaxRescanIntervalS
 	} else if f.RescanIntervalS < 0 {
@@ -173,43 +161,6 @@ func (f *FolderConfiguration) prepare() {
 	}
 }
 
-func (f *FolderConfiguration) cleanedPath() string {
-	if f.RawPath == "" {
-		return ""
-	}
-
-	cleaned := f.RawPath
-
-	// Attempt tilde expansion; leave unchanged in case of error
-	if path, err := osutil.ExpandTilde(cleaned); err == nil {
-		cleaned = path
-	}
-
-	// Attempt absolutification; leave unchanged in case of error
-	if !filepath.IsAbs(cleaned) {
-		// Abs() looks like a fairly expensive syscall on Windows, while
-		// IsAbs() is a whole bunch of string mangling. I think IsAbs() may be
-		// somewhat faster in the general case, hence the outer if...
-		if path, err := filepath.Abs(cleaned); err == nil {
-			cleaned = path
-		}
-	}
-
-	// Attempt to enable long filename support on Windows. We may still not
-	// have an absolute path here if the previous steps failed.
-	if runtime.GOOS == "windows" && filepath.IsAbs(cleaned) && !strings.HasPrefix(f.RawPath, `\\`) {
-		return `\\?\` + cleaned
-	}
-
-	// If we're not on Windows, we want the path to end with a slash to
-	// penetrate symlinks. On Windows, paths must not end with a slash.
-	if runtime.GOOS != "windows" && cleaned[len(cleaned)-1] != filepath.Separator {
-		cleaned = cleaned + string(filepath.Separator)
-	}
-
-	return cleaned
-}
-
 type FolderDeviceConfigurationList []FolderDeviceConfiguration
 
 func (l FolderDeviceConfigurationList) Less(a, b int) bool {

lib/config/testdata/v22.xml

@@ -0,0 +1,16 @@
+<configuration version="22">
+    <folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
+        <filesystemType>basic</filesystemType>
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+        <minDiskFree unit="%">1</minDiskFree>
+        <maxConflicts>-1</maxConflicts>
+        <fsync>true</fsync>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
+        <address>tcp://a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
+        <address>tcp://b</address>
+    </device>
+</configuration>

lib/config/testdata/versioningconfig.xml

@@ -1,5 +1,5 @@
-<configuration version="10">
+<configuration version="22">
-    <folder id="test" directory="testdata/" ro="true">
+    <folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
         <versioning type="simple">
             <param key="foo" val="bar"/>
             <param key="baz" val="quux"/>

lib/db/set.go

@@ -25,6 +25,7 @@ import (
 type FileSet struct {
 	sequence   int64 // Our local sequence number
 	folder     string
+	fs         fs.Filesystem
 	db         *Instance
 	blockmap   *BlockMap
 	localSize  sizeTracker
@@ -113,10 +114,11 @@ func (s *sizeTracker) Size() Counts {
 	return s.Counts
 }
 
-func NewFileSet(folder string, db *Instance) *FileSet {
+func NewFileSet(folder string, fs fs.Filesystem, db *Instance) *FileSet {
 	var s = FileSet{
 		remoteSequence: make(map[protocol.DeviceID]int64),
 		folder:         folder,
+		fs:             fs,
 		db:             db,
 		blockmap:       NewBlockMap(db, db.folderIdx.ID([]byte(folder))),
 		updateMutex:    sync.NewMutex(),
@@ -303,7 +305,7 @@ func (s *FileSet) SetIndexID(device protocol.DeviceID, id protocol.IndexID) {
 func (s *FileSet) MtimeFS() *fs.MtimeFS {
 	prefix := s.db.mtimesKey([]byte(s.folder))
 	kv := NewNamespacedKV(s.db, string(prefix))
-	return fs.NewMtimeFS(fs.DefaultFilesystem, kv)
+	return fs.NewMtimeFS(s.fs, kv)
 }
 
 func (s *FileSet) ListDevices() []protocol.DeviceID {

lib/db/set_test.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/d4l3k/messagediff"
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -97,7 +98,7 @@ func (l fileList) String() string {
 func TestGlobalSet(t *testing.T) {
 	ldb := db.OpenMemory()
 
-	m := db.NewFileSet("test", ldb)
+	m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
 
 	local0 := fileList{
 		protocol.FileInfo{Name: "a", Sequence: 1, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
@@ -312,7 +313,7 @@ func TestGlobalSet(t *testing.T) {
 func TestNeedWithInvalid(t *testing.T) {
 	ldb := db.OpenMemory()
 
-	s := db.NewFileSet("test", ldb)
+	s := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
 
 	localHave := fileList{
 		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
@@ -349,7 +350,7 @@ func TestNeedWithInvalid(t *testing.T) {
 func TestUpdateToInvalid(t *testing.T) {
 	ldb := db.OpenMemory()
 
-	s := db.NewFileSet("test", ldb)
+	s := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
 
 	localHave := fileList{
 		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
@@ -381,7 +382,7 @@ func TestUpdateToInvalid(t *testing.T) {
 func TestInvalidAvailability(t *testing.T) {
 	ldb := db.OpenMemory()
 
-	s := db.NewFileSet("test", ldb)
+	s := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
 
 	remote0Have := fileList{
 		protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
@@ -419,7 +420,7 @@ func TestInvalidAvailability(t *testing.T) {
 func TestGlobalReset(t *testing.T) {
 	ldb := db.OpenMemory()
 
-	m := db.NewFileSet("test", ldb)
+	m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
 
 	local := []protocol.FileInfo{
 		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@@ -457,7 +458,7 @@ func TestGlobalReset(t *testing.T) {
 func TestNeed(t *testing.T) {
 	ldb := db.OpenMemory()
 
-	m := db.NewFileSet("test", ldb)
+	m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
 
 	local := []protocol.FileInfo{
 		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@@ -495,7 +496,7 @@ func TestNeed(t *testing.T) {
 func TestSequence(t *testing.T) {
 	ldb := db.OpenMemory()
 
-	m := db.NewFileSet("test", ldb)
+	m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
 
 	local1 := []protocol.FileInfo{
 		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@@ -525,7 +526,7 @@ func TestSequence(t *testing.T) {
 func TestListDropFolder(t *testing.T) {
 	ldb := db.OpenMemory()
 
-	s0 := db.NewFileSet("test0", ldb)
+	s0 := db.NewFileSet("test0", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
 	local1 := []protocol.FileInfo{
 		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@@ -533,7 +534,7 @@ func TestListDropFolder(t *testing.T) {
 	}
 	s0.Replace(protocol.LocalDeviceID, local1)
 
-	s1 := db.NewFileSet("test1", ldb)
+	s1 := db.NewFileSet("test1", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
 	local2 := []protocol.FileInfo{
 		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
 		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
@@ -575,7 +576,7 @@ func TestListDropFolder(t *testing.T) {
 func TestGlobalNeedWithInvalid(t *testing.T) {
 	ldb := db.OpenMemory()
 
-	s := db.NewFileSet("test1", ldb)
+	s := db.NewFileSet("test1", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
 
 	rem0 := fileList{
 		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
@@ -612,7 +613,7 @@ func TestGlobalNeedWithInvalid(t *testing.T) {
 func TestLongPath(t *testing.T) {
 	ldb := db.OpenMemory()
 
-	s := db.NewFileSet("test", ldb)
+	s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
 
 	var b bytes.Buffer
 	for i := 0; i < 100; i++ {
@@ -642,7 +643,7 @@ func TestCommitted(t *testing.T) {
 
 	ldb := db.OpenMemory()
 
-	s := db.NewFileSet("test", ldb)
+	s := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
 
 	local := []protocol.FileInfo{
 		{Name: string("file"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@@ -688,7 +689,7 @@ func BenchmarkUpdateOneFile(b *testing.B) {
 		os.RemoveAll("testdata/benchmarkupdate.db")
 	}()
 
-	m := db.NewFileSet("test", ldb)
+	m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
 	m.Replace(protocol.LocalDeviceID, local0)
 	l := local0[4:5]
 
@@ -703,7 +704,7 @@ func BenchmarkUpdateOneFile(b *testing.B) {
 func TestIndexID(t *testing.T) {
 	ldb := db.OpenMemory()
 
-	s := db.NewFileSet("test", ldb)
+	s := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
 
 	// The Index ID for some random device is zero by default.
 	id := s.IndexID(remoteDevice0)

lib/fs/basicfs.go

@@ -9,30 +9,156 @@ package fs
 import (
 	"errors"
 	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
 	"time"
+
+	"github.com/calmh/du"
+)
+
+var (
+	ErrInvalidFilename = errors.New("filename is invalid")
+	ErrNotRelative     = errors.New("not a relative path")
 )
 
 // The BasicFilesystem implements all aspects by delegating to package os.
+// All paths are relative to the root and cannot (should not) escape the root directory.
 type BasicFilesystem struct {
+	root string
 }
 
-func NewBasicFilesystem() *BasicFilesystem {
+func newBasicFilesystem(root string) *BasicFilesystem {
-	return new(BasicFilesystem)
+	// The reason it's done like this:
+	// C:          ->  C:\            ->  C:\        (issue that this is trying to fix)
+	// C:\somedir  ->  C:\somedir\    ->  C:\somedir
+	// C:\somedir\ ->  C:\somedir\\   ->  C:\somedir
+	// This way in the tests, we get away without OS specific separators
+	// in the test configs.
+	root = filepath.Dir(root + string(filepath.Separator))
+
+	// Attempt tilde expansion; leave unchanged in case of error
+	if path, err := ExpandTilde(root); err == nil {
+		root = path
+	}
+
+	// Attempt absolutification; leave unchanged in case of error
+	if !filepath.IsAbs(root) {
+		// Abs() looks like a fairly expensive syscall on Windows, while
+		// IsAbs() is a whole bunch of string mangling. I think IsAbs() may be
+		// somewhat faster in the general case, hence the outer if...
+		if path, err := filepath.Abs(root); err == nil {
+			root = path
+		}
+	}
+
+	// Attempt to enable long filename support on Windows. We may still not
+	// have an absolute path here if the previous steps failed.
+	if runtime.GOOS == "windows" {
+		if filepath.IsAbs(root) && !strings.HasPrefix(root, `\\`) {
+			root = `\\?\` + root
+		}
+		// If we're not on Windows, we want the path to end with a slash to
+		// penetrate symlinks. On Windows, paths must not end with a slash.
+	} else if root[len(root)-1] != filepath.Separator {
+		root = root + string(filepath.Separator)
+	}
+
+	return &BasicFilesystem{
+		root: root,
+	}
+}
+
+// rooted expands the relative path to the full path that is then used with os
+// package. If the relative path somehow causes the final path to escape the root
+// directoy, this returns an error, to prevent accessing files that are not in the
+// shared directory.
+func (f *BasicFilesystem) rooted(rel string) (string, error) {
+	// The root must not be empty.
+	if f.root == "" {
+		return "", ErrInvalidFilename
+	}
+
+	pathSep := string(PathSeparator)
+
+	// The expected prefix for the resulting path is the root, with a path
+	// separator at the end.
+	expectedPrefix := filepath.FromSlash(f.root)
+	if !strings.HasSuffix(expectedPrefix, pathSep) {
+		expectedPrefix += pathSep
+	}
+
+	// The relative path should be clean from internal dotdots and similar
+	// funkyness.
+	rel = filepath.FromSlash(rel)
+	if filepath.Clean(rel) != rel {
+		return "", ErrInvalidFilename
+	}
+
+	// It is not acceptable to attempt to traverse upwards.
+	switch rel {
+	case "..", pathSep:
+		return "", ErrNotRelative
+	}
+	if strings.HasPrefix(rel, ".."+pathSep) {
+		return "", ErrNotRelative
+	}
+
+	if strings.HasPrefix(rel, pathSep+pathSep) {
+		// The relative path may pretend to be an absolute path within the
+		// root, but the double path separator on Windows implies something
+		// else. It would get cleaned by the Join below, but it's out of
+		// spec anyway.
+		return "", ErrNotRelative
+	}
+
+	// The supposedly correct path is the one filepath.Join will return, as
+	// it does cleaning and so on. Check that one first to make sure no
+	// obvious escape attempts have been made.
+	joined := filepath.Join(f.root, rel)
+	if rel == "." && !strings.HasSuffix(joined, pathSep) {
+		joined += pathSep
+	}
+	if !strings.HasPrefix(joined, expectedPrefix) {
+		return "", ErrNotRelative
+	}
+
+	return joined, nil
+}
+
+func (f *BasicFilesystem) unrooted(path string) string {
+	return strings.TrimPrefix(strings.TrimPrefix(path, f.root), string(PathSeparator))
 }
 
 func (f *BasicFilesystem) Chmod(name string, mode FileMode) error {
+	name, err := f.rooted(name)
+	if err != nil {
+		return err
+	}
 	return os.Chmod(name, os.FileMode(mode))
 }
 
 func (f *BasicFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error {
+	name, err := f.rooted(name)
+	if err != nil {
+		return err
+	}
 	return os.Chtimes(name, atime, mtime)
 }
 
 func (f *BasicFilesystem) Mkdir(name string, perm FileMode) error {
+	name, err := f.rooted(name)
+	if err != nil {
+		return err
+	}
 	return os.Mkdir(name, os.FileMode(perm))
 }
 
 func (f *BasicFilesystem) Lstat(name string) (FileInfo, error) {
+	name, err := f.rooted(name)
+	if err != nil {
+		return nil, err
+	}
 	fi, err := underlyingLstat(name)
 	if err != nil {
 		return nil, err
@@ -41,14 +167,38 @@ func (f *BasicFilesystem) Lstat(name string) (FileInfo, error) {
 }
 
 func (f *BasicFilesystem) Remove(name string) error {
+	name, err := f.rooted(name)
+	if err != nil {
+		return err
+	}
 	return os.Remove(name)
 }
 
+func (f *BasicFilesystem) RemoveAll(name string) error {
+	name, err := f.rooted(name)
+	if err != nil {
+		return err
+	}
+	return os.RemoveAll(name)
+}
+
 func (f *BasicFilesystem) Rename(oldpath, newpath string) error {
+	oldpath, err := f.rooted(oldpath)
+	if err != nil {
+		return err
+	}
+	newpath, err = f.rooted(newpath)
+	if err != nil {
+		return err
+	}
 	return os.Rename(oldpath, newpath)
 }
 
 func (f *BasicFilesystem) Stat(name string) (FileInfo, error) {
+	name, err := f.rooted(name)
+	if err != nil {
+		return nil, err
+	}
 	fi, err := os.Stat(name)
 	if err != nil {
 		return nil, err
@@ -57,7 +207,11 @@ func (f *BasicFilesystem) Stat(name string) (FileInfo, error) {
 }
 
 func (f *BasicFilesystem) DirNames(name string) ([]string, error) {
-	fd, err := os.OpenFile(name, os.O_RDONLY, 0777)
+	name, err := f.rooted(name)
+	if err != nil {
+		return nil, err
+	}
+	fd, err := os.OpenFile(name, OptReadOnly, 0777)
 	if err != nil {
 		return nil, err
 	}
@@ -72,19 +226,39 @@ func (f *BasicFilesystem) DirNames(name string) ([]string, error) {
 }
 
 func (f *BasicFilesystem) Open(name string) (File, error) {
-	fd, err := os.Open(name)
+	rootedName, err := f.rooted(name)
 	if err != nil {
 		return nil, err
 	}
-	return fsFile{fd}, err
+	fd, err := os.Open(rootedName)
+	if err != nil {
+		return nil, err
+	}
+	return fsFile{fd, name}, err
+}
+
+func (f *BasicFilesystem) OpenFile(name string, flags int, mode FileMode) (File, error) {
+	rootedName, err := f.rooted(name)
+	if err != nil {
+		return nil, err
+	}
+	fd, err := os.OpenFile(rootedName, flags, os.FileMode(mode))
+	if err != nil {
+		return nil, err
+	}
+	return fsFile{fd, name}, err
 }
 
 func (f *BasicFilesystem) Create(name string) (File, error) {
-	fd, err := os.Create(name)
+	rootedName, err := f.rooted(name)
 	if err != nil {
 		return nil, err
 	}
-	return fsFile{fd}, err
+	fd, err := os.Create(rootedName)
+	if err != nil {
+		return nil, err
+	}
+	return fsFile{fd, name}, err
 }
 
 func (f *BasicFilesystem) Walk(root string, walkFn WalkFunc) error {
@@ -92,9 +266,47 @@ func (f *BasicFilesystem) Walk(root string, walkFn WalkFunc) error {
 	return errors.New("not implemented")
 }
 
+func (f *BasicFilesystem) Glob(pattern string) ([]string, error) {
+	pattern, err := f.rooted(pattern)
+	if err != nil {
+		return nil, err
+	}
+	files, err := filepath.Glob(pattern)
+	unrooted := make([]string, len(files))
+	for i := range files {
+		unrooted[i] = f.unrooted(files[i])
+	}
+	return unrooted, err
+}
+
+func (f *BasicFilesystem) Usage(name string) (Usage, error) {
+	name, err := f.rooted(name)
+	if err != nil {
+		return Usage{}, err
+	}
+	u, err := du.Get(name)
+	return Usage{
+		Free:  u.FreeBytes,
+		Total: u.TotalBytes,
+	}, err
+}
+
+func (f *BasicFilesystem) Type() FilesystemType {
+	return FilesystemTypeBasic
+}
+
+func (f *BasicFilesystem) URI() string {
+	return strings.TrimPrefix(f.root, `\\?\`)
+}
+
 // fsFile implements the fs.File interface on top of an os.File
 type fsFile struct {
 	*os.File
+	name string
+}
+
+func (f fsFile) Name() string {
+	return f.name
 }
 
 func (f fsFile) Stat() (FileInfo, error) {
@@ -105,6 +317,17 @@ func (f fsFile) Stat() (FileInfo, error) {
 	return fsFileInfo{info}, nil
 }
 
+func (f fsFile) Sync() error {
+	err := f.File.Sync()
+	// On Windows, fsyncing a directory returns a "handle is invalid"
+	// So we swallow that and let things go through in order not to have to add
+	// a separate way of syncing directories versus files.
+	if err != nil && (runtime.GOOS != "windows" || !strings.Contains(err.Error(), "handle is invalid")) {
+		return err
+	}
+	return nil
+}
+
 // fsFileInfo implements the fs.FileInfo interface on top of an os.FileInfo.
 type fsFileInfo struct {
 	os.FileInfo

lib/fs/basicfs_symlink_unix.go

@@ -1,29 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build !windows
-
-package fs
-
-import "os"
-
-var symlinksSupported = true
-
-func DisableSymlinks() {
-	symlinksSupported = false
-}
-
-func (BasicFilesystem) SymlinksSupported() bool {
-	return symlinksSupported
-}
-
-func (BasicFilesystem) CreateSymlink(name, target string) error {
-	return os.Symlink(target, name)
-}
-
-func (BasicFilesystem) ReadSymlink(path string) (string, error) {
-	return os.Readlink(path)
-}

lib/fs/basicfs_symlink_windows.go

@@ -1,27 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build windows
-
-package fs
-
-import "errors"
-
-var errNotSupported = errors.New("symlinks not supported")
-
-func DisableSymlinks() {}
-
-func (BasicFilesystem) SymlinksSupported() bool {
-	return false
-}
-
-func (BasicFilesystem) ReadSymlink(path string) (string, error) {
-	return "", errNotSupported
-}
-
-func (BasicFilesystem) CreateSymlink(path, target string) error {
-	return errNotSupported
-}

lib/fs/basicfs_test.go

@@ -0,0 +1,486 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package fs
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+)
+
+func setup(t *testing.T) (Filesystem, string) {
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return newBasicFilesystem(dir), dir
+}
+
+func TestChmodFile(t *testing.T) {
+	fs, dir := setup(t)
+	path := filepath.Join(dir, "file")
+	defer os.RemoveAll(dir)
+
+	defer os.Chmod(path, 0666)
+
+	fd, err := os.Create(path)
+	if err != nil {
+		t.Error(err)
+	}
+	fd.Close()
+
+	if err := os.Chmod(path, 0666); err != nil {
+		t.Error(err)
+	}
+
+	if stat, err := os.Stat(path); err != nil || stat.Mode()&os.ModePerm != 0666 {
+		t.Errorf("wrong perm: %t %#o", err == nil, stat.Mode()&os.ModePerm)
+	}
+
+	if err := fs.Chmod("file", 0444); err != nil {
+		t.Error(err)
+	}
+
+	if stat, err := os.Stat(path); err != nil || stat.Mode()&os.ModePerm != 0444 {
+		t.Errorf("wrong perm: %t %#o", err == nil, stat.Mode()&os.ModePerm)
+	}
+}
+
+func TestChmodDir(t *testing.T) {
+	fs, dir := setup(t)
+	path := filepath.Join(dir, "dir")
+	defer os.RemoveAll(dir)
+
+	mode := os.FileMode(0755)
+	if runtime.GOOS == "windows" {
+		mode = os.FileMode(0777)
+	}
+
+	defer os.Chmod(path, mode)
+
+	if err := os.Mkdir(path, mode); err != nil {
+		t.Error(err)
+	}
+
+	if stat, err := os.Stat(path); err != nil || stat.Mode()&os.ModePerm != mode {
+		t.Errorf("wrong perm: %t %#o", err == nil, stat.Mode()&os.ModePerm)
+	}
+
+	if err := fs.Chmod("dir", 0555); err != nil {
+		t.Error(err)
+	}
+
+	if stat, err := os.Stat(path); err != nil || stat.Mode()&os.ModePerm != 0555 {
+		t.Errorf("wrong perm: %t %#o", err == nil, stat.Mode()&os.ModePerm)
+	}
+}
+
+func TestChtimes(t *testing.T) {
+	fs, dir := setup(t)
+	path := filepath.Join(dir, "file")
+	defer os.RemoveAll(dir)
+	fd, err := os.Create(path)
+	if err != nil {
+		t.Error(err)
+	}
+	fd.Close()
+
+	mtime := time.Now().Add(-time.Hour)
+
+	fs.Chtimes("file", mtime, mtime)
+
+	stat, err := os.Stat(path)
+	if err != nil {
+		t.Error(err)
+	}
+
+	diff := stat.ModTime().Sub(mtime)
+	if diff > 3*time.Second || diff < -3*time.Second {
+		t.Errorf("%s != %s", stat.Mode(), mtime)
+	}
+}
+
+func TestCreate(t *testing.T) {
+	fs, dir := setup(t)
+	path := filepath.Join(dir, "file")
+	defer os.RemoveAll(dir)
+
+	if _, err := os.Stat(path); err == nil {
+		t.Errorf("exists?")
+	}
+
+	fd, err := fs.Create("file")
+	if err != nil {
+		t.Error(err)
+	}
+	fd.Close()
+
+	if _, err := os.Stat(path); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestCreateSymlink(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("windows not supported")
+	}
+
+	fs, dir := setup(t)
+	path := filepath.Join(dir, "file")
+	defer os.RemoveAll(dir)
+
+	if err := fs.CreateSymlink("blah", "file"); err != nil {
+		t.Error(err)
+	}
+
+	if target, err := os.Readlink(path); err != nil || target != "blah" {
+		t.Error("target", target, "err", err)
+	}
+
+	if err := os.Remove(path); err != nil {
+		t.Error(err)
+	}
+
+	if err := fs.CreateSymlink(filepath.Join("..", "blah"), "file"); err != nil {
+		t.Error(err)
+	}
+
+	if target, err := os.Readlink(path); err != nil || target != filepath.Join("..", "blah") {
+		t.Error("target", target, "err", err)
+	}
+}
+
+func TestDirNames(t *testing.T) {
+	fs, dir := setup(t)
+	defer os.RemoveAll(dir)
+
+	// Case differences
+	testCases := []string{
+		"a",
+		"bC",
+	}
+	sort.Strings(testCases)
+
+	for _, sub := range testCases {
+		if err := os.Mkdir(filepath.Join(dir, sub), 0777); err != nil {
+			t.Error(err)
+		}
+	}
+
+	if dirs, err := fs.DirNames("."); err != nil || len(dirs) != len(testCases) {
+		t.Errorf("%s %s %s", err, dirs, testCases)
+	} else {
+		sort.Strings(dirs)
+		for i := range dirs {
+			if dirs[i] != testCases[i] {
+				t.Errorf("%s != %s", dirs[i], testCases[i])
+			}
+		}
+	}
+}
+
+func TestNames(t *testing.T) {
+	// Tests that all names are without the root directory.
+	fs, dir := setup(t)
+	defer os.RemoveAll(dir)
+
+	expected := "file"
+	fd, err := fs.Create(expected)
+	if err != nil {
+		t.Error(err)
+	}
+	defer fd.Close()
+
+	if fd.Name() != expected {
+		t.Errorf("incorrect %s != %s", fd.Name(), expected)
+	}
+	if stat, err := fd.Stat(); err != nil || stat.Name() != expected {
+		t.Errorf("incorrect %s != %s (%v)", stat.Name(), expected, err)
+	}
+
+	if err := fs.Mkdir("dir", 0777); err != nil {
+		t.Error(err)
+	}
+
+	expected = filepath.Join("dir", "file")
+	fd, err = fs.Create(expected)
+	if err != nil {
+		t.Error(err)
+	}
+	defer fd.Close()
+
+	if fd.Name() != expected {
+		t.Errorf("incorrect %s != %s", fd.Name(), expected)
+	}
+
+	// os.fd.Stat() returns just base, so do we.
+	if stat, err := fd.Stat(); err != nil || stat.Name() != filepath.Base(expected) {
+		t.Errorf("incorrect %s != %s (%v)", stat.Name(), filepath.Base(expected), err)
+	}
+}
+
+func TestGlob(t *testing.T) {
+	// Tests that all names are without the root directory.
+	fs, dir := setup(t)
+	defer os.RemoveAll(dir)
+
+	for _, dirToCreate := range []string{
+		filepath.Join("a", "test", "b"),
+		filepath.Join("a", "best", "b"),
+		filepath.Join("a", "best", "c"),
+	} {
+		if err := fs.MkdirAll(dirToCreate, 0777); err != nil {
+			t.Error(err)
+		}
+	}
+
+	testCases := []struct {
+		pattern string
+		matches []string
+	}{
+		{
+			filepath.Join("a", "?est", "?"),
+			[]string{
+				filepath.Join("a", "test", "b"),
+				filepath.Join("a", "best", "b"),
+				filepath.Join("a", "best", "c"),
+			},
+		},
+		{
+			filepath.Join("a", "?est", "b"),
+			[]string{
+				filepath.Join("a", "test", "b"),
+				filepath.Join("a", "best", "b"),
+			},
+		},
+		{
+			filepath.Join("a", "best", "?"),
+			[]string{
+				filepath.Join("a", "best", "b"),
+				filepath.Join("a", "best", "c"),
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		results, err := fs.Glob(testCase.pattern)
+		sort.Strings(results)
+		sort.Strings(testCase.matches)
+		if err != nil {
+			t.Error(err)
+		}
+		if len(results) != len(testCase.matches) {
+			t.Errorf("result count mismatch")
+		}
+		for i := range testCase.matches {
+			if results[i] != testCase.matches[i] {
+				t.Errorf("%s != %s", results[i], testCase.matches[i])
+			}
+		}
+	}
+}
+
+func TestUsage(t *testing.T) {
+	fs, dir := setup(t)
+	defer os.RemoveAll(dir)
+	usage, err := fs.Usage(".")
+	if err != nil {
+		if runtime.GOOS == "netbsd" || runtime.GOOS == "openbsd" || runtime.GOOS == "solaris" {
+			t.Skip()
+		}
+		t.Errorf("Unexpected error: %s", err)
+	}
+	if usage.Free < 1 {
+		t.Error("Disk is full?", usage.Free)
+	}
+}
+
+func TestWindowsPaths(t *testing.T) {
+	if runtime.GOOS != "windows" {
+		t.Skip("Not useful on non-Windows")
+		return
+	}
+
+	testCases := []struct {
+		input        string
+		expectedRoot string
+		expectedURI  string
+	}{
+		{`e:\`, `\\?\e:\`, `e:\`},
+		{`\\?\e:\`, `\\?\e:\`, `e:\`},
+		{`\\192.0.2.22\network\share`, `\\192.0.2.22\network\share`, `\\192.0.2.22\network\share`},
+	}
+
+	for _, testCase := range testCases {
+		fs := newBasicFilesystem(testCase.input)
+		if fs.root != testCase.expectedRoot {
+			t.Errorf("root %q != %q", fs.root, testCase.expectedRoot)
+		}
+		if fs.URI() != testCase.expectedURI {
+			t.Errorf("uri %q != %q", fs.URI(), testCase.expectedURI)
+		}
+	}
+
+	fs := newBasicFilesystem(`relative\path`)
+	if fs.root == `relative\path` || !strings.HasPrefix(fs.root, "\\\\?\\") {
+		t.Errorf("%q == %q, expected absolutification", fs.root, `relative\path`)
+	}
+}
+
+func TestRooted(t *testing.T) {
+	type testcase struct {
+		root   string
+		rel    string
+		joined string
+		ok     bool
+	}
+	cases := []testcase{
+		// Valid cases
+		{"foo", "bar", "foo/bar", true},
+		{"foo", "/bar", "foo/bar", true},
+		{"foo/", "bar", "foo/bar", true},
+		{"foo/", "/bar", "foo/bar", true},
+		{"baz/foo", "bar", "baz/foo/bar", true},
+		{"baz/foo", "/bar", "baz/foo/bar", true},
+		{"baz/foo/", "bar", "baz/foo/bar", true},
+		{"baz/foo/", "/bar", "baz/foo/bar", true},
+		{"foo", "bar/baz", "foo/bar/baz", true},
+		{"foo", "/bar/baz", "foo/bar/baz", true},
+		{"foo/", "bar/baz", "foo/bar/baz", true},
+		{"foo/", "/bar/baz", "foo/bar/baz", true},
+		{"baz/foo", "bar/baz", "baz/foo/bar/baz", true},
+		{"baz/foo", "/bar/baz", "baz/foo/bar/baz", true},
+		{"baz/foo/", "bar/baz", "baz/foo/bar/baz", true},
+		{"baz/foo/", "/bar/baz", "baz/foo/bar/baz", true},
+
+		// Not escape attempts, but oddly formatted relative paths. Disallowed.
+		{"foo", "./bar", "", false},
+		{"baz/foo", "./bar", "", false},
+		{"foo", "./bar/baz", "", false},
+		{"baz/foo", "./bar/baz", "", false},
+		{"baz/foo", "bar/../baz", "", false},
+		{"baz/foo", "/bar/../baz", "", false},
+		{"baz/foo", "./bar/../baz", "", false},
+		{"baz/foo", "bar/../baz", "", false},
+		{"baz/foo", "/bar/../baz", "", false},
+		{"baz/foo", "./bar/../baz", "", false},
+
+		// Results in an allowed path, but does it by probing. Disallowed.
+		{"foo", "../foo", "", false},
+		{"foo", "../foo/bar", "", false},
+		{"baz/foo", "../foo/bar", "", false},
+		{"baz/foo", "../../baz/foo/bar", "", false},
+		{"baz/foo", "bar/../../foo/bar", "", false},
+		{"baz/foo", "bar/../../../baz/foo/bar", "", false},
+
+		// Escape attempts.
+		{"foo", "", "", false},
+		{"foo", "/", "", false},
+		{"foo", "..", "", false},
+		{"foo", "/..", "", false},
+		{"foo", "../", "", false},
+		{"foo", "../bar", "", false},
+		{"foo", "../foobar", "", false},
+		{"foo/", "../bar", "", false},
+		{"foo/", "../foobar", "", false},
+		{"baz/foo", "../bar", "", false},
+		{"baz/foo", "../foobar", "", false},
+		{"baz/foo/", "../bar", "", false},
+		{"baz/foo/", "../foobar", "", false},
+		{"baz/foo/", "bar/../../quux/baz", "", false},
+
+		// Empty root is a misconfiguration.
+		{"", "/foo", "", false},
+		{"", "foo", "", false},
+		{"", ".", "", false},
+		{"", "..", "", false},
+		{"", "/", "", false},
+		{"", "", "", false},
+
+		// Root=/ is valid, and things should be verified as usual.
+		{"/", "foo", "/foo", true},
+		{"/", "/foo", "/foo", true},
+		{"/", "../foo", "", false},
+		{"/", "..", "", false},
+		{"/", "/", "", false},
+		{"/", "", "", false},
+
+		// special case for filesystems to be able to MkdirAll('.') for example
+		{"/", ".", "/", true},
+	}
+
+	if runtime.GOOS == "windows" {
+		extraCases := []testcase{
+			{`c:\`, `foo`, `c:\foo`, true},
+			{`\\?\c:\`, `foo`, `\\?\c:\foo`, true},
+			{`c:\`, `\foo`, `c:\foo`, true},
+			{`\\?\c:\`, `\foo`, `\\?\c:\foo`, true},
+			{`c:\`, `\\foo`, ``, false},
+			{`c:\`, ``, ``, false},
+			{`c:\`, `\`, ``, false},
+			{`\\?\c:\`, `\\foo`, ``, false},
+			{`\\?\c:\`, ``, ``, false},
+			{`\\?\c:\`, `\`, ``, false},
+
+			// makes no sense, but will be treated simply as a bad filename
+			{`c:\foo`, `d:\bar`, `c:\foo\d:\bar`, true},
+
+			// special case for filesystems to be able to MkdirAll('.') for example
+			{`c:\`, `.`, `c:\`, true},
+			{`\\?\c:\`, `.`, `\\?\c:\`, true},
+		}
+
+		for _, tc := range cases {
+			// Add case where root is backslashed, rel is forward slashed
+			extraCases = append(extraCases, testcase{
+				root:   filepath.FromSlash(tc.root),
+				rel:    tc.rel,
+				joined: tc.joined,
+				ok:     tc.ok,
+			})
+			// and the opposite
+			extraCases = append(extraCases, testcase{
+				root:   tc.root,
+				rel:    filepath.FromSlash(tc.rel),
+				joined: tc.joined,
+				ok:     tc.ok,
+			})
+			// and both backslashed
+			extraCases = append(extraCases, testcase{
+				root:   filepath.FromSlash(tc.root),
+				rel:    filepath.FromSlash(tc.rel),
+				joined: tc.joined,
+				ok:     tc.ok,
+			})
+		}
+
+		cases = append(cases, extraCases...)
+	}
+
+	for _, tc := range cases {
+		fs := BasicFilesystem{root: tc.root}
+		res, err := fs.rooted(tc.rel)
+		if tc.ok {
+			if err != nil {
+				t.Errorf("Unexpected error for rooted(%q, %q): %v", tc.root, tc.rel, err)
+				continue
+			}
+			exp := filepath.FromSlash(tc.joined)
+			if res != exp {
+				t.Errorf("Unexpected result for rooted(%q, %q): %q != expected %q", tc.root, tc.rel, res, exp)
+			}
+		} else if err == nil {
+			t.Errorf("Unexpected pass for rooted(%q, %q) => %q", tc.root, tc.rel, res)
+			continue
+		}
+	}
+}

lib/fs/basicfs_unix.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// +build !windows
+
+package fs
+
+import "os"
+
+func (BasicFilesystem) SymlinksSupported() bool {
+	return true
+}
+
+func (f *BasicFilesystem) CreateSymlink(target, name string) error {
+	name, err := f.rooted(name)
+	if err != nil {
+		return err
+	}
+	return os.Symlink(target, name)
+}
+
+func (f *BasicFilesystem) ReadSymlink(name string) (string, error) {
+	name, err := f.rooted(name)
+	if err != nil {
+		return "", err
+	}
+	return os.Readlink(name)
+}
+
+func (f *BasicFilesystem) MkdirAll(name string, perm FileMode) error {
+	name, err := f.rooted(name)
+	if err != nil {
+		return err
+	}
+	return os.MkdirAll(name, os.FileMode(perm))
+}
+
+// Unhide is a noop on unix, as unhiding files requires renaming them.
+// We still check that the relative path does not try to escape the root
+func (f *BasicFilesystem) Unhide(name string) error {
+	_, err := f.rooted(name)
+	return err
+}
+
+// Hide is a noop on unix, as hiding files requires renaming them.
+// We still check that the relative path does not try to escape the root
+func (f *BasicFilesystem) Hide(name string) error {
+	_, err := f.rooted(name)
+	return err
+}
+
+func (f *BasicFilesystem) Roots() ([]string, error) {
+	return []string{"/"}, nil
+}

lib/fs/basicfs_windows.go

@@ -0,0 +1,165 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// +build windows
+
+package fs
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+	"unsafe"
+)
+
+var errNotSupported = errors.New("symlinks not supported")
+
+func (BasicFilesystem) SymlinksSupported() bool {
+	return false
+}
+
+func (BasicFilesystem) ReadSymlink(path string) (string, error) {
+	return "", errNotSupported
+}
+
+func (BasicFilesystem) CreateSymlink(path, target string) error {
+	return errNotSupported
+}
+
+// MkdirAll creates a directory named path, along with any necessary parents,
+// and returns nil, or else returns an error.
+// The permission bits perm are used for all directories that MkdirAll creates.
+// If path is already a directory, MkdirAll does nothing and returns nil.
+func (f *BasicFilesystem) MkdirAll(path string, perm FileMode) error {
+	path, err := f.rooted(path)
+	if err != nil {
+		return err
+	}
+
+	return f.mkdirAll(path, os.FileMode(perm))
+}
+
+// Required due to https://github.com/golang/go/issues/10900
+func (f *BasicFilesystem) mkdirAll(path string, perm os.FileMode) error {
+	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
+	dir, err := os.Stat(path)
+	if err == nil {
+		if dir.IsDir() {
+			return nil
+		}
+		return &os.PathError{
+			Op:   "mkdir",
+			Path: path,
+			Err:  syscall.ENOTDIR,
+		}
+	}
+
+	// Slow path: make sure parent exists and then call Mkdir for path.
+	i := len(path)
+	for i > 0 && IsPathSeparator(path[i-1]) { // Skip trailing path separator.
+		i--
+	}
+
+	j := i
+	for j > 0 && !IsPathSeparator(path[j-1]) { // Scan backward over element.
+		j--
+	}
+
+	if j > 1 {
+		// Create parent
+		parent := path[0 : j-1]
+		if parent != filepath.VolumeName(parent) {
+			err = os.MkdirAll(parent, perm)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// Parent now exists; invoke Mkdir and use its result.
+	err = os.Mkdir(path, perm)
+	if err != nil {
+		// Handle arguments like "foo/." by
+		// double-checking that directory doesn't exist.
+		dir, err1 := os.Lstat(path)
+		if err1 == nil && dir.IsDir() {
+			return nil
+		}
+		return err
+	}
+	return nil
+}
+
+func (f *BasicFilesystem) Unhide(name string) error {
+	name, err := f.rooted(name)
+	if err != nil {
+		return err
+	}
+	p, err := syscall.UTF16PtrFromString(name)
+	if err != nil {
+		return err
+	}
+
+	attrs, err := syscall.GetFileAttributes(p)
+	if err != nil {
+		return err
+	}
+
+	attrs &^= syscall.FILE_ATTRIBUTE_HIDDEN
+	return syscall.SetFileAttributes(p, attrs)
+}
+
+func (f *BasicFilesystem) Hide(name string) error {
+	name, err := f.rooted(name)
+	if err != nil {
+		return err
+	}
+	p, err := syscall.UTF16PtrFromString(name)
+	if err != nil {
+		return err
+	}
+
+	attrs, err := syscall.GetFileAttributes(p)
+	if err != nil {
+		return err
+	}
+
+	attrs |= syscall.FILE_ATTRIBUTE_HIDDEN
+	return syscall.SetFileAttributes(p, attrs)
+}
+
+func (f *BasicFilesystem) Roots() ([]string, error) {
+	kernel32, err := syscall.LoadDLL("kernel32.dll")
+	if err != nil {
+		return nil, err
+	}
+	getLogicalDriveStringsHandle, err := kernel32.FindProc("GetLogicalDriveStringsA")
+	if err != nil {
+		return nil, err
+	}
+
+	buffer := [1024]byte{}
+	bufferSize := uint32(len(buffer))
+
+	hr, _, _ := getLogicalDriveStringsHandle.Call(uintptr(unsafe.Pointer(&bufferSize)), uintptr(unsafe.Pointer(&buffer)))
+	if hr == 0 {
+		return nil, fmt.Errorf("Syscall failed")
+	}
+
+	var drives []string
+	parts := bytes.Split(buffer[:], []byte{0})
+	for _, part := range parts {
+		if len(part) == 0 {
+			break
+		}
+		drives = append(drives, string(part))
+	}
+
+	return drives, nil
+}

lib/fs/debug.go

@@ -0,0 +1,22 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package fs
+
+import (
+	"os"
+	"strings"
+
+	"github.com/syncthing/syncthing/lib/logger"
+)
+
+var (
+	l = logger.DefaultLogger.NewFacility("filesystem", "Filesystem access")
+)
+
+func init() {
+	l.SetDebug("filesystem", strings.Contains(os.Getenv("STTRACE"), "filesystem") || os.Getenv("STTRACE") == "all")
+}

lib/fs/errorfs.go

@@ -0,0 +1,41 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package fs
+
+import "time"
+
+type errorFilesystem struct {
+	err    error
+	fsType FilesystemType
+	uri    string
+}
+
+func (fs *errorFilesystem) Chmod(name string, mode FileMode) error                      { return fs.err }
+func (fs *errorFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error { return fs.err }
+func (fs *errorFilesystem) Create(name string) (File, error)                            { return nil, fs.err }
+func (fs *errorFilesystem) CreateSymlink(name, target string) error                     { return fs.err }
+func (fs *errorFilesystem) DirNames(name string) ([]string, error)                      { return nil, fs.err }
+func (fs *errorFilesystem) Lstat(name string) (FileInfo, error)                         { return nil, fs.err }
+func (fs *errorFilesystem) Mkdir(name string, perm FileMode) error                      { return fs.err }
+func (fs *errorFilesystem) MkdirAll(name string, perm FileMode) error                   { return fs.err }
+func (fs *errorFilesystem) Open(name string) (File, error)                              { return nil, fs.err }
+func (fs *errorFilesystem) OpenFile(string, int, FileMode) (File, error)                { return nil, fs.err }
+func (fs *errorFilesystem) ReadSymlink(name string) (string, error)                     { return "", fs.err }
+func (fs *errorFilesystem) Remove(name string) error                                    { return fs.err }
+func (fs *errorFilesystem) RemoveAll(name string) error                                 { return fs.err }
+func (fs *errorFilesystem) Rename(oldname, newname string) error                        { return fs.err }
+func (fs *errorFilesystem) Stat(name string) (FileInfo, error)                          { return nil, fs.err }
+func (fs *errorFilesystem) SymlinksSupported() bool                                     { return false }
+func (fs *errorFilesystem) Walk(root string, walkFn WalkFunc) error                     { return fs.err }
+func (fs *errorFilesystem) Unhide(name string) error                                    { return fs.err }
+func (fs *errorFilesystem) Hide(name string) error                                      { return fs.err }
+func (fs *errorFilesystem) Glob(pattern string) ([]string, error)                       { return nil, fs.err }
+func (fs *errorFilesystem) SyncDir(name string) error                                   { return fs.err }
+func (fs *errorFilesystem) Roots() ([]string, error)                                    { return nil, fs.err }
+func (fs *errorFilesystem) Usage(name string) (Usage, error)                            { return Usage{}, fs.err }
+func (fs *errorFilesystem) Type() FilesystemType                                        { return fs.fsType }
+func (fs *errorFilesystem) URI() string                                                 { return fs.uri }

lib/fs/filesystem.go

@@ -7,6 +7,7 @@
 package fs
 
 import (
+	"errors"
 	"io"
 	"os"
 	"path/filepath"
@@ -22,23 +23,38 @@ type Filesystem interface {
 	DirNames(name string) ([]string, error)
 	Lstat(name string) (FileInfo, error)
 	Mkdir(name string, perm FileMode) error
+	MkdirAll(name string, perm FileMode) error
 	Open(name string) (File, error)
+	OpenFile(name string, flags int, mode FileMode) (File, error)
 	ReadSymlink(name string) (string, error)
 	Remove(name string) error
+	RemoveAll(name string) error
 	Rename(oldname, newname string) error
 	Stat(name string) (FileInfo, error)
 	SymlinksSupported() bool
 	Walk(root string, walkFn WalkFunc) error
+	Hide(name string) error
+	Unhide(name string) error
+	Glob(pattern string) ([]string, error)
+	Roots() ([]string, error)
+	Usage(name string) (Usage, error)
+	Type() FilesystemType
+	URI() string
 }
 
 // The File interface abstracts access to a regular file, being a somewhat
 // smaller interface than os.File
 type File interface {
-	io.Reader
-	io.WriterAt
 	io.Closer
+	io.Reader
+	io.ReaderAt
+	io.Seeker
+	io.Writer
+	io.WriterAt
+	Name() string
 	Truncate(size int64) error
 	Stat() (FileInfo, error)
+	Sync() error
 }
 
 // The FileInfo interface is almost the same as os.FileInfo, but with the
@@ -59,12 +75,27 @@ type FileInfo interface {
 // FileMode is similar to os.FileMode
 type FileMode uint32
 
-// ModePerm is the equivalent of os.ModePerm
+// Usage represents filesystem space usage
-const ModePerm = FileMode(os.ModePerm)
+type Usage struct {
+	Free  int64
+	Total int64
+}
 
-// DefaultFilesystem is the fallback to use when nothing explicitly has
+// Equivalents from os package.
-// been passed.
+
-var DefaultFilesystem Filesystem = NewWalkFilesystem(NewBasicFilesystem())
+const ModePerm = FileMode(os.ModePerm)
+const ModeSetgid = FileMode(os.ModeSetgid)
+const ModeSetuid = FileMode(os.ModeSetuid)
+const ModeSticky = FileMode(os.ModeSticky)
+const PathSeparator = os.PathSeparator
+const OptAppend = os.O_APPEND
+const OptCreate = os.O_CREATE
+const OptExclusive = os.O_EXCL
+const OptReadOnly = os.O_RDONLY
+const OptReadWrite = os.O_RDWR
+const OptSync = os.O_SYNC
+const OptTruncate = os.O_TRUNC
+const OptWriteOnly = os.O_WRONLY
 
 // SkipDir is used as a return value from WalkFuncs to indicate that
 // the directory named in the call is to be skipped. It is not returned
@@ -76,3 +107,29 @@ var IsExist = os.IsExist
 
 // IsNotExist is the equivalent of os.IsNotExist
 var IsNotExist = os.IsNotExist
+
+// IsPermission is the equivalent of os.IsPermission
+var IsPermission = os.IsPermission
+
+// IsPathSeparator is the equivalent of os.IsPathSeparator
+var IsPathSeparator = os.IsPathSeparator
+
+func NewFilesystem(fsType FilesystemType, uri string) Filesystem {
+	var fs Filesystem
+	switch fsType {
+	case FilesystemTypeBasic:
+		fs = NewWalkFilesystem(newBasicFilesystem(uri))
+	default:
+		l.Debugln("Unknown filesystem", fsType, uri)
+		fs = &errorFilesystem{
+			fsType: fsType,
+			uri:    uri,
+			err:    errors.New("filesystem with type " + fsType.String() + " does not exist."),
+		}
+	}
+
+	if l.ShouldDebug("filesystem") {
+		fs = &logFilesystem{fs}
+	}
+	return fs
+}

lib/fs/logfs.go

@@ -0,0 +1,158 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package fs
+
+import (
+	"fmt"
+	"path/filepath"
+	"runtime"
+	"time"
+)
+
+type logFilesystem struct {
+	Filesystem
+}
+
+func getCaller() string {
+	_, file, line, ok := runtime.Caller(2)
+	if !ok {
+		return "unknown"
+	}
+	return fmt.Sprintf("%s:%d", filepath.Base(file), line)
+}
+
+func (fs *logFilesystem) Chmod(name string, mode FileMode) error {
+	err := fs.Filesystem.Chmod(name, mode)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Chmod", name, mode, err)
+	return err
+}
+
+func (fs *logFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error {
+	err := fs.Filesystem.Chtimes(name, atime, mtime)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Chtimes", name, atime, mtime, err)
+	return err
+}
+
+func (fs *logFilesystem) Create(name string) (File, error) {
+	file, err := fs.Filesystem.Create(name)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Create", name, file, err)
+	return file, err
+}
+
+func (fs *logFilesystem) CreateSymlink(name, target string) error {
+	err := fs.Filesystem.CreateSymlink(name, target)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "CreateSymlink", name, target, err)
+	return err
+}
+
+func (fs *logFilesystem) DirNames(name string) ([]string, error) {
+	names, err := fs.Filesystem.DirNames(name)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "DirNames", name, names, err)
+	return names, err
+}
+
+func (fs *logFilesystem) Lstat(name string) (FileInfo, error) {
+	info, err := fs.Filesystem.Lstat(name)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Lstat", name, info, err)
+	return info, err
+}
+
+func (fs *logFilesystem) Mkdir(name string, perm FileMode) error {
+	err := fs.Filesystem.Mkdir(name, perm)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Mkdir", name, perm, err)
+	return err
+}
+
+func (fs *logFilesystem) MkdirAll(name string, perm FileMode) error {
+	err := fs.Filesystem.MkdirAll(name, perm)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "MkdirAll", name, perm, err)
+	return err
+}
+
+func (fs *logFilesystem) Open(name string) (File, error) {
+	file, err := fs.Filesystem.Open(name)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Open", name, file, err)
+	return file, err
+}
+
+func (fs *logFilesystem) OpenFile(name string, flags int, mode FileMode) (File, error) {
+	file, err := fs.Filesystem.OpenFile(name, flags, mode)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "OpenFile", name, flags, mode, file, err)
+	return file, err
+}
+
+func (fs *logFilesystem) ReadSymlink(name string) (string, error) {
+	target, err := fs.Filesystem.ReadSymlink(name)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "ReadSymlink", name, target, err)
+	return target, err
+}
+
+func (fs *logFilesystem) Remove(name string) error {
+	err := fs.Filesystem.Remove(name)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Remove", name, err)
+	return err
+}
+
+func (fs *logFilesystem) RemoveAll(name string) error {
+	err := fs.Filesystem.RemoveAll(name)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "RemoveAll", name, err)
+	return err
+}
+
+func (fs *logFilesystem) Rename(oldname, newname string) error {
+	err := fs.Filesystem.Rename(oldname, newname)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Rename", oldname, newname, err)
+	return err
+}
+
+func (fs *logFilesystem) Stat(name string) (FileInfo, error) {
+	info, err := fs.Filesystem.Stat(name)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Stat", name, info, err)
+	return info, err
+}
+
+func (fs *logFilesystem) SymlinksSupported() bool {
+	supported := fs.Filesystem.SymlinksSupported()
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "SymlinksSupported", supported)
+	return supported
+}
+
+func (fs *logFilesystem) Walk(root string, walkFn WalkFunc) error {
+	err := fs.Filesystem.Walk(root, walkFn)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Walk", root, walkFn, err)
+	return err
+}
+
+func (fs *logFilesystem) Unhide(name string) error {
+	err := fs.Filesystem.Unhide(name)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Unhide", name, err)
+	return err
+}
+
+func (fs *logFilesystem) Hide(name string) error {
+	err := fs.Filesystem.Hide(name)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Hide", name, err)
+	return err
+}
+
+func (fs *logFilesystem) Glob(name string) ([]string, error) {
+	names, err := fs.Filesystem.Glob(name)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Glob", name, names, err)
+	return names, err
+}
+
+func (fs *logFilesystem) Roots() ([]string, error) {
+	roots, err := fs.Filesystem.Roots()
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Roots", roots, err)
+	return roots, err
+}
+
+func (fs *logFilesystem) Usage(name string) (Usage, error) {
+	usage, err := fs.Filesystem.Usage(name)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Usage", name, usage, err)
+	return usage, err
+}

lib/fs/mtimefs.go

@@ -6,12 +6,7 @@
 
 package fs
 
-import (
+import "time"
-	"os"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/osutil"
-)
 
 // The database is where we store the virtual mtimes
 type database interface {
@@ -20,36 +15,34 @@ type database interface {
 	Delete(key string)
 }
 
-// variable so that we can mock it for testing
-var osChtimes = os.Chtimes
-
 // The MtimeFS is a filesystem with nanosecond mtime precision, regardless
 // of what shenanigans the underlying filesystem gets up to. A nil MtimeFS
 // just does the underlying operations with no additions.
 type MtimeFS struct {
 	Filesystem
-	db database
+	chtimes func(string, time.Time, time.Time) error
+	db      database
 }
 
 func NewMtimeFS(underlying Filesystem, db database) *MtimeFS {
 	return &MtimeFS{
 		Filesystem: underlying,
+		chtimes:    underlying.Chtimes, // for mocking it out in the tests
 		db:         db,
 	}
 }
 
 func (f *MtimeFS) Chtimes(name string, atime, mtime time.Time) error {
 	if f == nil {
-		return osChtimes(name, atime, mtime)
+		return f.chtimes(name, atime, mtime)
 	}
 
 	// Do a normal Chtimes call, don't care if it succeeds or not.
-	osChtimes(name, atime, mtime)
+	f.chtimes(name, atime, mtime)
 
 	// Stat the file to see what happened. Here we *do* return an error,
-	// because it might be "does not exist" or similar. osutil.Lstat is the
+	// because it might be "does not exist" or similar.
-	// souped up version to account for Android breakage.
+	info, err := f.Filesystem.Lstat(name)
-	info, err := osutil.Lstat(name)
 	if err != nil {
 		return err
 	}

lib/fs/mtimefs_test.go

@@ -25,22 +25,22 @@ func TestMtimeFS(t *testing.T) {
 	// a random time with nanosecond precision
 	testTime := time.Unix(1234567890, 123456789)
 
-	mtimefs := NewMtimeFS(DefaultFilesystem, make(mapStore))
+	mtimefs := NewMtimeFS(newBasicFilesystem("."), make(mapStore))
 
 	// Do one Chtimes call that will go through to the normal filesystem
-	osChtimes = os.Chtimes
+	mtimefs.chtimes = os.Chtimes
 	if err := mtimefs.Chtimes("testdata/exists0", testTime, testTime); err != nil {
 		t.Error("Should not have failed:", err)
 	}
 
 	// Do one call that gets an error back from the underlying Chtimes
-	osChtimes = failChtimes
+	mtimefs.chtimes = failChtimes
 	if err := mtimefs.Chtimes("testdata/exists1", testTime, testTime); err != nil {
 		t.Error("Should not have failed:", err)
 	}
 
 	// Do one call that gets struck by an exceptionally evil Chtimes
-	osChtimes = evilChtimes
+	mtimefs.chtimes = evilChtimes
 	if err := mtimefs.Chtimes("testdata/exists2", testTime, testTime); err != nil {
 		t.Error("Should not have failed:", err)
 	}

lib/fs/types.go

@@ -0,0 +1,36 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package fs
+
+type FilesystemType int
+
+const (
+	FilesystemTypeBasic FilesystemType = iota // default is basic
+)
+
+func (t FilesystemType) String() string {
+	switch t {
+	case FilesystemTypeBasic:
+		return "basic"
+	default:
+		return "unknown"
+	}
+}
+
+func (t FilesystemType) MarshalText() ([]byte, error) {
+	return []byte(t.String()), nil
+}
+
+func (t *FilesystemType) UnmarshalText(bs []byte) error {
+	switch string(bs) {
+	case "basic":
+		*t = FilesystemTypeBasic
+	default:
+		*t = FilesystemTypeBasic
+	}
+	return nil
+}

lib/fs/util.go

@@ -0,0 +1,55 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package fs
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+var errNoHome = errors.New("no home directory found - set $HOME (or the platform equivalent)")
+
+func ExpandTilde(path string) (string, error) {
+	if path == "~" {
+		return getHomeDir()
+	}
+
+	path = filepath.FromSlash(path)
+	if !strings.HasPrefix(path, fmt.Sprintf("~%c", PathSeparator)) {
+		return path, nil
+	}
+
+	home, err := getHomeDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(home, path[2:]), nil
+}
+
+func getHomeDir() (string, error) {
+	var home string
+
+	switch runtime.GOOS {
+	case "windows":
+		home = filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath"))
+		if home == "" {
+			home = os.Getenv("UserProfile")
+		}
+	default:
+		home = os.Getenv("HOME")
+	}
+
+	if home == "" {
+		return "", errNoHome
+	}
+
+	return home, nil
+}

lib/fs/walkfs.go

@@ -28,16 +28,16 @@ import "path/filepath"
 // Walk skips the remaining files in the containing directory.
 type WalkFunc func(path string, info FileInfo, err error) error
 
-type WalkFilesystem struct {
+type walkFilesystem struct {
 	Filesystem
 }
 
-func NewWalkFilesystem(next Filesystem) *WalkFilesystem {
+func NewWalkFilesystem(next Filesystem) Filesystem {
-	return &WalkFilesystem{next}
+	return &walkFilesystem{next}
 }
 
 // walk recursively descends path, calling walkFn.
-func (f *WalkFilesystem) walk(path string, info FileInfo, walkFn WalkFunc) error {
+func (f *walkFilesystem) walk(path string, info FileInfo, walkFn WalkFunc) error {
 	err := walkFn(path, info, nil)
 	if err != nil {
 		if info.IsDir() && err == SkipDir {
@@ -80,7 +80,7 @@ func (f *WalkFilesystem) walk(path string, info FileInfo, walkFn WalkFunc) error
 // order, which makes the output deterministic but means that for very
 // large directories Walk can be inefficient.
 // Walk does not follow symbolic links.
-func (f *WalkFilesystem) Walk(root string, walkFn WalkFunc) error {
+func (f *walkFilesystem) Walk(root string, walkFn WalkFunc) error {
 	info, err := f.Lstat(root)
 	if err != nil {
 		return walkFn(root, nil, err)

lib/ignore/ignore.go

@@ -12,13 +12,13 @@ import (
 	"crypto/md5"
 	"fmt"
 	"io"
-	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"time"
 
 	"github.com/gobwas/glob"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/sync"
 )
@@ -77,6 +77,7 @@ type ChangeDetector interface {
 }
 
 type Matcher struct {
+	fs             fs.Filesystem
 	lines          []string  // exact lines read from .stignore
 	patterns       []Pattern // patterns including those from included files
 	withCache      bool
@@ -105,8 +106,9 @@ func WithChangeDetector(cd ChangeDetector) Option {
 	}
 }
 
-func New(opts ...Option) *Matcher {
+func New(fs fs.Filesystem, opts ...Option) *Matcher {
 	m := &Matcher{
+		fs:   fs,
 		stop: make(chan struct{}),
 		mut:  sync.NewMutex(),
 	}
@@ -114,7 +116,7 @@ func New(opts ...Option) *Matcher {
 		opt(m)
 	}
 	if m.changeDetector == nil {
-		m.changeDetector = newModtimeChecker()
+		m.changeDetector = newModtimeChecker(fs)
 	}
 	if m.withCache {
 		go m.clean(2 * time.Hour)
@@ -130,7 +132,7 @@ func (m *Matcher) Load(file string) error {
 		return nil
 	}
 
-	fd, err := os.Open(file)
+	fd, err := m.fs.Open(file)
 	if err != nil {
 		m.parseLocked(&bytes.Buffer{}, file)
 		return err
@@ -156,7 +158,7 @@ func (m *Matcher) Parse(r io.Reader, file string) error {
 }
 
 func (m *Matcher) parseLocked(r io.Reader, file string) error {
-	lines, patterns, err := parseIgnoreFile(r, file, m.changeDetector)
+	lines, patterns, err := parseIgnoreFile(m.fs, r, file, m.changeDetector)
 	// Error is saved and returned at the end. We process the patterns
 	// (possibly blank) anyway.
 
@@ -298,12 +300,12 @@ func hashPatterns(patterns []Pattern) string {
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
 
-func loadIgnoreFile(file string, cd ChangeDetector) ([]string, []Pattern, error) {
+func loadIgnoreFile(fs fs.Filesystem, file string, cd ChangeDetector) ([]string, []Pattern, error) {
 	if cd.Seen(file) {
 		return nil, nil, fmt.Errorf("multiple include of ignore file %q", file)
 	}
 
-	fd, err := os.Open(file)
+	fd, err := fs.Open(file)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -316,10 +318,10 @@ func loadIgnoreFile(file string, cd ChangeDetector) ([]string, []Pattern, error)
 
 	cd.Remember(file, info.ModTime())
 
-	return parseIgnoreFile(fd, file, cd)
+	return parseIgnoreFile(fs, fd, file, cd)
 }
 
-func parseIgnoreFile(fd io.Reader, currentFile string, cd ChangeDetector) ([]string, []Pattern, error) {
+func parseIgnoreFile(fs fs.Filesystem, fd io.Reader, currentFile string, cd ChangeDetector) ([]string, []Pattern, error) {
 	var lines []string
 	var patterns []Pattern
 
@@ -386,7 +388,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, cd ChangeDetector) ([]str
 		} else if strings.HasPrefix(line, "#include ") {
 			includeRel := line[len("#include "):]
 			includeFile := filepath.Join(filepath.Dir(currentFile), includeRel)
-			_, includePatterns, err := loadIgnoreFile(includeFile, cd)
+			_, includePatterns, err := loadIgnoreFile(fs, includeFile, cd)
 			if err != nil {
 				return fmt.Errorf("include of %q: %v", includeRel, err)
 			}
@@ -450,7 +452,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, cd ChangeDetector) ([]str
 // path must be clean (i.e., in canonical shortest form).
 func IsInternal(file string) bool {
 	internals := []string{".stfolder", ".stignore", ".stversions"}
-	pathSep := string(os.PathSeparator)
+	pathSep := string(fs.PathSeparator)
 	for _, internal := range internals {
 		if file == internal {
 			return true
@@ -463,8 +465,8 @@ func IsInternal(file string) bool {
 }
 
 // WriteIgnores is a convenience function to avoid code duplication
-func WriteIgnores(path string, content []string) error {
+func WriteIgnores(filesystem fs.Filesystem, path string, content []string) error {
-	fd, err := osutil.CreateAtomic(path)
+	fd, err := osutil.CreateAtomicFilesystem(filesystem, path)
 	if err != nil {
 		return err
 	}
@@ -476,18 +478,20 @@ func WriteIgnores(path string, content []string) error {
 	if err := fd.Close(); err != nil {
 		return err
 	}
-	osutil.HideFile(path)
+	filesystem.Hide(path)
 
 	return nil
 }
 
 // modtimeChecker is the default implementation of ChangeDetector
 type modtimeChecker struct {
+	fs       fs.Filesystem
 	modtimes map[string]time.Time
 }
 
-func newModtimeChecker() *modtimeChecker {
+func newModtimeChecker(fs fs.Filesystem) *modtimeChecker {
 	return &modtimeChecker{
+		fs:       fs,
 		modtimes: map[string]time.Time{},
 	}
 }
@@ -507,7 +511,7 @@ func (c *modtimeChecker) Reset() {
 
 func (c *modtimeChecker) Changed() bool {
 	for name, modtime := range c.modtimes {
-		info, err := os.Stat(name)
+		info, err := c.fs.Stat(name)
 		if err != nil {
 			return true
 		}

lib/ignore/ignore_test.go

@@ -15,11 +15,14 @@ import (
 	"runtime"
 	"testing"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/fs"
+	"github.com/syncthing/syncthing/lib/osutil"
 )
 
 func TestIgnore(t *testing.T) {
-	pats := New(WithCache(true))
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), WithCache(true))
-	err := pats.Load("testdata/.stignore")
+	err := pats.Load(".stignore")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -68,7 +71,7 @@ func TestExcludes(t *testing.T) {
 	i*2
 	!ign2
 	`
-	pats := New(WithCache(true))
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -113,7 +116,7 @@ func TestFlagOrder(t *testing.T) {
 	(?i)(?d)(?d)!ign9
 	(?d)(?d)!ign10
 	`
-	pats := New(WithCache(true))
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -148,7 +151,7 @@ func TestDeletables(t *testing.T) {
 	ign7
 	(?i)ign8
 	`
-	pats := New(WithCache(true))
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -187,7 +190,7 @@ func TestBadPatterns(t *testing.T) {
 	}
 
 	for _, pat := range badPatterns {
-		err := New(WithCache(true)).Parse(bytes.NewBufferString(pat), ".stignore")
+		err := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)).Parse(bytes.NewBufferString(pat), ".stignore")
 		if err == nil {
 			t.Errorf("No error for pattern %q", pat)
 		}
@@ -195,7 +198,7 @@ func TestBadPatterns(t *testing.T) {
 }
 
 func TestCaseSensitivity(t *testing.T) {
-	ign := New(WithCache(true))
+	ign := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := ign.Parse(bytes.NewBufferString("test"), ".stignore")
 	if err != nil {
 		t.Error(err)
@@ -225,29 +228,36 @@ func TestCaseSensitivity(t *testing.T) {
 }
 
 func TestCaching(t *testing.T) {
-	fd1, err := ioutil.TempFile("", "")
+	dir, err := ioutil.TempDir("", "")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	fd2, err := ioutil.TempFile("", "")
+	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
+
+	fd1, err := osutil.TempFile(fs, "", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fd2, err := osutil.TempFile(fs, "", "")
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	defer fd1.Close()
 	defer fd2.Close()
-	defer os.Remove(fd1.Name())
+	defer fs.Remove(fd1.Name())
-	defer os.Remove(fd2.Name())
+	defer fs.Remove(fd2.Name())
 
-	_, err = fd1.WriteString("/x/\n#include " + filepath.Base(fd2.Name()) + "\n")
+	_, err = fd1.Write([]byte("/x/\n#include " + filepath.Base(fd2.Name()) + "\n"))
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	fd2.WriteString("/y/\n")
+	fd2.Write([]byte("/y/\n"))
 
-	pats := New(WithCache(true))
+	pats := New(fs, WithCache(true))
 	err = pats.Load(fd1.Name())
 	if err != nil {
 		t.Fatal(err)
@@ -280,10 +290,10 @@ func TestCaching(t *testing.T) {
 	// Modify the include file, expect empty cache. Ensure the timestamp on
 	// the file changes.
 
-	fd2.WriteString("/z/\n")
+	fd2.Write([]byte("/z/\n"))
 	fd2.Sync()
 	fakeTime := time.Now().Add(5 * time.Second)
-	os.Chtimes(fd2.Name(), fakeTime, fakeTime)
+	fs.Chtimes(fd2.Name(), fakeTime, fakeTime)
 
 	err = pats.Load(fd1.Name())
 	if err != nil {
@@ -312,10 +322,10 @@ func TestCaching(t *testing.T) {
 
 	// Modify the root file, expect cache to be invalidated
 
-	fd1.WriteString("/a/\n")
+	fd1.Write([]byte("/a/\n"))
 	fd1.Sync()
 	fakeTime = time.Now().Add(5 * time.Second)
-	os.Chtimes(fd1.Name(), fakeTime, fakeTime)
+	fs.Chtimes(fd1.Name(), fakeTime, fakeTime)
 
 	err = pats.Load(fd1.Name())
 	if err != nil {
@@ -354,7 +364,7 @@ func TestCommentsAndBlankLines(t *testing.T) {
 
 
 	`
-	pats := New(WithCache(true))
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Error(err)
@@ -382,7 +392,7 @@ flamingo
 *.crow
 *.crow
 	`
-	pats := New()
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."))
 	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		b.Error(err)
@@ -411,20 +421,27 @@ flamingo
 *.crow
 	`
 	// Caches per file, hence write the patterns to a file.
-	fd, err := ioutil.TempFile("", "")
+	dir, err := ioutil.TempDir("", "")
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	_, err = fd.WriteString(stignore)
+	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
+
+	fd, err := osutil.TempFile(fs, "", "")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	_, err = fd.Write([]byte(stignore))
 	defer fd.Close()
-	defer os.Remove(fd.Name())
+	defer fs.Remove(fd.Name())
 	if err != nil {
 		b.Fatal(err)
 	}
 
 	// Load the patterns
-	pats := New(WithCache(true))
+	pats := New(fs, WithCache(true))
 	err = pats.Load(fd.Name())
 	if err != nil {
 		b.Fatal(err)
@@ -445,22 +462,29 @@ flamingo
 }
 
 func TestCacheReload(t *testing.T) {
-	fd, err := ioutil.TempFile("", "")
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
+
+	fd, err := osutil.TempFile(fs, "", "")
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	defer fd.Close()
-	defer os.Remove(fd.Name())
+	defer fs.Remove(fd.Name())
 
 	// Ignore file matches f1 and f2
 
-	_, err = fd.WriteString("f1\nf2\n")
+	_, err = fd.Write([]byte("f1\nf2\n"))
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	pats := New(WithCache(true))
+	pats := New(fs, WithCache(true))
 	err = pats.Load(fd.Name())
 	if err != nil {
 		t.Fatal(err)
@@ -488,13 +512,13 @@ func TestCacheReload(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = fd.WriteString("f1\nf3\n")
+	_, err = fd.Write([]byte("f1\nf3\n"))
 	if err != nil {
 		t.Fatal(err)
 	}
 	fd.Sync()
 	fakeTime := time.Now().Add(5 * time.Second)
-	os.Chtimes(fd.Name(), fakeTime, fakeTime)
+	fs.Chtimes(fd.Name(), fakeTime, fakeTime)
 
 	err = pats.Load(fd.Name())
 	if err != nil {
@@ -515,7 +539,7 @@ func TestCacheReload(t *testing.T) {
 }
 
 func TestHash(t *testing.T) {
-	p1 := New(WithCache(true))
+	p1 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := p1.Load("testdata/.stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -531,7 +555,7 @@ func TestHash(t *testing.T) {
 	/ffile
 	lost+found
 	`
-	p2 := New(WithCache(true))
+	p2 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err = p2.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -546,7 +570,7 @@ func TestHash(t *testing.T) {
 	/ffile
 	lost+found
 	`
-	p3 := New(WithCache(true))
+	p3 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err = p3.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -570,7 +594,7 @@ func TestHash(t *testing.T) {
 }
 
 func TestHashOfEmpty(t *testing.T) {
-	p1 := New(WithCache(true))
+	p1 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := p1.Load("testdata/.stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -608,7 +632,7 @@ func TestWindowsPatterns(t *testing.T) {
 	a/b
 	c\d
 	`
-	pats := New(WithCache(true))
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -633,7 +657,7 @@ func TestAutomaticCaseInsensitivity(t *testing.T) {
 	A/B
 	c/d
 	`
-	pats := New(WithCache(true))
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -652,7 +676,7 @@ func TestCommas(t *testing.T) {
 	foo,bar.txt
 	{baz,quux}.txt
 	`
-	pats := New(WithCache(true))
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -683,7 +707,7 @@ func TestIssue3164(t *testing.T) {
 	(?d)(?i)/foo
 	(?d)(?i)**/bar
 	`
-	pats := New(WithCache(true))
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -719,7 +743,7 @@ func TestIssue3174(t *testing.T) {
 	stignore := `
 	*ä*
 	`
-	pats := New(WithCache(true))
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -734,7 +758,7 @@ func TestIssue3639(t *testing.T) {
 	stignore := `
 	foo/
 	`
-	pats := New(WithCache(true))
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -767,7 +791,7 @@ func TestIssue3674(t *testing.T) {
 		{"as/dc", true},
 	}
 
-	pats := New(WithCache(true))
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -799,7 +823,7 @@ func TestGobwasGlobIssue18(t *testing.T) {
 		{"bbaa", false},
 	}
 
-	pats := New(WithCache(true))
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -859,7 +883,7 @@ func TestRoot(t *testing.T) {
 		{"b", true},
 	}
 
-	pats := New(WithCache(true))
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -876,12 +900,12 @@ func TestRoot(t *testing.T) {
 func TestLines(t *testing.T) {
 	stignore := `
 	#include testdata/excludes
-	
+
 	!/a
 	/*
 	`
 
-	pats := New(WithCache(true))
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
 	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)

lib/model/model.go

@@ -14,7 +14,6 @@ import (
 	"fmt"
 	"io"
 	"net"
-	"os"
 	"path/filepath"
 	"reflect"
 	"runtime"
@@ -81,6 +80,7 @@ type Model struct {
 	clientVersion string
 
 	folderCfgs         map[string]config.FolderConfiguration                  // folder -> cfg
+	folderFs           map[string]fs.Filesystem                               // folder -> fs
 	folderFiles        map[string]*db.FileSet                                 // folder -> files
 	folderDevices      folderDeviceSet                                        // folder -> deviceIDs
 	deviceFolders      map[protocol.DeviceID][]string                         // deviceID -> folders
@@ -99,21 +99,18 @@ type Model struct {
 	pmut                sync.RWMutex                   // protects the above
 }
 
-type folderFactory func(*Model, config.FolderConfiguration, versioner.Versioner, *fs.MtimeFS) service
+type folderFactory func(*Model, config.FolderConfiguration, versioner.Versioner, fs.Filesystem) service
 
 var (
 	folderFactories = make(map[config.FolderType]folderFactory, 0)
 )
 
 var (
-	errFolderPathEmpty     = errors.New("folder path empty")
 	errFolderPathMissing   = errors.New("folder path missing")
 	errFolderMarkerMissing = errors.New("folder marker missing")
-	errInvalidFilename     = errors.New("filename is invalid")
 	errDeviceUnknown       = errors.New("unknown device")
 	errDevicePaused        = errors.New("device is paused")
 	errDeviceIgnored       = errors.New("device is ignored")
-	errNotRelative         = errors.New("not a relative path")
 	errFolderPaused        = errors.New("folder is paused")
 	errFolderMissing       = errors.New("no such folder")
 	errNetworkNotAllowed   = errors.New("network not allowed")
@@ -140,6 +137,7 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, clientName, clientVersi
 		clientName:          clientName,
 		clientVersion:       clientVersion,
 		folderCfgs:          make(map[string]config.FolderConfiguration),
+		folderFs:            make(map[string]fs.Filesystem),
 		folderFiles:         make(map[string]*db.FileSet),
 		folderDevices:       make(folderDeviceSet),
 		deviceFolders:       make(map[protocol.DeviceID][]string),
@@ -245,7 +243,7 @@ func (m *Model) startFolderLocked(folder string) config.FolderType {
 			l.Fatalf("Requested versioning type %q that does not exist", cfg.Versioning.Type)
 		}
 
-		ver = versionerFactory(folder, cfg.Path(), cfg.Versioning.Params)
+		ver = versionerFactory(folder, cfg.Filesystem(), cfg.Versioning.Params)
 		if service, ok := ver.(suture.Service); ok {
 			// The versioner implements the suture.Service interface, so
 			// expects to be run in the background in addition to being called
@@ -271,7 +269,12 @@ func (m *Model) warnAboutOverwritingProtectedFiles(folder string) {
 		return
 	}
 
-	folderLocation := m.folderCfgs[folder].Path()
+	// This is a bit of a hack.
+	ffs := m.folderCfgs[folder].Filesystem()
+	if ffs.Type() != fs.FilesystemTypeBasic {
+		return
+	}
+	folderLocation := ffs.URI()
 	ignores := m.folderIgnores[folder]
 
 	var filesAtRisk []string
@@ -300,6 +303,10 @@ func (m *Model) AddFolder(cfg config.FolderConfiguration) {
 		panic("cannot add empty folder id")
 	}
 
+	if len(cfg.Path) == 0 {
+		panic("cannot add empty folder path")
+	}
+
 	m.fmut.Lock()
 	m.addFolderLocked(cfg)
 	m.fmut.Unlock()
@@ -307,15 +314,16 @@ func (m *Model) AddFolder(cfg config.FolderConfiguration) {
 
 func (m *Model) addFolderLocked(cfg config.FolderConfiguration) {
 	m.folderCfgs[cfg.ID] = cfg
-	m.folderFiles[cfg.ID] = db.NewFileSet(cfg.ID, m.db)
+	folderFs := cfg.Filesystem()
+	m.folderFiles[cfg.ID] = db.NewFileSet(cfg.ID, folderFs, m.db)
 
 	for _, device := range cfg.Devices {
 		m.folderDevices.set(device.DeviceID, cfg.ID)
 		m.deviceFolders[device.DeviceID] = append(m.deviceFolders[device.DeviceID], cfg.ID)
 	}
 
-	ignores := ignore.New(ignore.WithCache(m.cacheIgnoredFiles))
+	ignores := ignore.New(folderFs, ignore.WithCache(m.cacheIgnoredFiles))
-	if err := ignores.Load(filepath.Join(cfg.Path(), ".stignore")); err != nil && !os.IsNotExist(err) {
+	if err := ignores.Load(".stignore"); err != nil && !fs.IsNotExist(err) {
 		l.Warnln("Loading ignores:", err)
 	}
 	m.folderIgnores[cfg.ID] = ignores
@@ -327,8 +335,8 @@ func (m *Model) RemoveFolder(folder string) {
 
 	// Delete syncthing specific files
 	folderCfg := m.folderCfgs[folder]
-	folderPath := folderCfg.Path()
+	fs := folderCfg.Filesystem()
-	os.Remove(filepath.Join(folderPath, ".stfolder"))
+	fs.Remove(".stfolder")
 
 	m.tearDownFolderLocked(folder)
 	// Remove it from the database
@@ -1139,16 +1147,10 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 	}
 	m.fmut.RLock()
 	folderCfg := m.folderCfgs[folder]
-	folderPath := folderCfg.Path()
 	folderIgnores := m.folderIgnores[folder]
 	m.fmut.RUnlock()
 
-	fn, err := rootedJoinedPath(folderPath, name)
+	folderFs := folderCfg.Filesystem()
-	if err != nil {
-		// Request tries to escape!
-		l.Debugf("%v Invalid REQ(in) tries to escape: %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, len(buf))
-		return protocol.ErrInvalid
-	}
 
 	// Having passed the rootedJoinedPath check above, we know "name" is
 	// acceptable relative to "folderPath" and in canonical form, so we can
@@ -1164,7 +1166,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 		return protocol.ErrNoSuchFile
 	}
 
-	if err := osutil.TraversesSymlink(folderPath, filepath.Dir(name)); err != nil {
+	if err := osutil.TraversesSymlink(folderFs, filepath.Dir(name)); err != nil {
 		l.Debugf("%v REQ(in) traversal check: %s - %s: %q / %q o=%d s=%d", m, err, deviceID, folder, name, offset, len(buf))
 		return protocol.ErrNoSuchFile
 	}
@@ -1172,29 +1174,29 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 	// Only check temp files if the flag is set, and if we are set to advertise
 	// the temp indexes.
 	if fromTemporary && !folderCfg.DisableTempIndexes {
-		tempFn := filepath.Join(folderPath, ignore.TempName(name))
+		tempFn := ignore.TempName(name)
 
-		if info, err := osutil.Lstat(tempFn); err != nil || !info.Mode().IsRegular() {
+		if info, err := folderFs.Lstat(tempFn); err != nil || !info.IsRegular() {
 			// Reject reads for anything that doesn't exist or is something
 			// other than a regular file.
 			return protocol.ErrNoSuchFile
 		}
 
-		if err := readOffsetIntoBuf(tempFn, offset, buf); err == nil {
+		if err := readOffsetIntoBuf(folderFs, tempFn, offset, buf); err == nil {
 			return nil
 		}
 		// Fall through to reading from a non-temp file, just incase the temp
 		// file has finished downloading.
 	}
 
-	if info, err := osutil.Lstat(fn); err != nil || !info.Mode().IsRegular() {
+	if info, err := folderFs.Lstat(name); err != nil || !info.IsRegular() {
 		// Reject reads for anything that doesn't exist or is something
 		// other than a regular file.
 		return protocol.ErrNoSuchFile
 	}
 
-	err = readOffsetIntoBuf(fn, offset, buf)
+	err := readOffsetIntoBuf(folderFs, name, offset, buf)
-	if os.IsNotExist(err) {
+	if fs.IsNotExist(err) {
 		return protocol.ErrNoSuchFile
 	} else if err != nil {
 		return protocol.ErrGeneric
@@ -1259,9 +1261,8 @@ func (m *Model) GetIgnores(folder string) ([]string, []string, error) {
 	}
 
 	if cfg, ok := m.cfg.Folders()[folder]; ok {
-		matcher := ignore.New()
+		matcher := ignore.New(cfg.Filesystem())
-		path := filepath.Join(cfg.Path(), ".stignore")
+		if err := matcher.Load(".stignore"); err != nil {
-		if err := matcher.Load(path); err != nil {
 			return nil, nil, err
 		}
 		return matcher.Lines(), matcher.Patterns(), nil
@@ -1276,7 +1277,7 @@ func (m *Model) SetIgnores(folder string, content []string) error {
 		return fmt.Errorf("Folder %s does not exist", folder)
 	}
 
-	if err := ignore.WriteIgnores(filepath.Join(cfg.Path(), ".stignore"), content); err != nil {
+	if err := ignore.WriteIgnores(cfg.Filesystem(), ".stignore", content); err != nil {
 		l.Warnln("Saving .stignore:", err)
 		return err
 	}
@@ -1610,8 +1611,6 @@ func (m *Model) updateLocals(folder string, fs []protocol.FileInfo) {
 }
 
 func (m *Model) diskChangeDetected(folderCfg config.FolderConfiguration, files []protocol.FileInfo, typeOfEvent events.EventType) {
-	path := strings.Replace(folderCfg.Path(), `\\?\`, "", 1)
-
 	for _, file := range files {
 		objType := "file"
 		action := "modified"
@@ -1634,10 +1633,6 @@ func (m *Model) diskChangeDetected(folderCfg config.FolderConfiguration, files [
 			action = "deleted"
 		}
 
-		// The full file path, adjusted to the local path separator character.  Also
-		// for windows paths, strip unwanted chars from the front.
-		path := filepath.Join(path, filepath.FromSlash(file.Name))
-
 		// Two different events can be fired here based on what EventType is passed into function
 		events.Default.Log(typeOfEvent, map[string]string{
 			"folder":     folderCfg.ID,
@@ -1645,7 +1640,7 @@ func (m *Model) diskChangeDetected(folderCfg config.FolderConfiguration, files [
 			"label":      folderCfg.Label,
 			"action":     action,
 			"type":       objType,
-			"path":       path,
+			"path":       filepath.FromSlash(file.Name),
 			"modifiedBy": file.ModifiedBy.String(),
 		})
 	}
@@ -1738,20 +1733,17 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
 		// not relevant, we just want the dotdot escape detection here. For
 		// historical reasons we may get paths that end in a slash. We
 		// remove that first to allow the rootedJoinedPath to pass.
-		sub = strings.TrimRight(sub, string(os.PathSeparator))
+		sub = strings.TrimRight(sub, string(fs.PathSeparator))
-		if _, err := rootedJoinedPath("root", sub); err != nil {
-			return errors.New("invalid subpath")
-		}
 		subDirs[i] = sub
 	}
 
 	m.fmut.Lock()
-	fs := m.folderFiles[folder]
+	fset := m.folderFiles[folder]
 	folderCfg := m.folderCfgs[folder]
 	ignores := m.folderIgnores[folder]
 	runner, ok := m.folderRunners[folder]
 	m.fmut.Unlock()
-	mtimefs := fs.MtimeFS()
+	mtimefs := fset.MtimeFS()
 
 	// Check if the ignore patterns changed as part of scanning this folder.
 	// If they did we should schedule a pull of the folder so that we
@@ -1778,7 +1770,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
 		return err
 	}
 
-	if err := ignores.Load(filepath.Join(folderCfg.Path(), ".stignore")); err != nil && !os.IsNotExist(err) {
+	if err := ignores.Load(".stignore"); err != nil && !fs.IsNotExist(err) {
 		err = fmt.Errorf("loading ignores: %v", err)
 		runner.setError(err)
 		l.Infof("Stopping folder %s due to error: %s", folderCfg.Description(), err)
@@ -1789,7 +1781,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
 	// directory, and don't scan subdirectories of things we've already
 	// scanned.
 	subDirs = unifySubs(subDirs, func(f string) bool {
-		_, ok := fs.Get(protocol.LocalDeviceID, f)
+		_, ok := fset.Get(protocol.LocalDeviceID, f)
 		return ok
 	})
 
@@ -1797,7 +1789,6 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
 
 	fchan, err := scanner.Walk(ctx, scanner.Config{
 		Folder:                folderCfg.ID,
-		Dir:                   folderCfg.Path(),
 		Subs:                  subDirs,
 		Matcher:               ignores,
 		BlockSize:             protocol.BlockSize,
@@ -1860,7 +1851,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
 	for _, sub := range subDirs {
 		var iterError error
 
-		fs.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool {
+		fset.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool {
 			f := fi.(db.FileInfoTruncated)
 			if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
 				if err := m.CheckFolderHealth(folder); err != nil {
@@ -1895,9 +1886,9 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
 				// The file is valid and not deleted. Lets check if it's
 				// still here.
 
-				if _, err := mtimefs.Lstat(filepath.Join(folderCfg.Path(), f.Name)); err != nil {
+				if _, err := mtimefs.Lstat(f.Name); err != nil {
 					// We don't specifically verify that the error is
-					// os.IsNotExist because there is a corner case when a
+					// fs.IsNotExist because there is a corner case when a
 					// directory is suddenly transformed into a file. When that
 					// happens, files that were in the directory (that is now a
 					// file) are deleted but will return a confusing error ("not a
@@ -2275,11 +2266,9 @@ func (m *Model) CheckFolderHealth(id string) error {
 
 // checkFolderPath returns nil if the folder path exists and has the marker file.
 func (m *Model) checkFolderPath(folder config.FolderConfiguration) error {
-	if folder.Path() == "" {
+	fs := folder.Filesystem()
-		return errFolderPathEmpty
-	}
 
-	if fi, err := os.Stat(folder.Path()); err != nil || !fi.IsDir() {
+	if fi, err := fs.Stat("."); err != nil || !fi.IsDir() {
 		return errFolderPathMissing
 	}
 
@@ -2293,30 +2282,35 @@ func (m *Model) checkFolderPath(folder config.FolderConfiguration) error {
 // checkFolderFreeSpace returns nil if the folder has the required amount of
 // free space, or if folder free space checking is disabled.
 func (m *Model) checkFolderFreeSpace(folder config.FolderConfiguration) error {
-	return m.checkFreeSpace(folder.MinDiskFree, folder.Path())
+	return m.checkFreeSpace(folder.MinDiskFree, folder.Filesystem())
 }
 
 // checkHomeDiskFree returns nil if the home disk has the required amount of
 // free space, or if home disk free space checking is disabled.
 func (m *Model) checkHomeDiskFree() error {
-	return m.checkFreeSpace(m.cfg.Options().MinHomeDiskFree, m.cfg.ConfigPath())
+	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, filepath.Dir(m.cfg.ConfigPath()))
+	return m.checkFreeSpace(m.cfg.Options().MinHomeDiskFree, fs)
 }
 
-func (m *Model) checkFreeSpace(req config.Size, path string) error {
+func (m *Model) checkFreeSpace(req config.Size, fs fs.Filesystem) error {
 	val := req.BaseValue()
 	if val <= 0 {
 		return nil
 	}
 
+	usage, err := fs.Usage(".")
+	if err != nil {
+		return fmt.Errorf("failed to check available storage space")
+	}
+
 	if req.Percentage() {
-		free, err := osutil.DiskFreePercentage(path)
+		freePct := (1 - float64(usage.Free)/float64(usage.Total)) * 100
-		if err == nil && free < val {
+		if err == nil && freePct < val {
-			return fmt.Errorf("insufficient space in %v: %f %% < %v", path, free, req)
+			return fmt.Errorf("insufficient space in %v %v: %f %% < %v", fs.Type(), fs.URI(), freePct, req)
 		}
 	} else {
-		free, err := osutil.DiskFreeBytes(path)
+		if err == nil && float64(usage.Free) < val {
-		if err == nil && float64(free) < val {
+			return fmt.Errorf("insufficient space in %v %v: %v < %v", fs.Type(), fs.URI(), usage.Free, req)
-			return fmt.Errorf("insufficient space in %v: %v < %v", path, free, req)
 		}
 	}
 
@@ -2533,8 +2527,8 @@ func stringSliceWithout(ss []string, s string) []string {
 	return ss
 }
 
-func readOffsetIntoBuf(file string, offset int64, buf []byte) error {
+func readOffsetIntoBuf(fs fs.Filesystem, file string, offset int64, buf []byte) error {
-	fd, err := os.Open(file)
+	fd, err := fs.Open(file)
 	if err != nil {
 		l.Debugln("readOffsetIntoBuf.Open", file, err)
 		return err
@@ -2585,7 +2579,7 @@ func simplifySortedPaths(subs []string) []string {
 next:
 	for _, sub := range subs {
 		for _, existing := range cleaned {
-			if sub == existing || strings.HasPrefix(sub, existing+string(os.PathSeparator)) {
+			if sub == existing || strings.HasPrefix(sub, existing+string(fs.PathSeparator)) {
 				continue next
 			}
 		}
@@ -2666,57 +2660,3 @@ func (s folderDeviceSet) sortedDevices(folder string) []protocol.DeviceID {
 	sort.Sort(protocol.DeviceIDs(devs))
 	return devs
 }
-
-// rootedJoinedPath takes a root and a supposedly relative path inside that
-// root and returns the joined path. An error is returned if the joined path
-// is not in fact inside the root.
-func rootedJoinedPath(root, rel string) (string, error) {
-	// The root must not be empty.
-	if root == "" {
-		return "", errInvalidFilename
-	}
-
-	pathSep := string(os.PathSeparator)
-
-	// The expected prefix for the resulting path is the root, with a path
-	// separator at the end.
-	expectedPrefix := filepath.FromSlash(root)
-	if !strings.HasSuffix(expectedPrefix, pathSep) {
-		expectedPrefix += pathSep
-	}
-
-	// The relative path should be clean from internal dotdots and similar
-	// funkyness.
-	rel = filepath.FromSlash(rel)
-	if filepath.Clean(rel) != rel {
-		return "", errInvalidFilename
-	}
-
-	// It is not acceptable to attempt to traverse upwards or refer to the
-	// root itself.
-	switch rel {
-	case ".", "..", pathSep:
-		return "", errNotRelative
-	}
-	if strings.HasPrefix(rel, ".."+pathSep) {
-		return "", errNotRelative
-	}
-
-	if strings.HasPrefix(rel, pathSep+pathSep) {
-		// The relative path may pretend to be an absolute path within the
-		// root, but the double path separator on Windows implies something
-		// else. It would get cleaned by the Join below, but it's out of
-		// spec anyway.
-		return "", errNotRelative
-	}
-
-	// The supposedly correct path is the one filepath.Join will return, as
-	// it does cleaning and so on. Check that one first to make sure no
-	// obvious escape attempts have been made.
-	joined := filepath.Join(root, rel)
-	if !strings.HasPrefix(joined, expectedPrefix) {
-		return "", errNotRelative
-	}
-
-	return joined, nil
-}

lib/model/model_test.go

@@ -25,8 +25,8 @@ import (
 	"github.com/d4l3k/messagediff"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/ignore"
-	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	srand "github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/scanner"
@@ -35,12 +35,14 @@ import (
 var device1, device2 protocol.DeviceID
 var defaultConfig *config.Wrapper
 var defaultFolderConfig config.FolderConfiguration
+var defaultFs fs.Filesystem
 
 func init() {
 	device1, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
 	device2, _ = protocol.DeviceIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
+	defaultFs = fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata")
 
-	defaultFolderConfig = config.NewFolderConfiguration("default", "testdata")
+	defaultFolderConfig = config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "testdata")
 	defaultFolderConfig.Devices = []config.FolderDeviceConfiguration{{DeviceID: device1}}
 	_defaultConfig := config.Configuration{
 		Folders: []config.FolderConfiguration{defaultFolderConfig},
@@ -513,14 +515,16 @@ func TestClusterConfig(t *testing.T) {
 	}
 	cfg.Folders = []config.FolderConfiguration{
 		{
-			ID: "folder1",
+			ID:   "folder1",
+			Path: "testdata",
 			Devices: []config.FolderDeviceConfiguration{
 				{DeviceID: device1},
 				{DeviceID: device2},
 			},
 		},
 		{
-			ID: "folder2",
+			ID:   "folder2",
+			Path: "testdata",
 			Devices: []config.FolderDeviceConfiguration{
 				{DeviceID: device1},
 				{DeviceID: device2},
@@ -622,13 +626,15 @@ func TestIntroducer(t *testing.T) {
 		},
 		Folders: []config.FolderConfiguration{
 			{
-				ID: "folder1",
+				ID:   "folder1",
+				Path: "testdata",
 				Devices: []config.FolderDeviceConfiguration{
 					{DeviceID: device1},
 				},
 			},
 			{
-				ID: "folder2",
+				ID:   "folder2",
+				Path: "testdata",
 				Devices: []config.FolderDeviceConfiguration{
 					{DeviceID: device1},
 				},
@@ -671,14 +677,16 @@ func TestIntroducer(t *testing.T) {
 		},
 		Folders: []config.FolderConfiguration{
 			{
-				ID: "folder1",
+				ID:   "folder1",
+				Path: "testdata",
 				Devices: []config.FolderDeviceConfiguration{
 					{DeviceID: device1},
 					{DeviceID: device2, IntroducedBy: device1},
 				},
 			},
 			{
-				ID: "folder2",
+				ID:   "folder2",
+				Path: "testdata",
 				Devices: []config.FolderDeviceConfiguration{
 					{DeviceID: device1},
 				},
@@ -726,14 +734,16 @@ func TestIntroducer(t *testing.T) {
 		},
 		Folders: []config.FolderConfiguration{
 			{
-				ID: "folder1",
+				ID:   "folder1",
+				Path: "testdata",
 				Devices: []config.FolderDeviceConfiguration{
 					{DeviceID: device1},
 					{DeviceID: device2, IntroducedBy: device1},
 				},
 			},
 			{
-				ID: "folder2",
+				ID:   "folder2",
+				Path: "testdata",
 				Devices: []config.FolderDeviceConfiguration{
 					{DeviceID: device1},
 					{DeviceID: device2, IntroducedBy: device1},
@@ -771,14 +781,16 @@ func TestIntroducer(t *testing.T) {
 		},
 		Folders: []config.FolderConfiguration{
 			{
-				ID: "folder1",
+				ID:   "folder1",
+				Path: "testdata",
 				Devices: []config.FolderDeviceConfiguration{
 					{DeviceID: device1},
 					{DeviceID: device2, IntroducedBy: device1},
 				},
 			},
 			{
-				ID: "folder2",
+				ID:   "folder2",
+				Path: "testdata",
 				Devices: []config.FolderDeviceConfiguration{
 					{DeviceID: device1},
 					{DeviceID: device2, IntroducedBy: device1},
@@ -816,14 +828,16 @@ func TestIntroducer(t *testing.T) {
 		},
 		Folders: []config.FolderConfiguration{
 			{
-				ID: "folder1",
+				ID:   "folder1",
+				Path: "testdata",
 				Devices: []config.FolderDeviceConfiguration{
 					{DeviceID: device1},
 					{DeviceID: device2, IntroducedBy: device1},
 				},
 			},
 			{
-				ID: "folder2",
+				ID:   "folder2",
+				Path: "testdata",
 				Devices: []config.FolderDeviceConfiguration{
 					{DeviceID: device1},
 				},
@@ -872,14 +886,16 @@ func TestIntroducer(t *testing.T) {
 		},
 		Folders: []config.FolderConfiguration{
 			{
-				ID: "folder1",
+				ID:   "folder1",
+				Path: "testdata",
 				Devices: []config.FolderDeviceConfiguration{
 					{DeviceID: device1},
 					{DeviceID: device2, IntroducedBy: device1},
 				},
 			},
 			{
-				ID: "folder2",
+				ID:   "folder2",
+				Path: "testdata",
 				Devices: []config.FolderDeviceConfiguration{
 					{DeviceID: device1},
 					{DeviceID: device2},
@@ -916,14 +932,16 @@ func TestIntroducer(t *testing.T) {
 		},
 		Folders: []config.FolderConfiguration{
 			{
-				ID: "folder1",
+				ID:   "folder1",
+				Path: "testdata",
 				Devices: []config.FolderDeviceConfiguration{
 					{DeviceID: device1},
 					{DeviceID: device2, IntroducedBy: device1},
 				},
 			},
 			{
-				ID: "folder2",
+				ID:   "folder2",
+				Path: "testdata",
 				Devices: []config.FolderDeviceConfiguration{
 					{DeviceID: device1},
 					{DeviceID: device2, IntroducedBy: protocol.LocalDeviceID},
@@ -1026,7 +1044,7 @@ func TestIgnores(t *testing.T) {
 	// because we will be changing the files on disk often enough that the
 	// mtimes will be unreliable to determine change status.
 	m.fmut.Lock()
-	m.folderIgnores["default"] = ignore.New(ignore.WithCache(true), ignore.WithChangeDetector(newAlwaysChanged()))
+	m.folderIgnores["default"] = ignore.New(defaultFs, ignore.WithCache(true), ignore.WithChangeDetector(newAlwaysChanged()))
 	m.fmut.Unlock()
 
 	// Make sure the initial scan has finished (ScanFolders is blocking)
@@ -1050,7 +1068,7 @@ func TestIgnores(t *testing.T) {
 	}
 
 	// Invalid path, marker should be missing, hence returns an error.
-	m.AddFolder(config.FolderConfiguration{ID: "fresh", RawPath: "XXX"})
+	m.AddFolder(config.FolderConfiguration{ID: "fresh", Path: "XXX"})
 	_, _, err = m.GetIgnores("fresh")
 	if err == nil {
 		t.Error("No error")
@@ -1069,14 +1087,14 @@ func TestIgnores(t *testing.T) {
 
 func TestROScanRecovery(t *testing.T) {
 	ldb := db.OpenMemory()
-	set := db.NewFileSet("default", ldb)
+	set := db.NewFileSet("default", defaultFs, ldb)
 	set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
 		{Name: "dummyfile"},
 	})
 
 	fcfg := config.FolderConfiguration{
 		ID:              "default",
-		RawPath:         "testdata/rotestfolder",
+		Path:            "testdata/rotestfolder",
 		Type:            config.FolderTypeSendOnly,
 		RescanIntervalS: 1,
 	}
@@ -1089,7 +1107,7 @@ func TestROScanRecovery(t *testing.T) {
 		},
 	})
 
-	os.RemoveAll(fcfg.RawPath)
+	os.RemoveAll(fcfg.Path)
 
 	m := NewModel(cfg, protocol.LocalDeviceID, "syncthing", "dev", ldb, nil)
 	m.AddFolder(fcfg)
@@ -1120,14 +1138,14 @@ func TestROScanRecovery(t *testing.T) {
 		return
 	}
 
-	os.Mkdir(fcfg.RawPath, 0700)
+	os.Mkdir(fcfg.Path, 0700)
 
 	if err := waitFor("folder marker missing"); err != nil {
 		t.Error(err)
 		return
 	}
 
-	fd, err := os.Create(filepath.Join(fcfg.RawPath, ".stfolder"))
+	fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder"))
 	if err != nil {
 		t.Error(err)
 		return
@@ -1139,14 +1157,14 @@ func TestROScanRecovery(t *testing.T) {
 		return
 	}
 
-	os.Remove(filepath.Join(fcfg.RawPath, ".stfolder"))
+	os.Remove(filepath.Join(fcfg.Path, ".stfolder"))
 
 	if err := waitFor("folder marker missing"); err != nil {
 		t.Error(err)
 		return
 	}
 
-	os.Remove(fcfg.RawPath)
+	os.Remove(fcfg.Path)
 
 	if err := waitFor("folder path missing"); err != nil {
 		t.Error(err)
@@ -1156,14 +1174,14 @@ func TestROScanRecovery(t *testing.T) {
 
 func TestRWScanRecovery(t *testing.T) {
 	ldb := db.OpenMemory()
-	set := db.NewFileSet("default", ldb)
+	set := db.NewFileSet("default", defaultFs, ldb)
 	set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
 		{Name: "dummyfile"},
 	})
 
 	fcfg := config.FolderConfiguration{
 		ID:              "default",
-		RawPath:         "testdata/rwtestfolder",
+		Path:            "testdata/rwtestfolder",
 		Type:            config.FolderTypeSendReceive,
 		RescanIntervalS: 1,
 	}
@@ -1176,7 +1194,7 @@ func TestRWScanRecovery(t *testing.T) {
 		},
 	})
 
-	os.RemoveAll(fcfg.RawPath)
+	os.RemoveAll(fcfg.Path)
 
 	m := NewModel(cfg, protocol.LocalDeviceID, "syncthing", "dev", ldb, nil)
 	m.AddFolder(fcfg)
@@ -1207,14 +1225,14 @@ func TestRWScanRecovery(t *testing.T) {
 		return
 	}
 
-	os.Mkdir(fcfg.RawPath, 0700)
+	os.Mkdir(fcfg.Path, 0700)
 
 	if err := waitFor("folder marker missing"); err != nil {
 		t.Error(err)
 		return
 	}
 
-	fd, err := os.Create(filepath.Join(fcfg.RawPath, ".stfolder"))
+	fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder"))
 	if err != nil {
 		t.Error(err)
 		return
@@ -1226,14 +1244,14 @@ func TestRWScanRecovery(t *testing.T) {
 		return
 	}
 
-	os.Remove(filepath.Join(fcfg.RawPath, ".stfolder"))
+	os.Remove(filepath.Join(fcfg.Path, ".stfolder"))
 
 	if err := waitFor("folder marker missing"); err != nil {
 		t.Error(err)
 		return
 	}
 
-	os.Remove(fcfg.RawPath)
+	os.Remove(fcfg.Path)
 
 	if err := waitFor("folder path missing"); err != nil {
 		t.Error(err)
@@ -1861,14 +1879,14 @@ func TestIssue3164(t *testing.T) {
 	f := protocol.FileInfo{
 		Name: "issue3164",
 	}
-	m := ignore.New()
+	m := ignore.New(defaultFs)
 	if err := m.Parse(bytes.NewBufferString("(?d)oktodelete"), ""); err != nil {
 		t.Fatal(err)
 	}
 
 	fl := sendReceiveFolder{
 		dbUpdates: make(chan dbUpdateJob, 1),
-		dir:       "testdata",
+		fs:        fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
 	}
 
 	fl.deleteDir(f, m)
@@ -1955,7 +1973,7 @@ func TestIssue2782(t *testing.T) {
 	if err := os.RemoveAll(testDir); err != nil {
 		t.Skip(err)
 	}
-	if err := osutil.MkdirAll(testDir+"/syncdir", 0755); err != nil {
+	if err := os.MkdirAll(testDir+"/syncdir", 0755); err != nil {
 		t.Skip(err)
 	}
 	if err := ioutil.WriteFile(testDir+"/syncdir/file", []byte("hello, world\n"), 0644); err != nil {
@@ -1968,7 +1986,7 @@ func TestIssue2782(t *testing.T) {
 
 	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
-	m.AddFolder(config.NewFolderConfiguration("default", "~/"+testName+"/synclink/"))
+	m.AddFolder(config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "~/"+testName+"/synclink/"))
 	m.StartFolder("default")
 	m.ServeBackground()
 	defer m.Stop()
@@ -1985,7 +2003,7 @@ func TestIssue2782(t *testing.T) {
 func TestIndexesForUnknownDevicesDropped(t *testing.T) {
 	dbi := db.OpenMemory()
 
-	files := db.NewFileSet("default", dbi)
+	files := db.NewFileSet("default", defaultFs, dbi)
 	files.Replace(device1, genFiles(1))
 	files.Replace(device2, genFiles(1))
 
@@ -1998,7 +2016,7 @@ func TestIndexesForUnknownDevicesDropped(t *testing.T) {
 	m.StartFolder("default")
 
 	// Remote sequence is cached, hence need to recreated.
-	files = db.NewFileSet("default", dbi)
+	files = db.NewFileSet("default", defaultFs, dbi)
 
 	if len(files.ListDevices()) != 1 {
 		t.Error("Expected one device")
@@ -2008,7 +2026,7 @@ func TestIndexesForUnknownDevicesDropped(t *testing.T) {
 func TestSharedWithClearedOnDisconnect(t *testing.T) {
 	dbi := db.OpenMemory()
 
-	fcfg := config.NewFolderConfiguration("default", "testdata")
+	fcfg := config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "testdata")
 	fcfg.Devices = []config.FolderDeviceConfiguration{
 		{DeviceID: device1},
 		{DeviceID: device2},
@@ -2247,7 +2265,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
 
 	dbi := db.OpenMemory()
 
-	fcfg := config.NewFolderConfiguration("default", "testdata")
+	fcfg := config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "testdata")
 	fcfg.Devices = []config.FolderDeviceConfiguration{
 		{DeviceID: device1},
 		{DeviceID: device2},
@@ -2335,151 +2353,6 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
 	}
 }
 
-func TestRootedJoinedPath(t *testing.T) {
-	type testcase struct {
-		root   string
-		rel    string
-		joined string
-		ok     bool
-	}
-	cases := []testcase{
-		// Valid cases
-		{"foo", "bar", "foo/bar", true},
-		{"foo", "/bar", "foo/bar", true},
-		{"foo/", "bar", "foo/bar", true},
-		{"foo/", "/bar", "foo/bar", true},
-		{"baz/foo", "bar", "baz/foo/bar", true},
-		{"baz/foo", "/bar", "baz/foo/bar", true},
-		{"baz/foo/", "bar", "baz/foo/bar", true},
-		{"baz/foo/", "/bar", "baz/foo/bar", true},
-		{"foo", "bar/baz", "foo/bar/baz", true},
-		{"foo", "/bar/baz", "foo/bar/baz", true},
-		{"foo/", "bar/baz", "foo/bar/baz", true},
-		{"foo/", "/bar/baz", "foo/bar/baz", true},
-		{"baz/foo", "bar/baz", "baz/foo/bar/baz", true},
-		{"baz/foo", "/bar/baz", "baz/foo/bar/baz", true},
-		{"baz/foo/", "bar/baz", "baz/foo/bar/baz", true},
-		{"baz/foo/", "/bar/baz", "baz/foo/bar/baz", true},
-
-		// Not escape attempts, but oddly formatted relative paths. Disallowed.
-		{"foo", "./bar", "", false},
-		{"baz/foo", "./bar", "", false},
-		{"foo", "./bar/baz", "", false},
-		{"baz/foo", "./bar/baz", "", false},
-		{"baz/foo", "bar/../baz", "", false},
-		{"baz/foo", "/bar/../baz", "", false},
-		{"baz/foo", "./bar/../baz", "", false},
-		{"baz/foo", "bar/../baz", "", false},
-		{"baz/foo", "/bar/../baz", "", false},
-		{"baz/foo", "./bar/../baz", "", false},
-
-		// Results in an allowed path, but does it by probing. Disallowed.
-		{"foo", "../foo", "", false},
-		{"foo", "../foo/bar", "", false},
-		{"baz/foo", "../foo/bar", "", false},
-		{"baz/foo", "../../baz/foo/bar", "", false},
-		{"baz/foo", "bar/../../foo/bar", "", false},
-		{"baz/foo", "bar/../../../baz/foo/bar", "", false},
-
-		// Escape attempts.
-		{"foo", "", "", false},
-		{"foo", "/", "", false},
-		{"foo", "..", "", false},
-		{"foo", "/..", "", false},
-		{"foo", "../", "", false},
-		{"foo", "../bar", "", false},
-		{"foo", "../foobar", "", false},
-		{"foo/", "../bar", "", false},
-		{"foo/", "../foobar", "", false},
-		{"baz/foo", "../bar", "", false},
-		{"baz/foo", "../foobar", "", false},
-		{"baz/foo/", "../bar", "", false},
-		{"baz/foo/", "../foobar", "", false},
-		{"baz/foo/", "bar/../../quux/baz", "", false},
-
-		// Empty root is a misconfiguration.
-		{"", "/foo", "", false},
-		{"", "foo", "", false},
-		{"", ".", "", false},
-		{"", "..", "", false},
-		{"", "/", "", false},
-		{"", "", "", false},
-
-		// Root=/ is valid, and things should be verified as usual.
-		{"/", "foo", "/foo", true},
-		{"/", "/foo", "/foo", true},
-		{"/", "../foo", "", false},
-		{"/", ".", "", false},
-		{"/", "..", "", false},
-		{"/", "/", "", false},
-		{"/", "", "", false},
-	}
-
-	if runtime.GOOS == "windows" {
-		extraCases := []testcase{
-			{`c:\`, `foo`, `c:\foo`, true},
-			{`\\?\c:\`, `foo`, `\\?\c:\foo`, true},
-			{`c:\`, `\foo`, `c:\foo`, true},
-			{`\\?\c:\`, `\foo`, `\\?\c:\foo`, true},
-
-			{`c:\`, `\\foo`, ``, false},
-			{`c:\`, ``, ``, false},
-			{`c:\`, `.`, ``, false},
-			{`c:\`, `\`, ``, false},
-			{`\\?\c:\`, `\\foo`, ``, false},
-			{`\\?\c:\`, ``, ``, false},
-			{`\\?\c:\`, `.`, ``, false},
-			{`\\?\c:\`, `\`, ``, false},
-
-			// makes no sense, but will be treated simply as a bad filename
-			{`c:\foo`, `d:\bar`, `c:\foo\d:\bar`, true},
-		}
-
-		for _, tc := range cases {
-			// Add case where root is backslashed, rel is forward slashed
-			extraCases = append(extraCases, testcase{
-				root:   filepath.FromSlash(tc.root),
-				rel:    tc.rel,
-				joined: tc.joined,
-				ok:     tc.ok,
-			})
-			// and the opposite
-			extraCases = append(extraCases, testcase{
-				root:   tc.root,
-				rel:    filepath.FromSlash(tc.rel),
-				joined: tc.joined,
-				ok:     tc.ok,
-			})
-			// and both backslashed
-			extraCases = append(extraCases, testcase{
-				root:   filepath.FromSlash(tc.root),
-				rel:    filepath.FromSlash(tc.rel),
-				joined: tc.joined,
-				ok:     tc.ok,
-			})
-		}
-
-		cases = append(cases, extraCases...)
-	}
-
-	for _, tc := range cases {
-		res, err := rootedJoinedPath(tc.root, tc.rel)
-		if tc.ok {
-			if err != nil {
-				t.Errorf("Unexpected error for rootedJoinedPath(%q, %q): %v", tc.root, tc.rel, err)
-				continue
-			}
-			exp := filepath.FromSlash(tc.joined)
-			if res != exp {
-				t.Errorf("Unexpected result for rootedJoinedPath(%q, %q): %q != expected %q", tc.root, tc.rel, res, exp)
-			}
-		} else if err == nil {
-			t.Errorf("Unexpected pass for rootedJoinedPath(%q, %q) => %q", tc.root, tc.rel, res)
-			continue
-		}
-	}
-}
-
 func addFakeConn(m *Model, dev protocol.DeviceID) *fakeConnection {
 	fc := &fakeConnection{id: dev, model: m}
 	m.AddConnection(fc, protocol.HelloResult{})

lib/model/requests_test.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -214,7 +215,7 @@ func TestRequestVersioningSymlinkAttack(t *testing.T) {
 	// deleted symlink to escape
 
 	cfg := defaultConfig.RawCopy()
-	cfg.Folders[0] = config.NewFolderConfiguration("default", "_tmpfolder")
+	cfg.Folders[0] = config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "_tmpfolder")
 	cfg.Folders[0].PullerSleepS = 1
 	cfg.Folders[0].Devices = []config.FolderDeviceConfiguration{
 		{DeviceID: device1},
@@ -287,7 +288,7 @@ func TestRequestVersioningSymlinkAttack(t *testing.T) {
 
 func setupModelWithConnection() (*Model, *fakeConnection) {
 	cfg := defaultConfig.RawCopy()
-	cfg.Folders[0] = config.NewFolderConfiguration("default", "_tmpfolder")
+	cfg.Folders[0] = config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "_tmpfolder")
 	cfg.Folders[0].PullerSleepS = 1
 	cfg.Folders[0].Devices = []config.FolderDeviceConfiguration{
 		{DeviceID: device1},

lib/model/rofolder.go

@@ -24,7 +24,7 @@ type sendOnlyFolder struct {
 	config.FolderConfiguration
 }
 
-func newSendOnlyFolder(model *Model, cfg config.FolderConfiguration, _ versioner.Versioner, _ *fs.MtimeFS) service {
+func newSendOnlyFolder(model *Model, cfg config.FolderConfiguration, _ versioner.Versioner, _ fs.Filesystem) service {
 	ctx, cancel := context.WithCancel(context.Background())
 
 	return &sendOnlyFolder{

lib/model/rwfolder.go

@@ -11,7 +11,6 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
-	"os"
 	"path/filepath"
 	"runtime"
 	"sort"
@@ -51,7 +50,7 @@ type copyBlocksState struct {
 }
 
 // Which filemode bits to preserve
-const retainBits = os.ModeSetgid | os.ModeSetuid | os.ModeSticky
+const retainBits = fs.ModeSetgid | fs.ModeSetuid | fs.ModeSticky
 
 var (
 	activity               = newDeviceActivity()
@@ -84,8 +83,7 @@ type sendReceiveFolder struct {
 	folder
 	config.FolderConfiguration
 
-	mtimeFS   *fs.MtimeFS
+	fs        fs.Filesystem
-	dir       string
 	versioner versioner.Versioner
 	sleep     time.Duration
 	pause     time.Duration
@@ -99,7 +97,7 @@ type sendReceiveFolder struct {
 	errorsMut sync.Mutex
 }
 
-func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, mtimeFS *fs.MtimeFS) service {
+func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service {
 	ctx, cancel := context.WithCancel(context.Background())
 
 	f := &sendReceiveFolder{
@@ -113,8 +111,7 @@ func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver vers
 		},
 		FolderConfiguration: cfg,
 
-		mtimeFS:   mtimeFS,
+		fs:        fs,
-		dir:       cfg.Path(),
 		versioner: ver,
 
 		queue:       newJobQueue(),
@@ -434,7 +431,7 @@ func (f *sendReceiveFolder) pullerIteration(ignores *ignore.Matcher) int {
 	for _, fi := range processDirectly {
 		// Verify that the thing we are handling lives inside a directory,
 		// and not a symlink or empty space.
-		if err := osutil.TraversesSymlink(f.dir, filepath.Dir(fi.Name)); err != nil {
+		if err := osutil.TraversesSymlink(f.fs, filepath.Dir(fi.Name)); err != nil {
 			f.newError(fi.Name, err)
 			continue
 		}
@@ -523,7 +520,7 @@ nextFile:
 
 		// Verify that the thing we are handling lives inside a directory,
 		// and not a symlink or empty space.
-		if err := osutil.TraversesSymlink(f.dir, filepath.Dir(fi.Name)); err != nil {
+		if err := osutil.TraversesSymlink(f.fs, filepath.Dir(fi.Name)); err != nil {
 			f.newError(fi.Name, err)
 			continue
 		}
@@ -610,12 +607,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo) {
 		})
 	}()
 
-	realName, err := rootedJoinedPath(f.dir, file.Name)
+	mode := fs.FileMode(file.Permissions & 0777)
-	if err != nil {
-		f.newError(file.Name, err)
-		return
-	}
-	mode := os.FileMode(file.Permissions & 0777)
 	if f.ignorePermissions(file) {
 		mode = 0777
 	}
@@ -625,13 +617,13 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo) {
 		l.Debugf("need dir\n\t%v\n\t%v", file, curFile)
 	}
 
-	info, err := f.mtimeFS.Lstat(realName)
+	info, err := f.fs.Lstat(file.Name)
 	switch {
 	// There is already something under that name, but it's a file/link.
 	// Most likely a file/link is getting replaced with a directory.
 	// Remove the file/link and fall through to directory creation.
 	case err == nil && (!info.IsDir() || info.IsSymlink()):
-		err = osutil.InWritableDir(os.Remove, realName)
+		err = osutil.InWritableDir(f.fs.Remove, f.fs, file.Name)
 		if err != nil {
 			l.Infof("Puller (folder %q, dir %q): %v", f.folderID, file.Name, err)
 			f.newError(file.Name, err)
@@ -640,28 +632,28 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo) {
 		fallthrough
 	// The directory doesn't exist, so we create it with the right
 	// mode bits from the start.
-	case err != nil && os.IsNotExist(err):
+	case err != nil && fs.IsNotExist(err):
 		// We declare a function that acts on only the path name, so
 		// we can pass it to InWritableDir. We use a regular Mkdir and
 		// not MkdirAll because the parent should already exist.
 		mkdir := func(path string) error {
-			err = os.Mkdir(path, mode)
+			err = f.fs.Mkdir(path, mode)
 			if err != nil || f.ignorePermissions(file) {
 				return err
 			}
 
 			// Stat the directory so we can check its permissions.
-			info, err := f.mtimeFS.Lstat(path)
+			info, err := f.fs.Lstat(path)
 			if err != nil {
 				return err
 			}
 
 			// Mask for the bits we want to preserve and add them in to the
 			// directories permissions.
-			return os.Chmod(path, mode|(os.FileMode(info.Mode())&retainBits))
+			return f.fs.Chmod(path, mode|(info.Mode()&retainBits))
 		}
 
-		if err = osutil.InWritableDir(mkdir, realName); err == nil {
+		if err = osutil.InWritableDir(mkdir, f.fs, file.Name); err == nil {
 			f.dbUpdates <- dbUpdateJob{file, dbUpdateHandleDir}
 		} else {
 			l.Infof("Puller (folder %q, dir %q): %v", f.folderID, file.Name, err)
@@ -681,7 +673,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo) {
 	// It's OK to change mode bits on stuff within non-writable directories.
 	if f.ignorePermissions(file) {
 		f.dbUpdates <- dbUpdateJob{file, dbUpdateHandleDir}
-	} else if err := os.Chmod(realName, mode|(os.FileMode(info.Mode())&retainBits)); err == nil {
+	} else if err := f.fs.Chmod(file.Name, mode|(fs.FileMode(info.Mode())&retainBits)); err == nil {
 		f.dbUpdates <- dbUpdateJob{file, dbUpdateHandleDir}
 	} else {
 		l.Infof("Puller (folder %q, dir %q): %v", f.folderID, file.Name, err)
@@ -712,12 +704,6 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo) {
 		})
 	}()
 
-	realName, err := rootedJoinedPath(f.dir, file.Name)
-	if err != nil {
-		f.newError(file.Name, err)
-		return
-	}
-
 	if shouldDebug() {
 		curFile, _ := f.model.CurrentFolderFile(f.folderID, file.Name)
 		l.Debugf("need symlink\n\t%v\n\t%v", file, curFile)
@@ -732,11 +718,11 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo) {
 		return
 	}
 
-	if _, err = f.mtimeFS.Lstat(realName); err == nil {
+	if _, err = f.fs.Lstat(file.Name); err == nil {
 		// There is already something under that name. Remove it to replace
 		// with the symlink. This also handles the "change symlink type"
 		// path.
-		err = osutil.InWritableDir(os.Remove, realName)
+		err = osutil.InWritableDir(f.fs.Remove, f.fs, file.Name)
 		if err != nil {
 			l.Infof("Puller (folder %q, dir %q): %v", f.folderID, file.Name, err)
 			f.newError(file.Name, err)
@@ -747,10 +733,10 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo) {
 	// We declare a function that acts on only the path name, so
 	// we can pass it to InWritableDir.
 	createLink := func(path string) error {
-		return os.Symlink(file.SymlinkTarget, path)
+		return f.fs.CreateSymlink(file.SymlinkTarget, path)
 	}
 
-	if err = osutil.InWritableDir(createLink, realName); err == nil {
+	if err = osutil.InWritableDir(createLink, f.fs, file.Name); err == nil {
 		f.dbUpdates <- dbUpdateJob{file, dbUpdateHandleSymlink}
 	} else {
 		l.Infof("Puller (folder %q, dir %q): %v", f.folderID, file.Name, err)
@@ -781,31 +767,21 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, matcher *ignore.Ma
 		})
 	}()
 
-	realName, err := rootedJoinedPath(f.dir, file.Name)
-	if err != nil {
-		f.newError(file.Name, err)
-		return
-	}
-
 	// Delete any temporary files lying around in the directory
-	dir, _ := os.Open(realName)
+
-	if dir != nil {
+	files, _ := f.fs.DirNames(file.Name)
-		files, _ := dir.Readdirnames(-1)
+	for _, dirFile := range files {
-		for _, dirFile := range files {
+		fullDirFile := filepath.Join(file.Name, dirFile)
-			fullDirFile := filepath.Join(file.Name, dirFile)
+		if ignore.IsTemporary(dirFile) || (matcher != nil && matcher.Match(fullDirFile).IsDeletable()) {
-			if ignore.IsTemporary(dirFile) || (matcher != nil &&
+			f.fs.RemoveAll(fullDirFile)
-				matcher.Match(fullDirFile).IsDeletable()) {
-				os.RemoveAll(filepath.Join(f.dir, fullDirFile))
-			}
 		}
-		dir.Close()
 	}
 
-	err = osutil.InWritableDir(os.Remove, realName)
+	err = osutil.InWritableDir(f.fs.Remove, f.fs, file.Name)
-	if err == nil || os.IsNotExist(err) {
+	if err == nil || fs.IsNotExist(err) {
 		// It was removed or it doesn't exist to start with
 		f.dbUpdates <- dbUpdateJob{file, dbUpdateDeleteDir}
-	} else if _, serr := f.mtimeFS.Lstat(realName); serr != nil && !os.IsPermission(serr) {
+	} else if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) {
 		// We get an error just looking at the directory, and it's not a
 		// permission problem. Lets assume the error is in fact some variant
 		// of "file does not exist" (possibly expressed as some parent being a
@@ -840,12 +816,6 @@ func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo) {
 		})
 	}()
 
-	realName, err := rootedJoinedPath(f.dir, file.Name)
-	if err != nil {
-		f.newError(file.Name, err)
-		return
-	}
-
 	cur, ok := f.model.CurrentFolderFile(f.folderID, file.Name)
 	if ok && f.inConflict(cur.Version, file.Version) {
 		// There is a conflict here. Move the file to a conflict copy instead
@@ -854,17 +824,17 @@ func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo) {
 		file.Version = file.Version.Merge(cur.Version)
 		err = osutil.InWritableDir(func(name string) error {
 			return f.moveForConflict(name, file.ModifiedBy.String())
-		}, realName)
+		}, f.fs, file.Name)
 	} else if f.versioner != nil && !cur.IsSymlink() {
-		err = osutil.InWritableDir(f.versioner.Archive, realName)
+		err = osutil.InWritableDir(f.versioner.Archive, f.fs, file.Name)
 	} else {
-		err = osutil.InWritableDir(os.Remove, realName)
+		err = osutil.InWritableDir(f.fs.Remove, f.fs, file.Name)
 	}
 
-	if err == nil || os.IsNotExist(err) {
+	if err == nil || fs.IsNotExist(err) {
 		// It was removed or it doesn't exist to start with
 		f.dbUpdates <- dbUpdateJob{file, dbUpdateDeleteFile}
-	} else if _, serr := f.mtimeFS.Lstat(realName); serr != nil && !os.IsPermission(serr) {
+	} else if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) {
 		// We get an error just looking at the file, and it's not a permission
 		// problem. Lets assume the error is in fact some variant of "file
 		// does not exist" (possibly expressed as some parent being a file and
@@ -915,24 +885,13 @@ func (f *sendReceiveFolder) renameFile(source, target protocol.FileInfo) {
 
 	l.Debugln(f, "taking rename shortcut", source.Name, "->", target.Name)
 
-	from, err := rootedJoinedPath(f.dir, source.Name)
-	if err != nil {
-		f.newError(source.Name, err)
-		return
-	}
-	to, err := rootedJoinedPath(f.dir, target.Name)
-	if err != nil {
-		f.newError(target.Name, err)
-		return
-	}
-
 	if f.versioner != nil {
-		err = osutil.Copy(from, to)
+		err = osutil.Copy(f.fs, source.Name, target.Name)
 		if err == nil {
-			err = osutil.InWritableDir(f.versioner.Archive, from)
+			err = osutil.InWritableDir(f.versioner.Archive, f.fs, source.Name)
 		}
 	} else {
-		err = osutil.TryRename(from, to)
+		err = osutil.TryRename(f.fs, source.Name, target.Name)
 	}
 
 	if err == nil {
@@ -955,7 +914,7 @@ func (f *sendReceiveFolder) renameFile(source, target protocol.FileInfo) {
 		// get rid of. Attempt to delete it instead so that we make *some*
 		// progress. The target is unhandled.
 
-		err = osutil.InWritableDir(os.Remove, from)
+		err = osutil.InWritableDir(f.fs.Remove, f.fs, source.Name)
 		if err != nil {
 			l.Infof("Puller (folder %q, file %q): delete %q after failed rename: %v", f.folderID, target.Name, source.Name, err)
 			f.newError(target.Name, err)
@@ -1041,26 +1000,16 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
 		return
 	}
 
-	// Figure out the absolute filenames we need once and for all
+	tempName := ignore.TempName(file.Name)
-	tempName, err := rootedJoinedPath(f.dir, ignore.TempName(file.Name))
-	if err != nil {
-		f.newError(file.Name, err)
-		return
-	}
-	realName, err := rootedJoinedPath(f.dir, file.Name)
-	if err != nil {
-		f.newError(file.Name, err)
-		return
-	}
 
 	if hasCurFile && !curFile.IsDirectory() && !curFile.IsSymlink() {
 		// Check that the file on disk is what we expect it to be according to
 		// the database. If there's a mismatch here, there might be local
 		// changes that we don't know about yet and we should scan before
 		// touching the file. If we can't stat the file we'll just pull it.
-		if info, err := f.mtimeFS.Lstat(realName); err == nil {
+		if info, err := f.fs.Lstat(file.Name); err == nil {
 			if !info.ModTime().Equal(curFile.ModTime()) || info.Size() != curFile.Size {
-				l.Debugln("file modified but not rescanned; not pulling:", realName)
+				l.Debugln("file modified but not rescanned; not pulling:", file.Name)
 				// Scan() is synchronous (i.e. blocks until the scan is
 				// completed and returns an error), but a scan can't happen
 				// while we're in the puller routine. Request the scan in the
@@ -1082,7 +1031,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
 
 	// Check for an old temporary file which might have some blocks we could
 	// reuse.
-	tempBlocks, err := scanner.HashFile(f.ctx, fs.DefaultFilesystem, tempName, protocol.BlockSize, nil, false)
+	tempBlocks, err := scanner.HashFile(f.ctx, f.fs, tempName, protocol.BlockSize, nil, false)
 	if err == nil {
 		// Check for any reusable blocks in the temp file
 		tempCopyBlocks, _ := scanner.BlockDiff(tempBlocks, file.Blocks)
@@ -1110,7 +1059,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
 			// Otherwise, discard the file ourselves in order for the
 			// sharedpuller not to panic when it fails to exclusively create a
 			// file which already exists
-			osutil.InWritableDir(os.Remove, tempName)
+			osutil.InWritableDir(f.fs.Remove, f.fs, tempName)
 		}
 	} else {
 		// Copy the blocks, as we don't want to shuffle them on the FileInfo
@@ -1119,8 +1068,8 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
 	}
 
 	if f.MinDiskFree.BaseValue() > 0 {
-		if free, err := osutil.DiskFreeBytes(f.dir); err == nil && free < blocksSize {
+		if usage, err := f.fs.Usage("."); err == nil && usage.Free < blocksSize {
-			l.Warnf(`Folder "%s": insufficient disk space in %s for %s: have %.2f MiB, need %.2f MiB`, f.folderID, f.dir, file.Name, float64(free)/1024/1024, float64(blocksSize)/1024/1024)
+			l.Warnf(`Folder "%s": insufficient disk space in %s for %s: have %.2f MiB, need %.2f MiB`, f.folderID, f.fs.URI(), file.Name, float64(usage.Free)/1024/1024, float64(blocksSize)/1024/1024)
 			f.newError(file.Name, errors.New("insufficient space"))
 			return
 		}
@@ -1141,9 +1090,10 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
 
 	s := sharedPullerState{
 		file:             file,
+		fs:               f.fs,
 		folder:           f.folderID,
 		tempName:         tempName,
-		realName:         realName,
+		realName:         file.Name,
 		copyTotal:        len(blocks),
 		copyNeeded:       len(blocks),
 		reused:           len(reused),
@@ -1170,20 +1120,15 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
 // shortcutFile sets file mode and modification time, when that's the only
 // thing that has changed.
 func (f *sendReceiveFolder) shortcutFile(file protocol.FileInfo) error {
-	realName, err := rootedJoinedPath(f.dir, file.Name)
-	if err != nil {
-		f.newError(file.Name, err)
-		return err
-	}
 	if !f.ignorePermissions(file) {
-		if err := os.Chmod(realName, os.FileMode(file.Permissions&0777)); err != nil {
+		if err := f.fs.Chmod(file.Name, fs.FileMode(file.Permissions&0777)); err != nil {
 			l.Infof("Puller (folder %q, file %q): shortcut: chmod: %v", f.folderID, file.Name, err)
 			f.newError(file.Name, err)
 			return err
 		}
 	}
 
-	f.mtimeFS.Chtimes(realName, file.ModTime(), file.ModTime()) // never fails
+	f.fs.Chtimes(file.Name, file.ModTime(), file.ModTime()) // never fails
 
 	// This may have been a conflict. We should merge the version vectors so
 	// that our clock doesn't move backwards.
@@ -1211,15 +1156,16 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 			f.model.progressEmitter.Register(state.sharedPullerState)
 		}
 
-		folderRoots := make(map[string]string)
+		folderFilesystems := make(map[string]fs.Filesystem)
 		var folders []string
 		f.model.fmut.RLock()
 		for folder, cfg := range f.model.folderCfgs {
-			folderRoots[folder] = cfg.Path()
+			folderFilesystems[folder] = cfg.Filesystem()
 			folders = append(folders, folder)
 		}
 		f.model.fmut.RUnlock()
 
+		var file fs.File
 		var weakHashFinder *weakhash.Finder
 
 		if weakhash.Enabled {
@@ -1237,9 +1183,12 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 				}
 
 				if len(hashesToFind) > 0 {
-					weakHashFinder, err = weakhash.NewFinder(state.realName, protocol.BlockSize, hashesToFind)
+					file, err = f.fs.Open(state.file.Name)
-					if err != nil {
+					if err == nil {
-						l.Debugln("weak hasher", err)
+						weakHashFinder, err = weakhash.NewFinder(file, protocol.BlockSize, hashesToFind)
+						if err != nil {
+							l.Debugln("weak hasher", err)
+						}
 					}
 				} else {
 					l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name)
@@ -1289,12 +1238,9 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 			}
 
 			if !found {
-				found = f.model.finder.Iterate(folders, block.Hash, func(folder, file string, index int32) bool {
+				found = f.model.finder.Iterate(folders, block.Hash, func(folder, path string, index int32) bool {
-					inFile, err := rootedJoinedPath(folderRoots[folder], file)
+					fs := folderFilesystems[folder]
-					if err != nil {
+					fd, err := fs.Open(path)
-						return false
-					}
-					fd, err := os.Open(inFile)
 					if err != nil {
 						return false
 					}
@@ -1308,8 +1254,8 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 					hash, err := scanner.VerifyBuffer(buf, block)
 					if err != nil {
 						if hash != nil {
-							l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, file, index, block.Hash, hash)
+							l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, path, index, block.Hash, hash)
-							err = f.model.finder.Fix(folder, file, index, block.Hash, hash)
+							err = f.model.finder.Fix(folder, path, index, block.Hash, hash)
 							if err != nil {
 								l.Warnln("finder fix:", err)
 							}
@@ -1323,7 +1269,7 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 					if err != nil {
 						state.fail("dst write", err)
 					}
-					if file == state.file.Name {
+					if path == state.file.Name {
 						state.copiedFromOrigin()
 					}
 					return true
@@ -1345,7 +1291,12 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 				state.copyDone(block)
 			}
 		}
-		weakHashFinder.Close()
+		if file != nil {
+			// os.File used to return invalid argument if nil.
+			// fs.File panics as it's an interface.
+			file.Close()
+		}
+
 		out <- state.sharedPullerState
 	}
 }
@@ -1426,12 +1377,12 @@ func (f *sendReceiveFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *
 func (f *sendReceiveFolder) performFinish(state *sharedPullerState) error {
 	// Set the correct permission bits on the new file
 	if !f.ignorePermissions(state.file) {
-		if err := os.Chmod(state.tempName, os.FileMode(state.file.Permissions&0777)); err != nil {
+		if err := f.fs.Chmod(state.tempName, fs.FileMode(state.file.Permissions&0777)); err != nil {
 			return err
 		}
 	}
 
-	if stat, err := f.mtimeFS.Lstat(state.realName); err == nil {
+	if stat, err := f.fs.Lstat(state.file.Name); err == nil {
 		// There is an old file or directory already in place. We need to
 		// handle that.
 
@@ -1445,7 +1396,7 @@ func (f *sendReceiveFolder) performFinish(state *sharedPullerState) error {
 			// and future hard ignores before attempting a directory delete.
 			// Should share code with f.deletDir().
 
-			if err = osutil.InWritableDir(os.Remove, state.realName); err != nil {
+			if err = osutil.InWritableDir(f.fs.Remove, f.fs, state.file.Name); err != nil {
 				return err
 			}
 
@@ -1458,7 +1409,7 @@ func (f *sendReceiveFolder) performFinish(state *sharedPullerState) error {
 			state.file.Version = state.file.Version.Merge(state.version)
 			err = osutil.InWritableDir(func(name string) error {
 				return f.moveForConflict(name, state.file.ModifiedBy.String())
-			}, state.realName)
+			}, f.fs, state.file.Name)
 			if err != nil {
 				return err
 			}
@@ -1468,7 +1419,7 @@ func (f *sendReceiveFolder) performFinish(state *sharedPullerState) error {
 			// file before we replace it. Archiving a non-existent file is not
 			// an error.
 
-			if err = f.versioner.Archive(state.realName); err != nil {
+			if err = f.versioner.Archive(state.file.Name); err != nil {
 				return err
 			}
 		}
@@ -1476,12 +1427,12 @@ func (f *sendReceiveFolder) performFinish(state *sharedPullerState) error {
 
 	// Replace the original content with the new one. If it didn't work,
 	// leave the temp file in place for reuse.
-	if err := osutil.TryRename(state.tempName, state.realName); err != nil {
+	if err := osutil.TryRename(f.fs, state.tempName, state.file.Name); err != nil {
 		return err
 	}
 
 	// Set the correct timestamp on the new file
-	f.mtimeFS.Chtimes(state.realName, state.file.ModTime(), state.file.ModTime()) // never fails
+	f.fs.Chtimes(state.file.Name, state.file.ModTime(), state.file.ModTime()) // never fails
 
 	// Record the updated file in the index
 	f.dbUpdates <- dbUpdateJob{state.file, dbUpdateHandleFile}
@@ -1540,26 +1491,7 @@ func (f *sendReceiveFolder) dbUpdaterRoutine() {
 	tick := time.NewTicker(maxBatchTime)
 	defer tick.Stop()
 
-	var changedFiles []string
+	changedDirs := make(map[string]struct{})
-	var changedDirs []string
-	if f.Fsync {
-		changedFiles = make([]string, 0, maxBatchSize)
-		changedDirs = make([]string, 0, maxBatchSize)
-	}
-
-	syncFilesOnce := func(files []string, syncFn func(string) error) {
-		sort.Strings(files)
-		var lastFile string
-		for _, file := range files {
-			if lastFile == file {
-				continue
-			}
-			lastFile = file
-			if err := syncFn(file); err != nil {
-				l.Infof("fsync %q failed: %v", file, err)
-			}
-		}
-	}
 
 	handleBatch := func() {
 		found := false
@@ -1567,20 +1499,16 @@ func (f *sendReceiveFolder) dbUpdaterRoutine() {
 
 		for _, job := range batch {
 			files = append(files, job.file)
-			if f.Fsync {
+
-				// collect changed files and dirs
+			switch job.jobType {
-				switch job.jobType {
+			case dbUpdateHandleFile, dbUpdateShortcutFile:
-				case dbUpdateHandleFile, dbUpdateShortcutFile:
+				changedDirs[filepath.Dir(job.file.Name)] = struct{}{}
-					changedFiles = append(changedFiles, filepath.Join(f.dir, job.file.Name))
+			case dbUpdateHandleDir:
-				case dbUpdateHandleDir:
+				changedDirs[job.file.Name] = struct{}{}
-					changedDirs = append(changedDirs, filepath.Join(f.dir, job.file.Name))
+			case dbUpdateHandleSymlink:
-				case dbUpdateHandleSymlink:
+				// fsyncing symlinks is only supported by MacOS, ignore
-					// fsyncing symlinks is only supported by MacOS, ignore
-				}
-				if job.jobType != dbUpdateShortcutFile {
-					changedDirs = append(changedDirs, filepath.Dir(filepath.Join(f.dir, job.file.Name)))
-				}
 			}
+
 			if job.file.IsInvalid() || (job.file.IsDirectory() && !job.file.IsSymlink()) {
 				continue
 			}
@@ -1593,12 +1521,18 @@ func (f *sendReceiveFolder) dbUpdaterRoutine() {
 			lastFile = job.file
 		}
 
-		if f.Fsync {
+		// sync directories
-			// sync files and dirs to disk
+		for dir := range changedDirs {
-			syncFilesOnce(changedFiles, osutil.SyncFile)
+			delete(changedDirs, dir)
-			changedFiles = changedFiles[:0]
+			fd, err := f.fs.Open(dir)
-			syncFilesOnce(changedDirs, osutil.SyncDir)
+			if err != nil {
-			changedDirs = changedDirs[:0]
+				l.Infof("fsync %q failed: %v", dir, err)
+				continue
+			}
+			if err := fd.Sync(); err != nil {
+				l.Infof("fsync %q failed: %v", dir, err)
+			}
+			fd.Close()
 		}
 
 		// All updates to file/folder objects that originated remotely
@@ -1669,14 +1603,14 @@ func removeAvailability(availabilities []Availability, availability Availability
 func (f *sendReceiveFolder) moveForConflict(name string, lastModBy string) error {
 	if strings.Contains(filepath.Base(name), ".sync-conflict-") {
 		l.Infoln("Conflict for", name, "which is already a conflict copy; not copying again.")
-		if err := os.Remove(name); err != nil && !os.IsNotExist(err) {
+		if err := f.fs.Remove(name); err != nil && !fs.IsNotExist(err) {
 			return err
 		}
 		return nil
 	}
 
 	if f.MaxConflicts == 0 {
-		if err := os.Remove(name); err != nil && !os.IsNotExist(err) {
+		if err := f.fs.Remove(name); err != nil && !fs.IsNotExist(err) {
 			return err
 		}
 		return nil
@@ -1685,8 +1619,8 @@ func (f *sendReceiveFolder) moveForConflict(name string, lastModBy string) error
 	ext := filepath.Ext(name)
 	withoutExt := name[:len(name)-len(ext)]
 	newName := withoutExt + time.Now().Format(".sync-conflict-20060102-150405-") + lastModBy + ext
-	err := os.Rename(name, newName)
+	err := f.fs.Rename(name, newName)
-	if os.IsNotExist(err) {
+	if fs.IsNotExist(err) {
 		// We were supposed to move a file away but it does not exist. Either
 		// the user has already moved it away, or the conflict was between a
 		// remote modification and a local delete. In either way it does not
@@ -1694,11 +1628,11 @@ func (f *sendReceiveFolder) moveForConflict(name string, lastModBy string) error
 		err = nil
 	}
 	if f.MaxConflicts > -1 {
-		matches, gerr := osutil.Glob(withoutExt + ".sync-conflict-????????-??????*" + ext)
+		matches, gerr := f.fs.Glob(withoutExt + ".sync-conflict-????????-??????*" + ext)
 		if gerr == nil && len(matches) > f.MaxConflicts {
 			sort.Sort(sort.Reverse(sort.StringSlice(matches)))
 			for _, match := range matches[f.MaxConflicts:] {
-				gerr = os.Remove(match)
+				gerr = f.fs.Remove(match)
 				if gerr != nil {
 					l.Debugln(f, "removing extra conflict", gerr)
 				}
@@ -1772,7 +1706,7 @@ func fileValid(file db.FileIntf) error {
 		return errSymlinksUnsupported
 
 	case runtime.GOOS == "windows" && windowsInvalidFilename(file.FileName()):
-		return errInvalidFilename
+		return fs.ErrInvalidFilename
 	}
 
 	return nil
@@ -1821,7 +1755,7 @@ func (l byComponentCount) Swap(a, b int) {
 func componentCount(name string) int {
 	count := 0
 	for _, codepoint := range name {
-		if codepoint == os.PathSeparator {
+		if codepoint == fs.PathSeparator {
 			count++
 		}
 	}

lib/model/rwfolder_test.go

@@ -87,8 +87,7 @@ func setUpSendReceiveFolder(model *Model) *sendReceiveFolder {
 			ctx:                 context.TODO(),
 		},
 
-		mtimeFS:   fs.NewMtimeFS(fs.DefaultFilesystem, db.NewNamespacedKV(model.db, "mtime")),
+		fs:        fs.NewMtimeFS(fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), db.NewNamespacedKV(model.db, "mtime")),
-		dir:       "testdata",
 		queue:     newJobQueue(),
 		errors:    make(map[string]string),
 		errorsMut: sync.NewMutex(),
@@ -246,7 +245,7 @@ func TestCopierFinder(t *testing.T) {
 	}
 
 	// Verify that the fetched blocks have actually been written to the temp file
-	blks, err := scanner.HashFile(context.TODO(), fs.DefaultFilesystem, tempFile, protocol.BlockSize, nil, false)
+	blks, err := scanner.HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, "."), tempFile, protocol.BlockSize, nil, false)
 	if err != nil {
 		t.Log(err)
 	}

lib/model/sharedpullerstate.go

@@ -8,10 +8,10 @@ package model
 
 import (
 	"io"
-	"os"
 	"path/filepath"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
 )
@@ -21,6 +21,7 @@ import (
 type sharedPullerState struct {
 	// Immutable, does not require locking
 	file        protocol.FileInfo // The new file (desired end state)
+	fs          fs.Filesystem
 	folder      string
 	tempName    string
 	realName    string
@@ -32,7 +33,7 @@ type sharedPullerState struct {
 
 	// Mutable, must be locked for access
 	err               error        // The first error we hit
-	fd                *os.File     // The fd of the temp file
+	fd                fs.File      // The fd of the temp file
 	copyTotal         int          // Total number of copy actions for the whole job
 	pullTotal         int          // Total number of pull actions for the whole job
 	copyOrigin        int          // Number of blocks copied from the original file
@@ -92,8 +93,8 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
 	// osutil.InWritableDir except we need to do more stuff so we duplicate it
 	// here.
 	dir := filepath.Dir(s.tempName)
-	if info, err := os.Stat(dir); err != nil {
+	if info, err := s.fs.Stat(dir); err != nil {
-		if os.IsNotExist(err) {
+		if fs.IsNotExist(err) {
 			// XXX: This works around a bug elsewhere, a race condition when
 			// things are deleted while being synced. However that happens, we
 			// end up with a directory for "foo" with the delete bit, but a
@@ -103,7 +104,7 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
 			// next scan it'll be found and the delete bit on it is removed.
 			// The user can then clean up as they like...
 			l.Infoln("Resurrecting directory", dir)
-			if err := os.MkdirAll(dir, 0755); err != nil {
+			if err := s.fs.MkdirAll(dir, 0755); err != nil {
 				s.failLocked("resurrect dir", err)
 				return nil, err
 			}
@@ -112,10 +113,10 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
 			return nil, err
 		}
 	} else if info.Mode()&0200 == 0 {
-		err := os.Chmod(dir, 0755)
+		err := s.fs.Chmod(dir, 0755)
 		if !s.ignorePerms && err == nil {
 			defer func() {
-				err := os.Chmod(dir, info.Mode().Perm())
+				err := s.fs.Chmod(dir, info.Mode()&fs.ModePerm)
 				if err != nil {
 					panic(err)
 				}
@@ -128,7 +129,7 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
 	// permissions will be set to the final value later, but in the meantime
 	// we don't want to have a temporary file with looser permissions than
 	// the final outcome.
-	mode := os.FileMode(s.file.Permissions) | 0600
+	mode := fs.FileMode(s.file.Permissions) | 0600
 	if s.ignorePerms {
 		// When ignorePerms is set we use a very permissive mode and let the
 		// system umask filter it.
@@ -137,9 +138,9 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
 
 	// Attempt to create the temp file
 	// RDWR because of issue #2994.
-	flags := os.O_RDWR
+	flags := fs.OptReadWrite
 	if s.reused == 0 {
-		flags |= os.O_CREATE | os.O_EXCL
+		flags |= fs.OptCreate | fs.OptExclusive
 	} else if !s.ignorePerms {
 		// With sufficiently bad luck when exiting or crashing, we may have
 		// had time to chmod the temp file to read only state but not yet
@@ -151,12 +152,12 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
 		// already and make no modification, as we would otherwise override
 		// what the umask dictates.
 
-		if err := os.Chmod(s.tempName, mode); err != nil {
+		if err := s.fs.Chmod(s.tempName, mode); err != nil {
 			s.failLocked("dst create chmod", err)
 			return nil, err
 		}
 	}
-	fd, err := os.OpenFile(s.tempName, flags, mode)
+	fd, err := s.fs.OpenFile(s.tempName, flags, mode)
 	if err != nil {
 		s.failLocked("dst create", err)
 		return nil, err
@@ -180,7 +181,7 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
 }
 
 // sourceFile opens the existing source file for reading
-func (s *sharedPullerState) sourceFile() (*os.File, error) {
+func (s *sharedPullerState) sourceFile() (fs.File, error) {
 	s.mut.Lock()
 	defer s.mut.Unlock()
 
@@ -190,7 +191,7 @@ func (s *sharedPullerState) sourceFile() (*os.File, error) {
 	}
 
 	// Attempt to open the existing file
-	fd, err := os.Open(s.realName)
+	fd, err := s.fs.Open(s.realName)
 	if err != nil {
 		s.failLocked("src open", err)
 		return nil, err
@@ -292,9 +293,12 @@ func (s *sharedPullerState) finalClose() (bool, error) {
 	}
 
 	if s.fd != nil {
+		// This is our error if we weren't errored before. Otherwise we
+		// keep the earlier error.
+		if fsyncErr := s.fd.Sync(); fsyncErr != nil && s.err == nil {
+			s.err = fsyncErr
+		}
 		if closeErr := s.fd.Close(); closeErr != nil && s.err == nil {
-			// This is our error if we weren't errored before. Otherwise we
-			// keep the earlier error.
 			s.err = closeErr
 		}
 		s.fd = nil

lib/model/sharedpullerstate_test.go

@@ -10,12 +10,14 @@ import (
 	"os"
 	"testing"
 
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
 func TestSourceFileOK(t *testing.T) {
 	s := sharedPullerState{
-		realName: "testdata/foo",
+		fs:       fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
+		realName: "foo",
 		mut:      sync.NewRWMutex(),
 	}
 
@@ -47,6 +49,7 @@ func TestSourceFileOK(t *testing.T) {
 
 func TestSourceFileBad(t *testing.T) {
 	s := sharedPullerState{
+		fs:       fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
 		realName: "nonexistent",
 		mut:      sync.NewRWMutex(),
 	}
@@ -73,7 +76,8 @@ func TestReadOnlyDir(t *testing.T) {
 	}()
 
 	s := sharedPullerState{
-		tempName: "testdata/read_only_dir/.temp_name",
+		fs:       fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
+		tempName: "read_only_dir/.temp_name",
 		mut:      sync.NewRWMutex(),
 	}
 

lib/osutil/atomic.go

@@ -8,10 +8,10 @@ package osutil
 
 import (
 	"errors"
-	"io/ioutil"
-	"os"
 	"path/filepath"
 	"runtime"
+
+	"github.com/syncthing/syncthing/lib/fs"
 )
 
 var (
@@ -25,7 +25,8 @@ var (
 // returned on Close, so a lazy user can ignore errors until Close.
 type AtomicWriter struct {
 	path string
-	next *os.File
+	next fs.File
+	fs   fs.Filesystem
 	err  error
 }
 
@@ -33,11 +34,19 @@ type AtomicWriter struct {
 // instead of the given name. The file is created with secure (0600)
 // permissions.
 func CreateAtomic(path string) (*AtomicWriter, error) {
+	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, filepath.Dir(path))
+	return CreateAtomicFilesystem(fs, filepath.Base(path))
+}
+
+// CreateAtomicFilesystem is like os.Create, except a temporary file name is used
+// instead of the given name. The file is created with secure (0600)
+// permissions.
+func CreateAtomicFilesystem(filesystem fs.Filesystem, path string) (*AtomicWriter, error) {
 	// The security of this depends on the tempfile having secure
 	// permissions, 0600, from the beginning. This is what ioutil.TempFile
 	// does. We have a test that verifies that that is the case, should this
 	// ever change in the standard library in the future.
-	fd, err := ioutil.TempFile(filepath.Dir(path), TempPrefix)
+	fd, err := TempFile(filesystem, filepath.Dir(path), TempPrefix)
 	if err != nil {
 		return nil, err
 	}
@@ -45,6 +54,7 @@ func CreateAtomic(path string) (*AtomicWriter, error) {
 	w := &AtomicWriter{
 		path: path,
 		next: fd,
+		fs:   filesystem,
 	}
 
 	return w, nil
@@ -71,7 +81,7 @@ func (w *AtomicWriter) Close() error {
 	}
 
 	// Try to not leave temp file around, but ignore error.
-	defer os.Remove(w.next.Name())
+	defer w.fs.Remove(w.next.Name())
 
 	if err := w.next.Sync(); err != nil {
 		w.err = err
@@ -88,17 +98,21 @@ func (w *AtomicWriter) Close() error {
 	// either. Return this error because it may be more informative. On non-
 	// Windows we want the atomic rename behavior so we don't attempt remove.
 	if runtime.GOOS == "windows" {
-		if err := os.Remove(w.path); err != nil && !os.IsNotExist(err) {
+		if err := w.fs.Remove(w.path); err != nil && !fs.IsNotExist(err) {
 			return err
 		}
 	}
 
-	if err := os.Rename(w.next.Name(), w.path); err != nil {
+	if err := w.fs.Rename(w.next.Name(), w.path); err != nil {
 		w.err = err
 		return err
 	}
 
-	SyncDir(filepath.Dir(w.next.Name()))
+	// fsync the directory too
+	if fd, err := w.fs.Open(filepath.Dir(w.next.Name())); err == nil {
+		fd.Sync()
+		fd.Close()
+	}
 
 	// Set w.err to return appropriately for any future operations.
 	w.err = ErrClosed

lib/osutil/fsroots_unix.go

@@ -1,13 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build !windows
-
-package osutil
-
-func GetFilesystemRoots() ([]string, error) {
-	return []string{"/"}, nil
-}

lib/osutil/fsroots_windows.go

@@ -1,46 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build windows
-
-package osutil
-
-import (
-	"bytes"
-	"fmt"
-	"syscall"
-	"unsafe"
-)
-
-func GetFilesystemRoots() ([]string, error) {
-	kernel32, err := syscall.LoadDLL("kernel32.dll")
-	if err != nil {
-		return nil, err
-	}
-	getLogicalDriveStringsHandle, err := kernel32.FindProc("GetLogicalDriveStringsA")
-	if err != nil {
-		return nil, err
-	}
-
-	buffer := [1024]byte{}
-	bufferSize := uint32(len(buffer))
-
-	hr, _, _ := getLogicalDriveStringsHandle.Call(uintptr(unsafe.Pointer(&bufferSize)), uintptr(unsafe.Pointer(&buffer)))
-	if hr == 0 {
-		return nil, fmt.Errorf("Syscall failed")
-	}
-
-	var drives []string
-	parts := bytes.Split(buffer[:], []byte{0})
-	for _, part := range parts {
-		if len(part) == 0 {
-			break
-		}
-		drives = append(drives, string(part))
-	}
-
-	return drives, nil
-}

lib/osutil/glob_unix.go

@@ -1,17 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build !windows
-
-package osutil
-
-import (
-	"path/filepath"
-)
-
-func Glob(pattern string) (matches []string, err error) {
-	return filepath.Glob(pattern)
-}

lib/osutil/glob_windows.go

@@ -1,96 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build windows
-
-package osutil
-
-import (
-	"os"
-	"path/filepath"
-	"runtime"
-	"sort"
-	"strings"
-)
-
-// Glob implements filepath.Glob, but works with Windows long path prefixes.
-// Deals with https://github.com/golang/go/issues/10577
-func Glob(pattern string) (matches []string, err error) {
-	if !hasMeta(pattern) {
-		if _, err = os.Lstat(pattern); err != nil {
-			return nil, nil
-		}
-		return []string{pattern}, nil
-	}
-
-	dir, file := filepath.Split(filepath.Clean(pattern))
-	switch dir {
-	case "":
-		dir = "."
-	case string(filepath.Separator):
-		// nothing
-	default:
-		if runtime.GOOS != "windows" || len(dir) < 2 || dir[len(dir)-2] != ':' {
-			dir = dir[0 : len(dir)-1] // chop off trailing separator, if it's not after the drive letter
-		}
-	}
-
-	if !hasMeta(dir) {
-		return glob(dir, file, nil)
-	}
-
-	var m []string
-	m, err = Glob(dir)
-	if err != nil {
-		return
-	}
-	for _, d := range m {
-		matches, err = glob(d, file, matches)
-		if err != nil {
-			return
-		}
-	}
-	return
-}
-
-func hasMeta(path string) bool {
-	// Strip off Windows long path prefix if it exists.
-	if strings.HasPrefix(path, "\\\\?\\") {
-		path = path[4:]
-	}
-	// TODO(niemeyer): Should other magic characters be added here?
-	return strings.IndexAny(path, "*?[") >= 0
-}
-
-func glob(dir, pattern string, matches []string) (m []string, e error) {
-	m = matches
-	fi, err := os.Stat(dir)
-	if err != nil {
-		return
-	}
-	if !fi.IsDir() {
-		return
-	}
-	d, err := os.Open(dir)
-	if err != nil {
-		return
-	}
-	defer d.Close()
-
-	names, _ := d.Readdirnames(-1)
-	sort.Strings(names)
-
-	for _, n := range names {
-		matched, err := filepath.Match(pattern, n)
-		if err != nil {
-			return m, err
-		}
-		if matched {
-			m = append(m, filepath.Join(dir, n))
-		}
-	}
-	return
-}

lib/osutil/glob_windows_test.go

@@ -1,29 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build windows
-
-package osutil_test
-
-import (
-	"testing"
-
-	"github.com/syncthing/syncthing/lib/osutil"
-)
-
-func TestGlob(t *testing.T) {
-	testcases := []string{
-		`C:\*`,
-		`\\?\C:\*`,
-		`\\?\C:\Users`,
-		`\\?\\\?\C:\Users`,
-	}
-	for _, tc := range testcases {
-		if _, err := osutil.Glob(tc); err != nil {
-			t.Fatalf("pattern %s failed: %v", tc, err)
-		}
-	}
-}

lib/osutil/hidden_unix.go

@@ -8,12 +8,4 @@
 
 package osutil
 
-func HideFile(path string) error {
-	return nil
-}
-
-func ShowFile(path string) error {
-	return nil
-}
-
 func HideConsole() {}

lib/osutil/hidden_windows.go

@@ -10,36 +10,6 @@ package osutil
 
 import "syscall"
 
-func HideFile(path string) error {
-	p, err := syscall.UTF16PtrFromString(path)
-	if err != nil {
-		return err
-	}
-
-	attrs, err := syscall.GetFileAttributes(p)
-	if err != nil {
-		return err
-	}
-
-	attrs |= syscall.FILE_ATTRIBUTE_HIDDEN
-	return syscall.SetFileAttributes(p, attrs)
-}
-
-func ShowFile(path string) error {
-	p, err := syscall.UTF16PtrFromString(path)
-	if err != nil {
-		return err
-	}
-
-	attrs, err := syscall.GetFileAttributes(p)
-	if err != nil {
-		return err
-	}
-
-	attrs &^= syscall.FILE_ATTRIBUTE_HIDDEN
-	return syscall.SetFileAttributes(p, attrs)
-}
-
 func HideConsole() {
 	getConsoleWindow := syscall.NewLazyDLL("kernel32.dll").NewProc("GetConsoleWindow")
 	showWindow := syscall.NewLazyDLL("user32.dll").NewProc("ShowWindow")

lib/osutil/lstat_broken.go

@@ -1,29 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build linux android
-
-package osutil
-
-import (
-	"os"
-	"syscall"
-	"time"
-)
-
-// Lstat is like os.Lstat, except lobotomized for Android. See
-// https://forum.syncthing.net/t/2395
-func Lstat(name string) (fi os.FileInfo, err error) {
-	for i := 0; i < 10; i++ { // We have to draw the line somewhere
-		fi, err = os.Lstat(name)
-		if err, ok := err.(*os.PathError); ok && err.Err == syscall.EINTR {
-			time.Sleep(time.Duration(i+1) * time.Millisecond)
-			continue
-		}
-		return
-	}
-	return
-}

lib/osutil/lstat_ok.go

@@ -1,15 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build !linux,!android
-
-package osutil
-
-import "os"
-
-func Lstat(name string) (fi os.FileInfo, err error) {
-	return os.Lstat(name)
-}

lib/osutil/mkdirall.go

@@ -1,17 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build !windows
-
-package osutil
-
-import (
-	"os"
-)
-
-func MkdirAll(path string, perm os.FileMode) error {
-	return os.MkdirAll(path, perm)
-}

lib/osutil/mkdirall_windows.go

@@ -1,93 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-//   * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//   * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Modified by Zillode to fix https://github.com/syncthing/syncthing/issues/1822
-// Sync with https://github.com/golang/go/blob/master/src/os/path.go
-// See https://github.com/golang/go/issues/10900
-
-package osutil
-
-import (
-	"os"
-	"path/filepath"
-	"syscall"
-)
-
-// MkdirAll creates a directory named path, along with any necessary parents,
-// and returns nil, or else returns an error.
-// The permission bits perm are used for all directories that MkdirAll creates.
-// If path is already a directory, MkdirAll does nothing and returns nil.
-func MkdirAll(path string, perm os.FileMode) error {
-	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
-	dir, err := os.Stat(path)
-	if err == nil {
-		if dir.IsDir() {
-			return nil
-		}
-		return &os.PathError{
-			Op:   "mkdir",
-			Path: path,
-			Err:  syscall.ENOTDIR,
-		}
-	}
-
-	// Slow path: make sure parent exists and then call Mkdir for path.
-	i := len(path)
-	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
-		i--
-	}
-
-	j := i
-	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
-		j--
-	}
-
-	if j > 1 {
-		// Create parent
-		parent := path[0 : j-1]
-		if parent != filepath.VolumeName(parent) {
-			err = MkdirAll(parent, perm)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	// Parent now exists; invoke Mkdir and use its result.
-	err = os.Mkdir(path, perm)
-	if err != nil {
-		// Handle arguments like "foo/." by
-		// double-checking that directory doesn't exist.
-		dir, err1 := os.Lstat(path)
-		if err1 == nil && dir.IsDir() {
-			return nil
-		}
-		return err
-	}
-	return nil
-}

lib/osutil/net.go

@@ -0,0 +1,56 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package osutil
+
+import (
+	"bytes"
+	"net"
+)
+
+// ResolveInterfaceAddresses returns available addresses of the given network
+// type for a given interface.
+func ResolveInterfaceAddresses(network, nameOrMac string) []string {
+	intf, err := net.InterfaceByName(nameOrMac)
+	if err == nil {
+		return interfaceAddresses(network, intf)
+	}
+
+	mac, err := net.ParseMAC(nameOrMac)
+	if err != nil {
+		return []string{nameOrMac}
+	}
+
+	intfs, err := net.Interfaces()
+	if err != nil {
+		return []string{nameOrMac}
+	}
+
+	for _, intf := range intfs {
+		if bytes.Equal(intf.HardwareAddr, mac) {
+			return interfaceAddresses(network, &intf)
+		}
+	}
+
+	return []string{nameOrMac}
+}
+
+func interfaceAddresses(network string, intf *net.Interface) []string {
+	var out []string
+	addrs, err := intf.Addrs()
+	if err != nil {
+		return out
+	}
+
+	for _, addr := range addrs {
+		ipnet, ok := addr.(*net.IPNet)
+		if ok && (network == "tcp" || (network == "tcp4" && len(ipnet.IP) == net.IPv4len) || (network == "tcp6" && len(ipnet.IP) == net.IPv6len)) {
+			out = append(out, ipnet.IP.String())
+		}
+	}
+
+	return out
+}

lib/osutil/osutil.go

@@ -9,19 +9,16 @@ package osutil
 
 import (
 	"errors"
-	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
 
-	"github.com/calmh/du"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
-var errNoHome = errors.New("no home directory found - set $HOME (or the platform equivalent)")
-
 // Try to keep this entire operation atomic-like. We shouldn't be doing this
 // often enough that there is any contention on this lock.
 var renameLock = sync.NewMutex()
@@ -29,12 +26,12 @@ var renameLock = sync.NewMutex()
 // TryRename renames a file, leaving source file intact in case of failure.
 // Tries hard to succeed on various systems by temporarily tweaking directory
 // permissions and removing the destination file when necessary.
-func TryRename(from, to string) error {
+func TryRename(filesystem fs.Filesystem, from, to string) error {
 	renameLock.Lock()
 	defer renameLock.Unlock()
 
-	return withPreparedTarget(from, to, func() error {
+	return withPreparedTarget(filesystem, from, to, func() error {
-		return os.Rename(from, to)
+		return filesystem.Rename(from, to)
 	})
 }
 
@@ -43,28 +40,28 @@ func TryRename(from, to string) error {
 // for situations like committing a temp file to it's final location.
 // Tries hard to succeed on various systems by temporarily tweaking directory
 // permissions and removing the destination file when necessary.
-func Rename(from, to string) error {
+func Rename(filesystem fs.Filesystem, from, to string) error {
 	// Don't leave a dangling temp file in case of rename error
 	if !(runtime.GOOS == "windows" && strings.EqualFold(from, to)) {
-		defer os.Remove(from)
+		defer filesystem.Remove(from)
 	}
-	return TryRename(from, to)
+	return TryRename(filesystem, from, to)
 }
 
 // Copy copies the file content from source to destination.
 // Tries hard to succeed on various systems by temporarily tweaking directory
 // permissions and removing the destination file when necessary.
-func Copy(from, to string) (err error) {
+func Copy(filesystem fs.Filesystem, from, to string) (err error) {
-	return withPreparedTarget(from, to, func() error {
+	return withPreparedTarget(filesystem, from, to, func() error {
-		return copyFileContents(from, to)
+		return copyFileContents(filesystem, from, to)
 	})
 }
 
 // InWritableDir calls fn(path), while making sure that the directory
 // containing `path` is writable for the duration of the call.
-func InWritableDir(fn func(string) error, path string) error {
+func InWritableDir(fn func(string) error, fs fs.Filesystem, path string) error {
 	dir := filepath.Dir(path)
-	info, err := os.Stat(dir)
+	info, err := fs.Stat(dir)
 	if err != nil {
 		return err
 	}
@@ -75,10 +72,10 @@ func InWritableDir(fn func(string) error, path string) error {
 		// A non-writeable directory (for this user; we assume that's the
 		// relevant part). Temporarily change the mode so we can delete the
 		// file or directory inside it.
-		err = os.Chmod(dir, 0755)
+		err = fs.Chmod(dir, 0755)
 		if err == nil {
 			defer func() {
-				err = os.Chmod(dir, info.Mode())
+				err = fs.Chmod(dir, info.Mode())
 				if err != nil {
 					// We managed to change the permission bits like a
 					// millisecond ago, so it'd be bizarre if we couldn't
@@ -92,59 +89,22 @@ func InWritableDir(fn func(string) error, path string) error {
 	return fn(path)
 }
 
-func ExpandTilde(path string) (string, error) {
-	if path == "~" {
-		return getHomeDir()
-	}
-
-	path = filepath.FromSlash(path)
-	if !strings.HasPrefix(path, fmt.Sprintf("~%c", os.PathSeparator)) {
-		return path, nil
-	}
-
-	home, err := getHomeDir()
-	if err != nil {
-		return "", err
-	}
-	return filepath.Join(home, path[2:]), nil
-}
-
-func getHomeDir() (string, error) {
-	var home string
-
-	switch runtime.GOOS {
-	case "windows":
-		home = filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath"))
-		if home == "" {
-			home = os.Getenv("UserProfile")
-		}
-	default:
-		home = os.Getenv("HOME")
-	}
-
-	if home == "" {
-		return "", errNoHome
-	}
-
-	return home, nil
-}
-
 // Tries hard to succeed on various systems by temporarily tweaking directory
 // permissions and removing the destination file when necessary.
-func withPreparedTarget(from, to string, f func() error) error {
+func withPreparedTarget(filesystem fs.Filesystem, from, to string, f func() error) error {
 	// Make sure the destination directory is writeable
 	toDir := filepath.Dir(to)
-	if info, err := os.Stat(toDir); err == nil && info.IsDir() && info.Mode()&0200 == 0 {
+	if info, err := filesystem.Stat(toDir); err == nil && info.IsDir() && info.Mode()&0200 == 0 {
-		os.Chmod(toDir, 0755)
+		filesystem.Chmod(toDir, 0755)
-		defer os.Chmod(toDir, info.Mode())
+		defer filesystem.Chmod(toDir, info.Mode())
 	}
 
 	// On Windows, make sure the destination file is writeable (or we can't delete it)
 	if runtime.GOOS == "windows" {
-		os.Chmod(to, 0666)
+		filesystem.Chmod(to, 0666)
 		if !strings.EqualFold(from, to) {
-			err := os.Remove(to)
+			err := filesystem.Remove(to)
-			if err != nil && !os.IsNotExist(err) {
+			if err != nil && !fs.IsNotExist(err) {
 				return err
 			}
 		}
@@ -156,13 +116,13 @@ func withPreparedTarget(from, to string, f func() error) error {
 // by dst. The file will be created if it does not already exist. If the
 // destination file exists, all it's contents will be replaced by the contents
 // of the source file.
-func copyFileContents(src, dst string) (err error) {
+func copyFileContents(filesystem fs.Filesystem, src, dst string) (err error) {
-	in, err := os.Open(src)
+	in, err := filesystem.Open(src)
 	if err != nil {
 		return
 	}
 	defer in.Close()
-	out, err := os.Create(dst)
+	out, err := filesystem.Create(dst)
 	if err != nil {
 		return
 	}
@@ -193,13 +153,3 @@ func init() {
 func IsWindowsExecutable(path string) bool {
 	return execExts[strings.ToLower(filepath.Ext(path))]
 }
-
-func DiskFreeBytes(path string) (free int64, err error) {
-	u, err := du.Get(path)
-	return u.FreeBytes, err
-}
-
-func DiskFreePercentage(path string) (freePct float64, err error) {
-	u, err := du.Get(path)
-	return (float64(u.FreeBytes) / float64(u.TotalBytes)) * 100, err
-}

lib/osutil/osutil_test.go

@@ -11,6 +11,7 @@ import (
 	"runtime"
 	"testing"
 
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/osutil"
 )
 
@@ -21,6 +22,8 @@ func TestInWriteableDir(t *testing.T) {
 	}
 	defer os.RemoveAll("testdata")
 
+	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, ".")
+
 	os.Mkdir("testdata", 0700)
 	os.Mkdir("testdata/rw", 0700)
 	os.Mkdir("testdata/ro", 0500)
@@ -36,35 +39,35 @@ func TestInWriteableDir(t *testing.T) {
 
 	// These should succeed
 
-	err = osutil.InWritableDir(create, "testdata/file")
+	err = osutil.InWritableDir(create, fs, "testdata/file")
 	if err != nil {
 		t.Error("testdata/file:", err)
 	}
-	err = osutil.InWritableDir(create, "testdata/rw/foo")
+	err = osutil.InWritableDir(create, fs, "testdata/rw/foo")
 	if err != nil {
 		t.Error("testdata/rw/foo:", err)
 	}
-	err = osutil.InWritableDir(os.Remove, "testdata/rw/foo")
+	err = osutil.InWritableDir(os.Remove, fs, "testdata/rw/foo")
 	if err != nil {
 		t.Error("testdata/rw/foo:", err)
 	}
 
-	err = osutil.InWritableDir(create, "testdata/ro/foo")
+	err = osutil.InWritableDir(create, fs, "testdata/ro/foo")
 	if err != nil {
 		t.Error("testdata/ro/foo:", err)
 	}
-	err = osutil.InWritableDir(os.Remove, "testdata/ro/foo")
+	err = osutil.InWritableDir(os.Remove, fs, "testdata/ro/foo")
 	if err != nil {
 		t.Error("testdata/ro/foo:", err)
 	}
 
 	// These should not
 
-	err = osutil.InWritableDir(create, "testdata/nonexistent/foo")
+	err = osutil.InWritableDir(create, fs, "testdata/nonexistent/foo")
 	if err == nil {
 		t.Error("testdata/nonexistent/foo returned nil error")
 	}
-	err = osutil.InWritableDir(create, "testdata/file/foo")
+	err = osutil.InWritableDir(create, fs, "testdata/file/foo")
 	if err == nil {
 		t.Error("testdata/file/foo returned nil error")
 	}
@@ -101,8 +104,10 @@ func TestInWritableDirWindowsRemove(t *testing.T) {
 	create("testdata/windows/ro/readonly")
 	os.Chmod("testdata/windows/ro/readonly", 0500)
 
+	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, ".")
+
 	for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
-		err := osutil.InWritableDir(os.Remove, path)
+		err := osutil.InWritableDir(os.Remove, fs, path)
 		if err != nil {
 			t.Errorf("Unexpected error %s: %s", path, err)
 		}
@@ -174,6 +179,8 @@ func TestInWritableDirWindowsRename(t *testing.T) {
 	create("testdata/windows/ro/readonly")
 	os.Chmod("testdata/windows/ro/readonly", 0500)
 
+	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, ".")
+
 	for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
 		err := os.Rename(path, path+"new")
 		if err == nil {
@@ -183,11 +190,11 @@ func TestInWritableDirWindowsRename(t *testing.T) {
 	}
 
 	rename := func(path string) error {
-		return osutil.Rename(path, path+"new")
+		return osutil.Rename(fs, path, path+"new")
 	}
 
 	for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
-		err := osutil.InWritableDir(rename, path)
+		err := osutil.InWritableDir(rename, fs, path)
 		if err != nil {
 			t.Errorf("Unexpected error %s: %s", path, err)
 		}
@@ -197,18 +204,3 @@ func TestInWritableDirWindowsRename(t *testing.T) {
 		}
 	}
 }
-
-func TestDiskUsage(t *testing.T) {
-	free, err := osutil.DiskFreePercentage(".")
-	if err != nil {
-		if runtime.GOOS == "netbsd" ||
-			runtime.GOOS == "openbsd" ||
-			runtime.GOOS == "solaris" {
-			t.Skip()
-		}
-		t.Errorf("Unexpected error: %s", err)
-	}
-	if free < 1 {
-		t.Error("Disk is full?", free)
-	}
-}

lib/osutil/sync.go

@@ -1,34 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package osutil
-
-import (
-	"os"
-	"runtime"
-)
-
-func SyncFile(path string) error {
-	flag := 0
-	if runtime.GOOS == "windows" {
-		flag = os.O_WRONLY
-	}
-	fd, err := os.OpenFile(path, flag, 0)
-	if err != nil {
-		return err
-	}
-	defer fd.Close()
-	// MacOS and Windows do not flush the disk cache
-	return fd.Sync()
-}
-
-func SyncDir(path string) error {
-	if runtime.GOOS == "windows" {
-		// not supported by Windows
-		return nil
-	}
-	return SyncFile(path)
-}

lib/osutil/tempfile.go

@@ -0,0 +1,63 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package osutil
+
+import (
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/fs"
+)
+
+var rand uint32
+var randmu sync.Mutex
+
+func reseed() uint32 {
+	return uint32(time.Now().UnixNano() + int64(os.Getpid()))
+}
+
+func nextSuffix() string {
+	randmu.Lock()
+	r := rand
+	if r == 0 {
+		r = reseed()
+	}
+	r = r*1664525 + 1013904223 // constants from Numerical Recipes
+	rand = r
+	randmu.Unlock()
+	return strconv.Itoa(int(1e9 + r%1e9))[1:]
+}
+
+// TempFile creates a new temporary file in the directory dir
+// with a name beginning with prefix, opens the file for reading
+// and writing, and returns the resulting *os.File.
+// If dir is the empty string, TempFile uses the default directory
+// for temporary files (see os.TempDir).
+// Multiple programs calling TempFile simultaneously
+// will not choose the same file. The caller can use f.Name()
+// to find the pathname of the file. It is the caller's responsibility
+// to remove the file when no longer needed.
+func TempFile(filesystem fs.Filesystem, dir, prefix string) (f fs.File, err error) {
+	nconflict := 0
+	for i := 0; i < 10000; i++ {
+		name := filepath.Join(dir, prefix+nextSuffix())
+		f, err = filesystem.OpenFile(name, fs.OptReadWrite|fs.OptCreate|fs.OptExclusive, 0600)
+		if fs.IsExist(err) {
+			if nconflict++; nconflict > 10 {
+				randmu.Lock()
+				rand = reseed()
+				randmu.Unlock()
+			}
+			continue
+		}
+		break
+	}
+	return
+}

lib/osutil/traversessymlink.go

@@ -8,9 +8,10 @@ package osutil
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/syncthing/syncthing/lib/fs"
 )
 
 // TraversesSymlinkError is an error indicating symlink traversal
@@ -34,9 +35,10 @@ func (e NotADirectoryError) Error() string {
 // TraversesSymlink returns an error if base and any path component of name up to and
 // including filepath.Join(base, name) traverses a symlink.
 // Base and name must both be clean and name must be relative to base.
-func TraversesSymlink(base, name string) error {
+func TraversesSymlink(filesystem fs.Filesystem, name string) error {
+	base := "."
 	path := base
-	info, err := Lstat(path)
+	info, err := filesystem.Lstat(path)
 	if err != nil {
 		return err
 	}
@@ -51,17 +53,17 @@ func TraversesSymlink(base, name string) error {
 		return nil
 	}
 
-	parts := strings.Split(name, string(os.PathSeparator))
+	parts := strings.Split(name, string(fs.PathSeparator))
 	for _, part := range parts {
 		path = filepath.Join(path, part)
-		info, err := Lstat(path)
+		info, err := filesystem.Lstat(path)
 		if err != nil {
-			if os.IsNotExist(err) {
+			if fs.IsNotExist(err) {
 				return nil
 			}
 			return err
 		}
-		if info.Mode()&os.ModeSymlink != 0 {
+		if info.IsSymlink() {
 			return &TraversesSymlinkError{
 				path: strings.TrimPrefix(path, base),
 			}

lib/osutil/traversessymlink_test.go

@@ -12,17 +12,20 @@ import (
 	"os"
 	"testing"
 
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/osutil"
 )
 
 func TestTraversesSymlink(t *testing.T) {
 	os.RemoveAll("testdata")
 	defer os.RemoveAll("testdata")
-	os.MkdirAll("testdata/a/b/c", 0755)
+
-	os.Symlink("b", "testdata/a/l")
+	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata")
+	fs.MkdirAll("a/b/c", 0755)
+	fs.CreateSymlink("b", "a/l")
 
 	// a/l -> b, so a/l/c should resolve by normal stat
-	info, err := osutil.Lstat("testdata/a/l/c")
+	info, err := fs.Lstat("a/l/c")
 	if err != nil {
 		t.Fatal("unexpected error", err)
 	}
@@ -52,7 +55,7 @@ func TestTraversesSymlink(t *testing.T) {
 	}
 
 	for _, tc := range cases {
-		if res := osutil.TraversesSymlink("testdata", tc.name); tc.traverses == (res == nil) {
+		if res := osutil.TraversesSymlink(fs, tc.name); tc.traverses == (res == nil) {
 			t.Errorf("TraversesSymlink(%q) = %v, should be %v", tc.name, res, tc.traverses)
 		}
 	}
@@ -63,10 +66,11 @@ var traversesSymlinkResult error
 func BenchmarkTraversesSymlink(b *testing.B) {
 	os.RemoveAll("testdata")
 	defer os.RemoveAll("testdata")
-	os.MkdirAll("testdata/a/b/c", 0755)
+	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata")
+	fs.MkdirAll("a/b/c", 0755)
 
 	for i := 0; i < b.N; i++ {
-		traversesSymlinkResult = osutil.TraversesSymlink("testdata", "a/b/c")
+		traversesSymlinkResult = osutil.TraversesSymlink(fs, "a/b/c")
 	}
 
 	b.ReportAllocs()

lib/scanner/blockqueue.go

@@ -9,7 +9,6 @@ package scanner
 import (
 	"context"
 	"errors"
-	"path/filepath"
 
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
@@ -64,7 +63,6 @@ func HashFile(ctx context.Context, fs fs.Filesystem, path string, blockSize int,
 // is closed and all items handled.
 type parallelHasher struct {
 	fs            fs.Filesystem
-	dir           string
 	blockSize     int
 	workers       int
 	outbox        chan<- protocol.FileInfo
@@ -75,10 +73,9 @@ type parallelHasher struct {
 	wg            sync.WaitGroup
 }
 
-func newParallelHasher(ctx context.Context, fs fs.Filesystem, dir string, blockSize, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) {
+func newParallelHasher(ctx context.Context, fs fs.Filesystem, blockSize, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) {
 	ph := &parallelHasher{
 		fs:            fs,
-		dir:           dir,
 		blockSize:     blockSize,
 		workers:       workers,
 		outbox:        outbox,
@@ -111,7 +108,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) {
 				panic("Bug. Asked to hash a directory or a deleted file.")
 			}
 
-			blocks, err := HashFile(ctx, ph.fs, filepath.Join(ph.dir, f.Name), ph.blockSize, ph.counter, ph.useWeakHashes)
+			blocks, err := HashFile(ctx, ph.fs, f.Name, ph.blockSize, ph.counter, ph.useWeakHashes)
 			if err != nil {
 				l.Debugln("hash error:", f.Name, err)
 				continue

lib/scanner/infinitefs_test.go

@@ -19,6 +19,7 @@ import (
 )
 
 type infiniteFS struct {
+	fs.Filesystem
 	width    int   // number of files and directories per level
 	depth    int   // number of tree levels to simulate
 	filesize int64 // size of each file in bytes
@@ -50,18 +51,6 @@ func (i infiniteFS) Open(name string) (fs.File, error) {
 	return &fakeFile{name, i.filesize, 0}, nil
 }
 
-func (infiniteFS) Chmod(name string, mode fs.FileMode) error                   { return errNotSupp }
-func (infiniteFS) Chtimes(name string, atime time.Time, mtime time.Time) error { return errNotSupp }
-func (infiniteFS) Create(name string) (fs.File, error)                         { return nil, errNotSupp }
-func (infiniteFS) CreateSymlink(name, target string) error                     { return errNotSupp }
-func (infiniteFS) Mkdir(name string, perm fs.FileMode) error                   { return errNotSupp }
-func (infiniteFS) ReadSymlink(name string) (string, error)                     { return "", errNotSupp }
-func (infiniteFS) Remove(name string) error                                    { return errNotSupp }
-func (infiniteFS) Rename(oldname, newname string) error                        { return errNotSupp }
-func (infiniteFS) Stat(name string) (fs.FileInfo, error)                       { return nil, errNotSupp }
-func (infiniteFS) SymlinksSupported() bool                                     { return false }
-func (infiniteFS) Walk(root string, walkFn fs.WalkFunc) error                  { return errNotSupp }
-
 type fakeInfo struct {
 	name string
 	size int64
@@ -71,7 +60,7 @@ func (f fakeInfo) Name() string       { return f.name }
 func (f fakeInfo) Mode() fs.FileMode  { return 0755 }
 func (f fakeInfo) Size() int64        { return f.size }
 func (f fakeInfo) ModTime() time.Time { return time.Unix(1234567890, 0) }
-func (f fakeInfo) IsDir() bool        { return strings.Contains(filepath.Base(f.name), "dir") }
+func (f fakeInfo) IsDir() bool        { return strings.Contains(filepath.Base(f.name), "dir") || f.name == "." }
 func (f fakeInfo) IsRegular() bool    { return !f.IsDir() }
 func (f fakeInfo) IsSymlink() bool    { return false }
 
@@ -81,6 +70,10 @@ type fakeFile struct {
 	readOffset int64
 }
 
+func (f *fakeFile) Name() string {
+	return f.name
+}
+
 func (f *fakeFile) Read(bs []byte) (int, error) {
 	remaining := f.size - f.readOffset
 	if remaining == 0 {
@@ -98,6 +91,10 @@ func (f *fakeFile) Stat() (fs.FileInfo, error) {
 	return fakeInfo{f.name, f.size}, nil
 }
 
-func (f *fakeFile) WriteAt(bs []byte, offs int64) (int, error) { return 0, errNotSupp }
+func (f *fakeFile) Write([]byte) (int, error)          { return 0, errNotSupp }
-func (f *fakeFile) Close() error                               { return nil }
+func (f *fakeFile) WriteAt([]byte, int64) (int, error) { return 0, errNotSupp }
-func (f *fakeFile) Truncate(size int64) error                  { return errNotSupp }
+func (f *fakeFile) Close() error                       { return nil }
+func (f *fakeFile) Truncate(size int64) error          { return errNotSupp }
+func (f *fakeFile) ReadAt([]byte, int64) (int, error)  { return 0, errNotSupp }
+func (f *fakeFile) Seek(int64, int) (int64, error)     { return 0, errNotSupp }
+func (f *fakeFile) Sync() error                        { return nil }

lib/scanner/walk.go

@@ -9,7 +9,6 @@ package scanner
 import (
 	"context"
 	"errors"
-	"path/filepath"
 	"runtime"
 	"sync/atomic"
 	"time"
@@ -42,8 +41,6 @@ func init() {
 type Config struct {
 	// Folder for which the walker has been created
 	Folder string
-	// Dir is the base directory for the walk
-	Dir string
 	// Limit walking to these paths within Dir, or no limit if Sub is empty
 	Subs []string
 	// BlockSize controls the size of the block used when hashing.
@@ -86,7 +83,7 @@ func Walk(ctx context.Context, cfg Config) (chan protocol.FileInfo, error) {
 		w.CurrentFiler = noCurrentFiler{}
 	}
 	if w.Filesystem == nil {
-		w.Filesystem = fs.DefaultFilesystem
+		panic("no filesystem specified")
 	}
 
 	return w.walk(ctx)
@@ -99,7 +96,7 @@ type walker struct {
 // Walk returns the list of files found in the local folder by scanning the
 // file system. Files are blockwise hashed.
 func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
-	l.Debugln("Walk", w.Dir, w.Subs, w.BlockSize, w.Matcher)
+	l.Debugln("Walk", w.Subs, w.BlockSize, w.Matcher)
 
 	if err := w.checkDir(); err != nil {
 		return nil, err
@@ -113,10 +110,10 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
 	go func() {
 		hashFiles := w.walkAndHashFiles(ctx, toHashChan, finishedChan)
 		if len(w.Subs) == 0 {
-			w.Filesystem.Walk(w.Dir, hashFiles)
+			w.Filesystem.Walk(".", hashFiles)
 		} else {
 			for _, sub := range w.Subs {
-				w.Filesystem.Walk(filepath.Join(w.Dir, sub), hashFiles)
+				w.Filesystem.Walk(sub, hashFiles)
 			}
 		}
 		close(toHashChan)
@@ -125,7 +122,7 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
 	// We're not required to emit scan progress events, just kick off hashers,
 	// and feed inputs directly from the walker.
 	if w.ProgressTickIntervalS < 0 {
-		newParallelHasher(ctx, w.Filesystem, w.Dir, w.BlockSize, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes)
+		newParallelHasher(ctx, w.Filesystem, w.BlockSize, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes)
 		return finishedChan, nil
 	}
 
@@ -156,7 +153,7 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
 		done := make(chan struct{})
 		progress := newByteCounter()
 
-		newParallelHasher(ctx, w.Filesystem, w.Dir, w.BlockSize, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes)
+		newParallelHasher(ctx, w.Filesystem, w.BlockSize, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes)
 
 		// A routine which actually emits the FolderScanProgress events
 		// every w.ProgressTicker ticks, until the hasher routines terminate.
@@ -166,13 +163,13 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
 			for {
 				select {
 				case <-done:
-					l.Debugln("Walk progress done", w.Dir, w.Subs, w.BlockSize, w.Matcher)
+					l.Debugln("Walk progress done", w.Folder, w.Subs, w.BlockSize, w.Matcher)
 					ticker.Stop()
 					return
 				case <-ticker.C:
 					current := progress.Total()
 					rate := progress.Rate()
-					l.Debugf("Walk %s %s current progress %d/%d at %.01f MiB/s (%d%%)", w.Dir, w.Subs, current, total, rate/1024/1024, current*100/total)
+					l.Debugf("Walk %s %s current progress %d/%d at %.01f MiB/s (%d%%)", w.Folder, w.Subs, current, total, rate/1024/1024, current*100/total)
 					events.Default.Log(events.FolderScanProgress, map[string]interface{}{
 						"folder":  w.Folder,
 						"current": current,
@@ -203,7 +200,7 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
 
 func (w *walker) walkAndHashFiles(ctx context.Context, fchan, dchan chan protocol.FileInfo) fs.WalkFunc {
 	now := time.Now()
-	return func(absPath string, info fs.FileInfo, err error) error {
+	return func(path string, info fs.FileInfo, err error) error {
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
@@ -219,58 +216,52 @@ func (w *walker) walkAndHashFiles(ctx context.Context, fchan, dchan chan protoco
 		}
 
 		if err != nil {
-			l.Debugln("error:", absPath, info, err)
+			l.Debugln("error:", path, info, err)
 			return skip
 		}
 
-		relPath, err := filepath.Rel(w.Dir, absPath)
+		if path == "." {
-		if err != nil {
-			l.Debugln("rel error:", absPath, err)
-			return skip
-		}
-
-		if relPath == "." {
 			return nil
 		}
 
-		info, err = w.Filesystem.Lstat(absPath)
+		info, err = w.Filesystem.Lstat(path)
 		// An error here would be weird as we've already gotten to this point, but act on it nonetheless
 		if err != nil {
 			return skip
 		}
 
-		if ignore.IsTemporary(relPath) {
+		if ignore.IsTemporary(path) {
-			l.Debugln("temporary:", relPath)
+			l.Debugln("temporary:", path)
 			if info.IsRegular() && info.ModTime().Add(w.TempLifetime).Before(now) {
-				w.Filesystem.Remove(absPath)
+				w.Filesystem.Remove(path)
-				l.Debugln("removing temporary:", relPath, info.ModTime())
+				l.Debugln("removing temporary:", path, info.ModTime())
 			}
 			return nil
 		}
 
-		if ignore.IsInternal(relPath) {
+		if ignore.IsInternal(path) {
-			l.Debugln("ignored (internal):", relPath)
+			l.Debugln("ignored (internal):", path)
 			return skip
 		}
 
-		if w.Matcher.Match(relPath).IsIgnored() {
+		if w.Matcher.Match(path).IsIgnored() {
-			l.Debugln("ignored (patterns):", relPath)
+			l.Debugln("ignored (patterns):", path)
 			return skip
 		}
 
-		if !utf8.ValidString(relPath) {
+		if !utf8.ValidString(path) {
-			l.Warnf("File name %q is not in UTF8 encoding; skipping.", relPath)
+			l.Warnf("File name %q is not in UTF8 encoding; skipping.", path)
 			return skip
 		}
 
-		relPath, shouldSkip := w.normalizePath(absPath, relPath)
+		path, shouldSkip := w.normalizePath(path)
 		if shouldSkip {
 			return skip
 		}
 
 		switch {
 		case info.IsSymlink():
-			if err := w.walkSymlink(ctx, absPath, relPath, dchan); err != nil {
+			if err := w.walkSymlink(ctx, path, dchan); err != nil {
 				return err
 			}
 			if info.IsDir() {
@@ -280,10 +271,10 @@ func (w *walker) walkAndHashFiles(ctx context.Context, fchan, dchan chan protoco
 			return nil
 
 		case info.IsDir():
-			err = w.walkDir(ctx, relPath, info, dchan)
+			err = w.walkDir(ctx, path, info, dchan)
 
 		case info.IsRegular():
-			err = w.walkRegular(ctx, relPath, info, fchan)
+			err = w.walkRegular(ctx, path, info, fchan)
 		}
 
 		return err
@@ -375,7 +366,7 @@ func (w *walker) walkDir(ctx context.Context, relPath string, info fs.FileInfo,
 
 // walkSymlink returns nil or an error, if the error is of the nature that
 // it should stop the entire walk.
-func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan chan protocol.FileInfo) error {
+func (w *walker) walkSymlink(ctx context.Context, relPath string, dchan chan protocol.FileInfo) error {
 	// Symlinks are not supported on Windows. We ignore instead of returning
 	// an error.
 	if runtime.GOOS == "windows" {
@@ -387,9 +378,9 @@ func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan
 	// checking that their existing blocks match with the blocks in
 	// the index.
 
-	target, err := w.Filesystem.ReadSymlink(absPath)
+	target, err := w.Filesystem.ReadSymlink(relPath)
 	if err != nil {
-		l.Debugln("readlink error:", absPath, err)
+		l.Debugln("readlink error:", relPath, err)
 		return nil
 	}
 
@@ -413,7 +404,7 @@ func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan
 		SymlinkTarget: target,
 	}
 
-	l.Debugln("symlink changedb:", absPath, f)
+	l.Debugln("symlink changedb:", relPath, f)
 
 	select {
 	case dchan <- f:
@@ -426,55 +417,58 @@ func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan
 
 // normalizePath returns the normalized relative path (possibly after fixing
 // it on disk), or skip is true.
-func (w *walker) normalizePath(absPath, relPath string) (normPath string, skip bool) {
+func (w *walker) normalizePath(path string) (normPath string, skip bool) {
 	if runtime.GOOS == "darwin" {
 		// Mac OS X file names should always be NFD normalized.
-		normPath = norm.NFD.String(relPath)
+		normPath = norm.NFD.String(path)
 	} else {
 		// Every other OS in the known universe uses NFC or just plain
 		// doesn't bother to define an encoding. In our case *we* do care,
 		// so we enforce NFC regardless.
-		normPath = norm.NFC.String(relPath)
+		normPath = norm.NFC.String(path)
 	}
 
-	if relPath != normPath {
+	if path != normPath {
 		// The file name was not normalized.
 
 		if !w.AutoNormalize {
 			// We're not authorized to do anything about it, so complain and skip.
 
-			l.Warnf("File name %q is not in the correct UTF8 normalization form; skipping.", relPath)
+			l.Warnf("File name %q is not in the correct UTF8 normalization form; skipping.", path)
 			return "", true
 		}
 
 		// We will attempt to normalize it.
-		normalizedPath := filepath.Join(w.Dir, normPath)
+		if _, err := w.Filesystem.Lstat(normPath); fs.IsNotExist(err) {
-		if _, err := w.Filesystem.Lstat(normalizedPath); fs.IsNotExist(err) {
 			// Nothing exists with the normalized filename. Good.
-			if err = w.Filesystem.Rename(absPath, normalizedPath); err != nil {
+			if err = w.Filesystem.Rename(path, normPath); err != nil {
-				l.Infof(`Error normalizing UTF8 encoding of file "%s": %v`, relPath, err)
+				l.Infof(`Error normalizing UTF8 encoding of file "%s": %v`, path, err)
 				return "", true
 			}
-			l.Infof(`Normalized UTF8 encoding of file name "%s".`, relPath)
+			l.Infof(`Normalized UTF8 encoding of file name "%s".`, path)
 		} else {
 			// There is something already in the way at the normalized
 			// file name.
-			l.Infof(`File "%s" has UTF8 encoding conflict with another file; ignoring.`, relPath)
+			l.Infof(`File "%s" path has UTF8 encoding conflict with another file; ignoring.`, path)
 			return "", true
 		}
 	}
 
-	return normPath, false
+	return path, false
 }
 
 func (w *walker) checkDir() error {
-	if info, err := w.Filesystem.Lstat(w.Dir); err != nil {
+	info, err := w.Filesystem.Lstat(".")
+	if err != nil {
 		return err
-	} else if !info.IsDir() {
-		return errors.New(w.Dir + ": not a directory")
-	} else {
-		l.Debugln("checkDir", w.Dir, info)
 	}
+
+	if !info.IsDir() {
+		return errors.New(w.Filesystem.URI() + ": not a directory")
+	}
+
+	l.Debugln("checkDir", w.Filesystem.Type(), w.Filesystem.URI(), info)
+
 	return nil
 }
 

lib/scanner/walk_test.go

@@ -23,7 +23,6 @@ import (
 	"github.com/d4l3k/messagediff"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/ignore"
-	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"golang.org/x/text/unicode/norm"
 )
@@ -54,18 +53,18 @@ func init() {
 }
 
 func TestWalkSub(t *testing.T) {
-	ignores := ignore.New()
+	ignores := ignore.New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."))
 	err := ignores.Load("testdata/.stignore")
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	fchan, err := Walk(context.TODO(), Config{
-		Dir:       "testdata",
+		Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
-		Subs:      []string{"dir2"},
+		Subs:       []string{"dir2"},
-		BlockSize: 128 * 1024,
+		BlockSize:  128 * 1024,
-		Matcher:   ignores,
+		Matcher:    ignores,
-		Hashers:   2,
+		Hashers:    2,
 	})
 	var files []protocol.FileInfo
 	for f := range fchan {
@@ -90,7 +89,7 @@ func TestWalkSub(t *testing.T) {
 }
 
 func TestWalk(t *testing.T) {
-	ignores := ignore.New()
+	ignores := ignore.New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."))
 	err := ignores.Load("testdata/.stignore")
 	if err != nil {
 		t.Fatal(err)
@@ -98,10 +97,10 @@ func TestWalk(t *testing.T) {
 	t.Log(ignores)
 
 	fchan, err := Walk(context.TODO(), Config{
-		Dir:       "testdata",
+		Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
-		BlockSize: 128 * 1024,
+		BlockSize:  128 * 1024,
-		Matcher:   ignores,
+		Matcher:    ignores,
-		Hashers:   2,
+		Hashers:    2,
 	})
 
 	if err != nil {
@@ -122,9 +121,9 @@ func TestWalk(t *testing.T) {
 
 func TestWalkError(t *testing.T) {
 	_, err := Walk(context.TODO(), Config{
-		Dir:       "testdata-missing",
+		Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata-missing"),
-		BlockSize: 128 * 1024,
+		BlockSize:  128 * 1024,
-		Hashers:   2,
+		Hashers:    2,
 	})
 
 	if err == nil {
@@ -132,8 +131,8 @@ func TestWalkError(t *testing.T) {
 	}
 
 	_, err = Walk(context.TODO(), Config{
-		Dir:       "testdata/bar",
+		Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata/bar"),
-		BlockSize: 128 * 1024,
+		BlockSize:  128 * 1024,
 	})
 
 	if err == nil {
@@ -220,9 +219,11 @@ func TestNormalization(t *testing.T) {
 
 	numValid := len(tests) - numInvalid
 
+	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, ".")
+
 	for _, s1 := range tests {
 		// Create a directory for each of the interesting strings above
-		if err := osutil.MkdirAll(filepath.Join("testdata/normalization", s1), 0755); err != nil {
+		if err := fs.MkdirAll(filepath.Join("testdata/normalization", s1), 0755); err != nil {
 			t.Fatal(err)
 		}
 
@@ -231,10 +232,10 @@ func TestNormalization(t *testing.T) {
 			// file names. Ensure that the file doesn't exist when it's
 			// created. This detects and fails if there's file name
 			// normalization stuff at the filesystem level.
-			if fd, err := os.OpenFile(filepath.Join("testdata/normalization", s1, s2), os.O_CREATE|os.O_EXCL, 0644); err != nil {
+			if fd, err := fs.OpenFile(filepath.Join("testdata/normalization", s1, s2), os.O_CREATE|os.O_EXCL, 0644); err != nil {
 				t.Fatal(err)
 			} else {
-				fd.WriteString("test")
+				fd.Write([]byte("test"))
 				fd.Close()
 			}
 		}
@@ -245,11 +246,11 @@ func TestNormalization(t *testing.T) {
 	// make sure it all gets done. In production, things will be correct
 	// eventually...
 
-	_, err := walkDir("testdata/normalization")
+	_, err := walkDir(fs, "testdata/normalization")
 	if err != nil {
 		t.Fatal(err)
 	}
-	tmp, err := walkDir("testdata/normalization")
+	tmp, err := walkDir(fs, "testdata/normalization")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -299,8 +300,8 @@ func TestWalkSymlinkUnix(t *testing.T) {
 	// Scan it
 
 	fchan, err := Walk(context.TODO(), Config{
-		Dir:       "_symlinks",
+		Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"),
-		BlockSize: 128 * 1024,
+		BlockSize:  128 * 1024,
 	})
 
 	if err != nil {
@@ -344,8 +345,8 @@ func TestWalkSymlinkWindows(t *testing.T) {
 	// Scan it
 
 	fchan, err := Walk(context.TODO(), Config{
-		Dir:       "_symlinks",
+		Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"),
-		BlockSize: 128 * 1024,
+		BlockSize:  128 * 1024,
 	})
 
 	if err != nil {
@@ -364,9 +365,10 @@ func TestWalkSymlinkWindows(t *testing.T) {
 	}
 }
 
-func walkDir(dir string) ([]protocol.FileInfo, error) {
+func walkDir(fs fs.Filesystem, dir string) ([]protocol.FileInfo, error) {
 	fchan, err := Walk(context.TODO(), Config{
-		Dir:           dir,
+		Filesystem:    fs,
+		Subs:          []string{dir},
 		BlockSize:     128 * 1024,
 		AutoNormalize: true,
 		Hashers:       2,
@@ -435,7 +437,7 @@ func BenchmarkHashFile(b *testing.B) {
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
-		if _, err := HashFile(context.TODO(), fs.DefaultFilesystem, testdataName, protocol.BlockSize, nil, true); err != nil {
+		if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.BlockSize, nil, true); err != nil {
 			b.Fatal(err)
 		}
 	}
@@ -467,15 +469,17 @@ func TestStopWalk(t *testing.T) {
 	// many directories. It'll take a while to scan, giving us time to
 	// cancel it and make sure the scan stops.
 
-	fs := fs.NewWalkFilesystem(&infiniteFS{100, 100, 1e6})
+	// Use an errorFs as the backing fs for the rest of the interface
+	// The way we get it is a bit hacky tho.
+	errorFs := fs.NewFilesystem(fs.FilesystemType(-1), ".")
+	fs := fs.NewWalkFilesystem(&infiniteFS{errorFs, 100, 100, 1e6})
 
 	const numHashers = 4
 	ctx, cancel := context.WithCancel(context.Background())
 	fchan, err := Walk(ctx, Config{
-		Dir:                   "testdir",
+		Filesystem:            fs,
 		BlockSize:             128 * 1024,
 		Hashers:               numHashers,
-		Filesystem:            fs,
 		ProgressTickIntervalS: -1, // Don't attempt to build the full list of files before starting to scan...
 	})
 

lib/versioner/external.go

@@ -10,10 +10,11 @@ import (
 	"errors"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"strings"
 
-	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/fs"
+
+	"github.com/kballard/go-shellquote"
 )
 
 func init() {
@@ -23,15 +24,15 @@ func init() {
 
 type External struct {
 	command    string
-	folderPath string
+	filesystem fs.Filesystem
 }
 
-func NewExternal(folderID, folderPath string, params map[string]string) Versioner {
+func NewExternal(folderID string, filesystem fs.Filesystem, params map[string]string) Versioner {
 	command := params["command"]
 
 	s := External{
 		command:    command,
-		folderPath: folderPath,
+		filesystem: filesystem,
 	}
 
 	l.Debugf("instantiated %#v", s)
@@ -41,29 +42,41 @@ func NewExternal(folderID, folderPath string, params map[string]string) Versione
 // Archive moves the named file away to a version archive. If this function
 // returns nil, the named file does not exist any more (has been archived).
 func (v External) Archive(filePath string) error {
-	info, err := osutil.Lstat(filePath)
+	info, err := v.filesystem.Lstat(filePath)
-	if os.IsNotExist(err) {
+	if fs.IsNotExist(err) {
 		l.Debugln("not archiving nonexistent file", filePath)
 		return nil
 	} else if err != nil {
 		return err
 	}
-	if info.Mode()&os.ModeSymlink != 0 {
+	if info.IsSymlink() {
 		panic("bug: attempting to version a symlink")
 	}
 
 	l.Debugln("archiving", filePath)
 
-	inFolderPath, err := filepath.Rel(v.folderPath, filePath)
-	if err != nil {
-		return err
-	}
-
 	if v.command == "" {
 		return errors.New("Versioner: command is empty, please enter a valid command")
 	}
 
-	cmd := exec.Command(v.command, v.folderPath, inFolderPath)
+	words, err := shellquote.Split(v.command)
+	if err != nil {
+		return errors.New("Versioner: command is invalid: " + err.Error())
+	}
+
+	context := map[string]string{
+		"%FOLDER_FILESYSTEM%": v.filesystem.Type().String(),
+		"%FOLDER_PATH%":       v.filesystem.URI(),
+		"%FILE_PATH%":         filePath,
+	}
+
+	for i, word := range words {
+		if replacement, ok := context[word]; ok {
+			words[i] = replacement
+		}
+	}
+
+	cmd := exec.Command(words[0], words[1:]...)
 	env := os.Environ()
 	// filter STGUIAUTH and STGUIAPIKEY from environment variables
 	filteredEnv := []string{}
@@ -73,13 +86,14 @@ func (v External) Archive(filePath string) error {
 		}
 	}
 	cmd.Env = filteredEnv
-	err = cmd.Run()
+	combinedOutput, err := cmd.CombinedOutput()
+	l.Debugln("external command output:", string(combinedOutput))
 	if err != nil {
 		return err
 	}
 
 	// return error if the file was not removed
-	if _, err = osutil.Lstat(filePath); os.IsNotExist(err) {
+	if _, err = v.filesystem.Lstat(filePath); fs.IsNotExist(err) {
 		return nil
 	}
 	return errors.New("Versioner: file was not removed by external script")

lib/versioner/external_test.go

@@ -12,6 +12,8 @@ import (
 	"path/filepath"
 	"runtime"
 	"testing"
+
+	"github.com/syncthing/syncthing/lib/fs"
 )
 
 func TestExternalNoCommand(t *testing.T) {
@@ -28,8 +30,8 @@ func TestExternalNoCommand(t *testing.T) {
 	// The versioner should fail due to missing command.
 
 	e := External{
+		filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "."),
 		command:    "nonexistent command",
-		folderPath: "testdata/folder path",
 	}
 	if err := e.Archive(file); err == nil {
 		t.Error("Command should have failed")
@@ -43,12 +45,12 @@ func TestExternalNoCommand(t *testing.T) {
 }
 
 func TestExternal(t *testing.T) {
-	cmd := "./_external_test/external.sh"
+	cmd := "./_external_test/external.sh %FOLDER_PATH% %FILE_PATH%"
 	if runtime.GOOS == "windows" {
-		cmd = `.\_external_test\external.bat`
+		cmd = `.\\_external_test\\external.bat %FOLDER_PATH% %FILE_PATH%`
 	}
 
-	file := "testdata/folder path/dir (parens)/long filename (parens).txt"
+	file := filepath.Join("testdata", "folder path", "dir (parens)", "/long filename (parens).txt")
 	prepForRemoval(t, file)
 	defer os.RemoveAll("testdata")
 
@@ -61,8 +63,8 @@ func TestExternal(t *testing.T) {
 	// The versioner should run successfully.
 
 	e := External{
+		filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "."),
 		command:    cmd,
-		folderPath: "testdata/folder path",
 	}
 	if err := e.Archive(file); err != nil {
 		t.Fatal(err)

lib/versioner/simple.go

@@ -7,10 +7,10 @@
 package versioner
 
 import (
-	"os"
 	"path/filepath"
 	"strconv"
 
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/util"
 )
@@ -21,19 +21,19 @@ func init() {
 }
 
 type Simple struct {
-	keep       int
+	keep int
-	folderPath string
+	fs   fs.Filesystem
 }
 
-func NewSimple(folderID, folderPath string, params map[string]string) Versioner {
+func NewSimple(folderID string, fs fs.Filesystem, params map[string]string) Versioner {
 	keep, err := strconv.Atoi(params["keep"])
 	if err != nil {
 		keep = 5 // A reasonable default
 	}
 
 	s := Simple{
-		keep:       keep,
+		keep: keep,
-		folderPath: folderPath,
+		fs:   fs,
 	}
 
 	l.Debugf("instantiated %#v", s)
@@ -43,24 +43,24 @@ func NewSimple(folderID, folderPath string, params map[string]string) Versioner
 // Archive moves the named file away to a version archive. If this function
 // returns nil, the named file does not exist any more (has been archived).
 func (v Simple) Archive(filePath string) error {
-	fileInfo, err := osutil.Lstat(filePath)
+	info, err := v.fs.Lstat(filePath)
-	if os.IsNotExist(err) {
+	if fs.IsNotExist(err) {
 		l.Debugln("not archiving nonexistent file", filePath)
 		return nil
 	} else if err != nil {
 		return err
 	}
-	if fileInfo.Mode()&os.ModeSymlink != 0 {
+	if info.IsSymlink() {
 		panic("bug: attempting to version a symlink")
 	}
 
-	versionsDir := filepath.Join(v.folderPath, ".stversions")
+	versionsDir := ".stversions"
-	_, err = os.Stat(versionsDir)
+	_, err = v.fs.Stat(versionsDir)
 	if err != nil {
-		if os.IsNotExist(err) {
+		if fs.IsNotExist(err) {
-			l.Debugln("creating versions dir", versionsDir)
+			l.Debugln("creating versions dir .stversions")
-			osutil.MkdirAll(versionsDir, 0755)
+			v.fs.Mkdir(versionsDir, 0755)
-			osutil.HideFile(versionsDir)
+			v.fs.Hide(versionsDir)
 		} else {
 			return err
 		}
@@ -69,28 +69,25 @@ func (v Simple) Archive(filePath string) error {
 	l.Debugln("archiving", filePath)
 
 	file := filepath.Base(filePath)
-	inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath))
+	inFolderPath := filepath.Dir(filePath)
-	if err != nil {
-		return err
-	}
 
 	dir := filepath.Join(versionsDir, inFolderPath)
-	err = osutil.MkdirAll(dir, 0755)
+	err = v.fs.MkdirAll(dir, 0755)
-	if err != nil && !os.IsExist(err) {
+	if err != nil && !fs.IsExist(err) {
 		return err
 	}
 
-	ver := taggedFilename(file, fileInfo.ModTime().Format(TimeFormat))
+	ver := taggedFilename(file, info.ModTime().Format(TimeFormat))
 	dst := filepath.Join(dir, ver)
 	l.Debugln("moving to", dst)
-	err = osutil.Rename(filePath, dst)
+	err = osutil.Rename(v.fs, filePath, dst)
 	if err != nil {
 		return err
 	}
 
 	// Glob according to the new file~timestamp.ext pattern.
 	pattern := filepath.Join(dir, taggedFilename(file, TimeGlob))
-	newVersions, err := osutil.Glob(pattern)
+	newVersions, err := v.fs.Glob(pattern)
 	if err != nil {
 		l.Warnln("globbing:", err, "for", pattern)
 		return nil
@@ -98,7 +95,7 @@ func (v Simple) Archive(filePath string) error {
 
 	// Also according to the old file.ext~timestamp pattern.
 	pattern = filepath.Join(dir, file+"~"+TimeGlob)
-	oldVersions, err := osutil.Glob(pattern)
+	oldVersions, err := v.fs.Glob(pattern)
 	if err != nil {
 		l.Warnln("globbing:", err, "for", pattern)
 		return nil
@@ -111,7 +108,7 @@ func (v Simple) Archive(filePath string) error {
 	if len(versions) > v.keep {
 		for _, toRemove := range versions[:len(versions)-v.keep] {
 			l.Debugln("cleaning out", toRemove)
-			err = os.Remove(toRemove)
+			err = v.fs.Remove(toRemove)
 			if err != nil {
 				l.Warnln("removing old version:", err)
 			}

lib/versioner/simple_test.go

@@ -9,10 +9,11 @@ package versioner
 import (
 	"io/ioutil"
 	"math"
-	"os"
 	"path/filepath"
 	"testing"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/fs"
 )
 
 func TestTaggedFilename(t *testing.T) {
@@ -53,29 +54,28 @@ func TestSimpleVersioningVersionCount(t *testing.T) {
 	}
 
 	dir, err := ioutil.TempDir("", "")
-	defer os.RemoveAll(dir)
+	//defer os.RemoveAll(dir)
 	if err != nil {
 		t.Error(err)
 	}
 
-	v := NewSimple("", dir, map[string]string{"keep": "2"})
+	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
-	versionDir := filepath.Join(dir, ".stversions")
 
-	path := filepath.Join(dir, "test")
+	v := NewSimple("", fs, map[string]string{"keep": "2"})
+
+	path := "test"
 
 	for i := 1; i <= 3; i++ {
-		f, err := os.Create(path)
+		f, err := fs.Create(path)
 		if err != nil {
 			t.Error(err)
 		}
 		f.Close()
-		v.Archive(path)
+		if err := v.Archive(path); err != nil {
-
-		d, err := os.Open(versionDir)
-		if err != nil {
 			t.Error(err)
 		}
-		n, err := d.Readdirnames(-1)
+
+		n, err := fs.DirNames(".stversions")
 		if err != nil {
 			t.Error(err)
 		}
@@ -83,7 +83,6 @@ func TestSimpleVersioningVersionCount(t *testing.T) {
 		if float64(len(n)) != math.Min(float64(i), 2) {
 			t.Error("Wrong count")
 		}
-		d.Close()
 
 		time.Sleep(time.Second)
 	}

lib/versioner/staggered.go

@@ -12,7 +12,7 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/util"
 )
@@ -28,9 +28,9 @@ type Interval struct {
 }
 
 type Staggered struct {
-	versionsPath  string
 	cleanInterval int64
-	folderPath    string
+	folderFs      fs.Filesystem
+	versionsFs    fs.Filesystem
 	interval      [4]Interval
 	mutex         sync.Mutex
 
@@ -38,7 +38,7 @@ type Staggered struct {
 	testCleanDone chan struct{}
 }
 
-func NewStaggered(folderID, folderPath string, params map[string]string) Versioner {
+func NewStaggered(folderID string, folderFs fs.Filesystem, params map[string]string) Versioner {
 	maxAge, err := strconv.ParseInt(params["maxAge"], 10, 0)
 	if err != nil {
 		maxAge = 31536000 // Default: ~1 year
@@ -49,22 +49,20 @@ func NewStaggered(folderID, folderPath string, params map[string]string) Version
 	}
 
 	// Use custom path if set, otherwise .stversions in folderPath
-	var versionsDir string
+	var versionsFs fs.Filesystem
 	if params["versionsPath"] == "" {
-		versionsDir = filepath.Join(folderPath, ".stversions")
+		versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), ".stversions"))
-		l.Debugln("using default dir .stversions")
 	} else if filepath.IsAbs(params["versionsPath"]) {
-		l.Debugln("using dir", params["versionsPath"])
+		versionsFs = fs.NewFilesystem(folderFs.Type(), params["versionsPath"])
-		versionsDir = params["versionsPath"]
 	} else {
-		versionsDir = filepath.Join(folderPath, params["versionsPath"])
+		versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), params["versionsPath"]))
-		l.Debugln("using dir", versionsDir)
 	}
+	l.Debugln("%s folder using %s (%s) staggered versioner dir", folderID, versionsFs.URI(), versionsFs.Type())
 
 	s := &Staggered{
-		versionsPath:  versionsDir,
 		cleanInterval: cleanInterval,
-		folderPath:    folderPath,
+		folderFs:      folderFs,
+		versionsFs:    versionsFs,
 		interval: [4]Interval{
 			{30, 3600},       // first hour -> 30 sec between versions
 			{3600, 86400},    // next day -> 1 h between versions
@@ -102,12 +100,12 @@ func (v *Staggered) Stop() {
 }
 
 func (v *Staggered) clean() {
-	l.Debugln("Versioner clean: Waiting for lock on", v.versionsPath)
+	l.Debugln("Versioner clean: Waiting for lock on", v.versionsFs)
 	v.mutex.Lock()
 	defer v.mutex.Unlock()
-	l.Debugln("Versioner clean: Cleaning", v.versionsPath)
+	l.Debugln("Versioner clean: Cleaning", v.versionsFs)
 
-	if _, err := os.Stat(v.versionsPath); os.IsNotExist(err) {
+	if _, err := v.versionsFs.Stat("."); fs.IsNotExist(err) {
 		// There is no need to clean a nonexistent dir.
 		return
 	}
@@ -115,14 +113,14 @@ func (v *Staggered) clean() {
 	versionsPerFile := make(map[string][]string)
 	filesPerDir := make(map[string]int)
 
-	err := filepath.Walk(v.versionsPath, func(path string, f os.FileInfo, err error) error {
+	err := v.versionsFs.Walk(".", func(path string, f fs.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
-		if f.Mode().IsDir() && f.Mode()&os.ModeSymlink == 0 {
+		if f.IsDir() && !f.IsSymlink() {
 			filesPerDir[path] = 0
-			if path != v.versionsPath {
+			if path != "." {
 				dir := filepath.Dir(path)
 				filesPerDir[dir]++
 			}
@@ -155,25 +153,20 @@ func (v *Staggered) clean() {
 			continue
 		}
 
-		if path == v.versionsPath {
-			l.Debugln("Cleaner: versions dir is empty, don't delete", path)
-			continue
-		}
-
 		l.Debugln("Cleaner: deleting empty directory", path)
-		err = os.Remove(path)
+		err = v.versionsFs.Remove(path)
 		if err != nil {
 			l.Warnln("Versioner: can't remove directory", path, err)
 		}
 	}
 
-	l.Debugln("Cleaner: Finished cleaning", v.versionsPath)
+	l.Debugln("Cleaner: Finished cleaning", v.versionsFs)
 }
 
 func (v *Staggered) expire(versions []string) {
 	l.Debugln("Versioner: Expiring versions", versions)
 	for _, file := range v.toRemove(versions, time.Now()) {
-		if fi, err := osutil.Lstat(file); err != nil {
+		if fi, err := v.versionsFs.Lstat(file); err != nil {
 			l.Warnln("versioner:", err)
 			continue
 		} else if fi.IsDir() {
@@ -181,7 +174,7 @@ func (v *Staggered) expire(versions []string) {
 			continue
 		}
 
-		if err := os.Remove(file); err != nil {
+		if err := v.versionsFs.Remove(file); err != nil {
 			l.Warnf("Versioner: can't remove %q: %v", file, err)
 		}
 	}
@@ -203,7 +196,7 @@ func (v *Staggered) toRemove(versions []string, now time.Time) []string {
 		// If the file is older than the max age of the last interval, remove it
 		if lastIntv := v.interval[len(v.interval)-1]; lastIntv.end > 0 && age > lastIntv.end {
 			l.Debugln("Versioner: File over maximum age -> delete ", file)
-			err = os.Remove(file)
+			err = v.versionsFs.Remove(file)
 			if err != nil {
 				l.Warnf("Versioner: can't remove %q: %v", file, err)
 			}
@@ -240,26 +233,26 @@ func (v *Staggered) toRemove(versions []string, now time.Time) []string {
 // Archive moves the named file away to a version archive. If this function
 // returns nil, the named file does not exist any more (has been archived).
 func (v *Staggered) Archive(filePath string) error {
-	l.Debugln("Waiting for lock on ", v.versionsPath)
+	l.Debugln("Waiting for lock on ", v.versionsFs)
 	v.mutex.Lock()
 	defer v.mutex.Unlock()
 
-	info, err := osutil.Lstat(filePath)
+	info, err := v.folderFs.Lstat(filePath)
-	if os.IsNotExist(err) {
+	if fs.IsNotExist(err) {
 		l.Debugln("not archiving nonexistent file", filePath)
 		return nil
 	} else if err != nil {
 		return err
 	}
-	if info.Mode()&os.ModeSymlink != 0 {
+	if info.IsSymlink() {
 		panic("bug: attempting to version a symlink")
 	}
 
-	if _, err := os.Stat(v.versionsPath); err != nil {
+	if _, err := v.versionsFs.Stat("."); err != nil {
-		if os.IsNotExist(err) {
+		if fs.IsNotExist(err) {
-			l.Debugln("creating versions dir", v.versionsPath)
+			l.Debugln("creating versions dir", v.versionsFs)
-			osutil.MkdirAll(v.versionsPath, 0755)
+			v.versionsFs.MkdirAll(".", 0755)
-			osutil.HideFile(v.versionsPath)
+			v.versionsFs.Hide(".")
 		} else {
 			return err
 		}
@@ -268,36 +261,41 @@ func (v *Staggered) Archive(filePath string) error {
 	l.Debugln("archiving", filePath)
 
 	file := filepath.Base(filePath)
-	inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath))
+	inFolderPath := filepath.Dir(filePath)
 	if err != nil {
 		return err
 	}
 
-	dir := filepath.Join(v.versionsPath, inFolderPath)
+	err = v.versionsFs.MkdirAll(inFolderPath, 0755)
-	err = osutil.MkdirAll(dir, 0755)
+	if err != nil && !fs.IsExist(err) {
-	if err != nil && !os.IsExist(err) {
 		return err
 	}
 
 	ver := taggedFilename(file, time.Now().Format(TimeFormat))
-	dst := filepath.Join(dir, ver)
+	dst := filepath.Join(inFolderPath, ver)
 	l.Debugln("moving to", dst)
-	err = osutil.Rename(filePath, dst)
+
+	/// TODO: Fix this when we have an alternative filesystem implementation
+	if v.versionsFs.Type() != fs.FilesystemTypeBasic {
+		panic("bug: staggered versioner used with unsupported filesystem")
+	}
+
+	err = os.Rename(filepath.Join(v.folderFs.URI(), filePath), filepath.Join(v.versionsFs.URI(), dst))
 	if err != nil {
 		return err
 	}
 
 	// Glob according to the new file~timestamp.ext pattern.
-	pattern := filepath.Join(dir, taggedFilename(file, TimeGlob))
+	pattern := filepath.Join(inFolderPath, taggedFilename(file, TimeGlob))
-	newVersions, err := osutil.Glob(pattern)
+	newVersions, err := v.versionsFs.Glob(pattern)
 	if err != nil {
 		l.Warnln("globbing:", err, "for", pattern)
 		return nil
 	}
 
 	// Also according to the old file.ext~timestamp pattern.
-	pattern = filepath.Join(dir, file+"~"+TimeGlob)
+	pattern = filepath.Join(inFolderPath, file+"~"+TimeGlob)
-	oldVersions, err := osutil.Glob(pattern)
+	oldVersions, err := v.versionsFs.Glob(pattern)
 	if err != nil {
 		l.Warnln("globbing:", err, "for", pattern)
 		return nil

lib/versioner/staggered_test.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/d4l3k/messagediff"
+	"github.com/syncthing/syncthing/lib/fs"
 )
 
 func TestStaggeredVersioningVersionCount(t *testing.T) {
@@ -62,7 +63,7 @@ func TestStaggeredVersioningVersionCount(t *testing.T) {
 	os.MkdirAll("testdata/.stversions", 0755)
 	defer os.RemoveAll("testdata")
 
-	v := NewStaggered("", "testdata", map[string]string{"maxAge": strconv.Itoa(365 * 86400)}).(*Staggered)
+	v := NewStaggered("", fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), map[string]string{"maxAge": strconv.Itoa(365 * 86400)}).(*Staggered)
 	v.testCleanDone = make(chan struct{})
 	defer v.Stop()
 	go v.Serve()

lib/versioner/trashcan.go

@@ -8,11 +8,11 @@ package versioner
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 	"strconv"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/osutil"
 )
 
@@ -22,17 +22,17 @@ func init() {
 }
 
 type Trashcan struct {
-	folderPath   string
+	fs           fs.Filesystem
 	cleanoutDays int
 	stop         chan struct{}
 }
 
-func NewTrashcan(folderID, folderPath string, params map[string]string) Versioner {
+func NewTrashcan(folderID string, fs fs.Filesystem, params map[string]string) Versioner {
 	cleanoutDays, _ := strconv.Atoi(params["cleanoutDays"])
 	// On error we default to 0, "do not clean out the trash can"
 
 	s := &Trashcan{
-		folderPath:   folderPath,
+		fs:           fs,
 		cleanoutDays: cleanoutDays,
 		stop:         make(chan struct{}),
 	}
@@ -44,52 +44,47 @@ func NewTrashcan(folderID, folderPath string, params map[string]string) Versione
 // Archive moves the named file away to a version archive. If this function
 // returns nil, the named file does not exist any more (has been archived).
 func (t *Trashcan) Archive(filePath string) error {
-	info, err := osutil.Lstat(filePath)
+	info, err := t.fs.Lstat(filePath)
-	if os.IsNotExist(err) {
+	if fs.IsNotExist(err) {
 		l.Debugln("not archiving nonexistent file", filePath)
 		return nil
 	} else if err != nil {
 		return err
 	}
-	if info.Mode()&os.ModeSymlink != 0 {
+	if info.IsSymlink() {
 		panic("bug: attempting to version a symlink")
 	}
 
-	versionsDir := filepath.Join(t.folderPath, ".stversions")
+	versionsDir := ".stversions"
-	if _, err := os.Stat(versionsDir); err != nil {
+	if _, err := t.fs.Stat(versionsDir); err != nil {
-		if !os.IsNotExist(err) {
+		if !fs.IsNotExist(err) {
 			return err
 		}
 
 		l.Debugln("creating versions dir", versionsDir)
-		if err := osutil.MkdirAll(versionsDir, 0777); err != nil {
+		if err := t.fs.MkdirAll(versionsDir, 0777); err != nil {
 			return err
 		}
-		osutil.HideFile(versionsDir)
+		t.fs.Hide(versionsDir)
 	}
 
 	l.Debugln("archiving", filePath)
 
-	relativePath, err := filepath.Rel(t.folderPath, filePath)
+	archivedPath := filepath.Join(versionsDir, filePath)
-	if err != nil {
+	if err := t.fs.MkdirAll(filepath.Dir(archivedPath), 0777); err != nil && !fs.IsExist(err) {
-		return err
-	}
-
-	archivedPath := filepath.Join(versionsDir, relativePath)
-	if err := osutil.MkdirAll(filepath.Dir(archivedPath), 0777); err != nil && !os.IsExist(err) {
 		return err
 	}
 
 	l.Debugln("moving to", archivedPath)
 
-	if err := osutil.Rename(filePath, archivedPath); err != nil {
+	if err := osutil.Rename(t.fs, filePath, archivedPath); err != nil {
 		return err
 	}
 
 	// Set the mtime to the time the file was deleted. This is used by the
 	// cleanout routine. If this fails things won't work optimally but there's
 	// not much we can do about it so we ignore the error.
-	os.Chtimes(archivedPath, time.Now(), time.Now())
+	t.fs.Chtimes(archivedPath, time.Now(), time.Now())
 
 	return nil
 }
@@ -129,15 +124,15 @@ func (t *Trashcan) String() string {
 }
 
 func (t *Trashcan) cleanoutArchive() error {
-	versionsDir := filepath.Join(t.folderPath, ".stversions")
+	versionsDir := ".stversions"
-	if _, err := osutil.Lstat(versionsDir); os.IsNotExist(err) {
+	if _, err := t.fs.Lstat(versionsDir); fs.IsNotExist(err) {
 		return nil
 	}
 
 	cutoff := time.Now().Add(time.Duration(-24*t.cleanoutDays) * time.Hour)
 	currentDir := ""
 	filesInDir := 0
-	walkFn := func(path string, info os.FileInfo, err error) error {
+	walkFn := func(path string, info fs.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -147,7 +142,7 @@ func (t *Trashcan) cleanoutArchive() error {
 			// directory was empty and try to remove it. We ignore failure for
 			// the time being.
 			if currentDir != "" && filesInDir == 0 {
-				os.Remove(currentDir)
+				t.fs.Remove(currentDir)
 			}
 			currentDir = path
 			filesInDir = 0
@@ -156,7 +151,7 @@ func (t *Trashcan) cleanoutArchive() error {
 
 		if info.ModTime().Before(cutoff) {
 			// The file is too old; remove it.
-			os.Remove(path)
+			t.fs.Remove(path)
 		} else {
 			// Keep this file, and remember it so we don't unnecessarily try
 			// to remove this directory.
@@ -165,14 +160,14 @@ func (t *Trashcan) cleanoutArchive() error {
 		return nil
 	}
 
-	if err := filepath.Walk(versionsDir, walkFn); err != nil {
+	if err := t.fs.Walk(versionsDir, walkFn); err != nil {
 		return err
 	}
 
 	// The last directory seen by the walkFn may not have been removed as it
 	// should be.
 	if currentDir != "" && filesInDir == 0 {
-		os.Remove(currentDir)
+		t.fs.Remove(currentDir)
 	}
 	return nil
 }

lib/versioner/trashcan_test.go

@@ -12,6 +12,8 @@ import (
 	"path/filepath"
 	"testing"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/fs"
 )
 
 func TestTrashcanCleanout(t *testing.T) {
@@ -49,7 +51,7 @@ func TestTrashcanCleanout(t *testing.T) {
 		}
 	}
 
-	versioner := NewTrashcan("default", "testdata", map[string]string{"cleanoutDays": "7"}).(*Trashcan)
+	versioner := NewTrashcan("default", fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), map[string]string{"cleanoutDays": "7"}).(*Trashcan)
 	if err := versioner.cleanoutArchive(); err != nil {
 		t.Fatal(err)
 	}

lib/versioner/versioner.go

@@ -8,11 +8,13 @@
 // simple default versioning scheme.
 package versioner
 
+import "github.com/syncthing/syncthing/lib/fs"
+
 type Versioner interface {
 	Archive(filePath string) error
 }
 
-var Factories = map[string]func(folderID string, folderDir string, params map[string]string) Versioner{}
+var Factories = map[string]func(folderID string, filesystem fs.Filesystem, params map[string]string) Versioner{}
 
 const (
 	TimeFormat = "20060102-150405"

lib/weakhash/weakhash.go

@@ -9,7 +9,6 @@ package weakhash
 import (
 	"bufio"
 	"io"
-	"os"
 
 	"github.com/chmduquesne/rollinghash/adler32"
 )
@@ -72,27 +71,21 @@ func Find(ir io.Reader, hashesToFind []uint32, size int) (map[uint32][]int64, er
 	return offsets, nil
 }
 
-func NewFinder(path string, size int, hashesToFind []uint32) (*Finder, error) {
+func NewFinder(ir io.ReadSeeker, size int, hashesToFind []uint32) (*Finder, error) {
-	file, err := os.Open(path)
+	offsets, err := Find(ir, hashesToFind, size)
 	if err != nil {
 		return nil, err
 	}
 
-	offsets, err := Find(file, hashesToFind, size)
-	if err != nil {
-		file.Close()
-		return nil, err
-	}
-
 	return &Finder{
-		file:    file,
+		reader:  ir,
 		size:    size,
 		offsets: offsets,
 	}, nil
 }
 
 type Finder struct {
-	file    *os.File
+	reader  io.ReadSeeker
 	size    int
 	offsets map[uint32][]int64
 }
@@ -106,7 +99,11 @@ func (h *Finder) Iterate(hash uint32, buf []byte, iterFunc func(int64) bool) (bo
 	}
 
 	for _, offset := range h.offsets[hash] {
-		_, err := h.file.ReadAt(buf, offset)
+		_, err := h.reader.Seek(offset, io.SeekStart)
+		if err != nil {
+			return false, err
+		}
+		_, err = h.reader.Read(buf)
 		if err != nil {
 			return false, err
 		}
@@ -116,10 +113,3 @@ func (h *Finder) Iterate(hash uint32, buf []byte, iterFunc func(int64) bool) (bo
 	}
 	return false, nil
 }
-
-// Close releases any resource associated with the finder
-func (h *Finder) Close() {
-	if h != nil {
-		h.file.Close()
-	}
-}

lib/weakhash/weakhash_test.go

@@ -11,6 +11,7 @@ package weakhash
 
 import (
 	"bytes"
+	"io"
 	"io/ioutil"
 	"os"
 	"reflect"
@@ -30,13 +31,15 @@ func TestFinder(t *testing.T) {
 	if _, err := f.Write(payload); err != nil {
 		t.Error(err)
 	}
+	if _, err := f.Seek(0, io.SeekStart); err != nil {
+		t.Error(err)
+	}
 
 	hashes := []uint32{65143183, 65798547}
-	finder, err := NewFinder(f.Name(), 4, hashes)
+	finder, err := NewFinder(f, 4, hashes)
 	if err != nil {
 		t.Error(err)
 	}
-	defer finder.Close()
 
 	expected := map[uint32][]int64{
 		65143183: {1, 27, 53, 79},

vendor/github.com/kballard/go-shellquote/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2014 Kevin Ballard
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
+OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/kballard/go-shellquote/doc.go

@@ -0,0 +1,3 @@
+// Shellquote provides utilities for joining/splitting strings using sh's
+// word-splitting rules.
+package shellquote

vendor/github.com/kballard/go-shellquote/quote.go

@@ -0,0 +1,102 @@
+package shellquote
+
+import (
+	"bytes"
+	"strings"
+	"unicode/utf8"
+)
+
+// Join quotes each argument and joins them with a space.
+// If passed to /bin/sh, the resulting string will be split back into the
+// original arguments.
+func Join(args ...string) string {
+	var buf bytes.Buffer
+	for i, arg := range args {
+		if i != 0 {
+			buf.WriteByte(' ')
+		}
+		quote(arg, &buf)
+	}
+	return buf.String()
+}
+
+const (
+	specialChars      = "\\'\"`${[|&;<>()*?!"
+	extraSpecialChars = " \t\n"
+	prefixChars       = "~"
+)
+
+func quote(word string, buf *bytes.Buffer) {
+	// We want to try to produce a "nice" output. As such, we will
+	// backslash-escape most characters, but if we encounter a space, or if we
+	// encounter an extra-special char (which doesn't work with
+	// backslash-escaping) we switch over to quoting the whole word. We do this
+	// with a space because it's typically easier for people to read multi-word
+	// arguments when quoted with a space rather than with ugly backslashes
+	// everywhere.
+	origLen := buf.Len()
+
+	if len(word) == 0 {
+		// oops, no content
+		buf.WriteString("''")
+		return
+	}
+
+	cur, prev := word, word
+	atStart := true
+	for len(cur) > 0 {
+		c, l := utf8.DecodeRuneInString(cur)
+		cur = cur[l:]
+		if strings.ContainsRune(specialChars, c) || (atStart && strings.ContainsRune(prefixChars, c)) {
+			// copy the non-special chars up to this point
+			if len(cur) < len(prev) {
+				buf.WriteString(prev[0 : len(prev)-len(cur)-l])
+			}
+			buf.WriteByte('\\')
+			buf.WriteRune(c)
+			prev = cur
+		} else if strings.ContainsRune(extraSpecialChars, c) {
+			// start over in quote mode
+			buf.Truncate(origLen)
+			goto quote
+		}
+		atStart = false
+	}
+	if len(prev) > 0 {
+		buf.WriteString(prev)
+	}
+	return
+
+quote:
+	// quote mode
+	// Use single-quotes, but if we find a single-quote in the word, we need
+	// to terminate the string, emit an escaped quote, and start the string up
+	// again
+	inQuote := false
+	for len(word) > 0 {
+		i := strings.IndexRune(word, '\'')
+		if i == -1 {
+			break
+		}
+		if i > 0 {
+			if !inQuote {
+				buf.WriteByte('\'')
+				inQuote = true
+			}
+			buf.WriteString(word[0:i])
+		}
+		word = word[i+1:]
+		if inQuote {
+			buf.WriteByte('\'')
+			inQuote = false
+		}
+		buf.WriteString("\\'")
+	}
+	if len(word) > 0 {
+		if !inQuote {
+			buf.WriteByte('\'')
+		}
+		buf.WriteString(word)
+		buf.WriteByte('\'')
+	}
+}

vendor/github.com/kballard/go-shellquote/unquote.go

@@ -0,0 +1,144 @@
+package shellquote
+
+import (
+	"bytes"
+	"errors"
+	"strings"
+	"unicode/utf8"
+)
+
+var (
+	UnterminatedSingleQuoteError = errors.New("Unterminated single-quoted string")
+	UnterminatedDoubleQuoteError = errors.New("Unterminated double-quoted string")
+	UnterminatedEscapeError      = errors.New("Unterminated backslash-escape")
+)
+
+var (
+	splitChars        = " \n\t"
+	singleChar        = '\''
+	doubleChar        = '"'
+	escapeChar        = '\\'
+	doubleEscapeChars = "$`\"\n\\"
+)
+
+// Split splits a string according to /bin/sh's word-splitting rules. It
+// supports backslash-escapes, single-quotes, and double-quotes. Notably it does
+// not support the $'' style of quoting. It also doesn't attempt to perform any
+// other sort of expansion, including brace expansion, shell expansion, or
+// pathname expansion.
+//
+// If the given input has an unterminated quoted string or ends in a
+// backslash-escape, one of UnterminatedSingleQuoteError,
+// UnterminatedDoubleQuoteError, or UnterminatedEscapeError is returned.
+func Split(input string) (words []string, err error) {
+	var buf bytes.Buffer
+	words = make([]string, 0)
+
+	for len(input) > 0 {
+		// skip any splitChars at the start
+		c, l := utf8.DecodeRuneInString(input)
+		if strings.ContainsRune(splitChars, c) {
+			input = input[l:]
+			continue
+		}
+
+		var word string
+		word, input, err = splitWord(input, &buf)
+		if err != nil {
+			return
+		}
+		words = append(words, word)
+	}
+	return
+}
+
+func splitWord(input string, buf *bytes.Buffer) (word string, remainder string, err error) {
+	buf.Reset()
+
+raw:
+	{
+		cur := input
+		for len(cur) > 0 {
+			c, l := utf8.DecodeRuneInString(cur)
+			cur = cur[l:]
+			if c == singleChar {
+				buf.WriteString(input[0 : len(input)-len(cur)-l])
+				input = cur
+				goto single
+			} else if c == doubleChar {
+				buf.WriteString(input[0 : len(input)-len(cur)-l])
+				input = cur
+				goto double
+			} else if c == escapeChar {
+				buf.WriteString(input[0 : len(input)-len(cur)-l])
+				input = cur
+				goto escape
+			} else if strings.ContainsRune(splitChars, c) {
+				buf.WriteString(input[0 : len(input)-len(cur)-l])
+				return buf.String(), cur, nil
+			}
+		}
+		if len(input) > 0 {
+			buf.WriteString(input)
+			input = ""
+		}
+		goto done
+	}
+
+escape:
+	{
+		if len(input) == 0 {
+			return "", "", UnterminatedEscapeError
+		}
+		c, l := utf8.DecodeRuneInString(input)
+		if c == '\n' {
+			// a backslash-escaped newline is elided from the output entirely
+		} else {
+			buf.WriteString(input[:l])
+		}
+		input = input[l:]
+	}
+	goto raw
+
+single:
+	{
+		i := strings.IndexRune(input, singleChar)
+		if i == -1 {
+			return "", "", UnterminatedSingleQuoteError
+		}
+		buf.WriteString(input[0:i])
+		input = input[i+1:]
+		goto raw
+	}
+
+double:
+	{
+		cur := input
+		for len(cur) > 0 {
+			c, l := utf8.DecodeRuneInString(cur)
+			cur = cur[l:]
+			if c == doubleChar {
+				buf.WriteString(input[0 : len(input)-len(cur)-l])
+				input = cur
+				goto raw
+			} else if c == escapeChar {
+				// bash only supports certain escapes in double-quoted strings
+				c2, l2 := utf8.DecodeRuneInString(cur)
+				cur = cur[l2:]
+				if strings.ContainsRune(doubleEscapeChars, c2) {
+					buf.WriteString(input[0 : len(input)-len(cur)-l-l2])
+					if c2 == '\n' {
+						// newline is special, skip the backslash entirely
+					} else {
+						buf.WriteRune(c2)
+					}
+					input = cur
+				}
+			}
+		}
+		return "", "", UnterminatedDoubleQuoteError
+	}
+
+done:
+	return buf.String(), input, nil
+}

vendor/manifest

@@ -249,6 +249,14 @@
 			"branch": "master",
 			"notests": true
 		},
+		{
+			"importpath": "github.com/kballard/go-shellquote",
+			"repository": "https://github.com/kballard/go-shellquote",
+			"vcs": "git",
+			"revision": "cd60e84ee657ff3dc51de0b4f55dd299a3e136f2",
+			"branch": "master",
+			"notests": true
+		},
 		{
 			"importpath": "github.com/klauspost/cpuid",
 			"repository": "https://github.com/klauspost/cpuid",