diff --git a/cmd/stcli/cmd_folders.go b/cmd/stcli/cmd_folders.go index 197e6d7a4..3190a441a 100644 --- a/cmd/stcli/cmd_folders.go +++ b/cmd/stcli/cmd_folders.go @@ -9,6 +9,7 @@ import ( "github.com/AudriusButkevicius/cli" "github.com/syncthing/syncthing/lib/config" + "github.com/syncthing/syncthing/lib/fs" ) func init() { @@ -102,8 +103,10 @@ func foldersList(c *cli.Context) { if !first { fmt.Fprintln(writer) } + fs := folder.Filesystem() fmt.Fprintln(writer, "ID:\t", folder.ID, "\t") - fmt.Fprintln(writer, "Path:\t", folder.RawPath, "\t(directory)") + fmt.Fprintln(writer, "Path:\t", fs.URI(), "\t(directory)") + fmt.Fprintln(writer, "Path type:\t", fs.Type(), "\t(directory-type)") fmt.Fprintln(writer, "Folder type:\t", folder.Type, "\t(type)") fmt.Fprintln(writer, "Ignore permissions:\t", folder.IgnorePerms, "\t(permissions)") fmt.Fprintln(writer, "Rescan interval in seconds:\t", folder.RescanIntervalS, "\t(rescan)") @@ -124,8 +127,9 @@ func foldersAdd(c *cli.Context) { abs, err := filepath.Abs(c.Args()[1]) die(err) folder := config.FolderConfiguration{ - ID: c.Args()[0], - RawPath: filepath.Clean(abs), + ID: c.Args()[0], + Path: filepath.Clean(abs), + FilesystemType: fs.FilesystemTypeBasic, } cfg.Folders = append(cfg.Folders, folder) setConfig(c, cfg) @@ -185,7 +189,9 @@ func foldersGet(c *cli.Context) { } switch arg { case "directory": - fmt.Println(folder.RawPath) + fmt.Println(folder.Filesystem().URI()) + case "directory-type": + fmt.Println(folder.Filesystem().Type()) case "type": fmt.Println(folder.Type) case "permissions": @@ -197,7 +203,7 @@ func foldersGet(c *cli.Context) { fmt.Println(folder.Versioning.Type) } default: - die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, type, permissions, versioning, versioning-") + die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, directory-type, type, permissions, versioning, versioning-") } return } @@ -220,7 +226,11 @@ func foldersSet(c *cli.Context) { } switch arg { case "directory": - cfg.Folders[i].RawPath = val + cfg.Folders[i].Path = val + case "directory-type": + var fsType fs.FilesystemType + fsType.UnmarshalText([]byte(val)) + cfg.Folders[i].FilesystemType = fsType case "type": var t config.FolderType if err := t.UnmarshalText([]byte(val)); err != nil { diff --git a/cmd/stindex/util.go b/cmd/stindex/util.go index 002a4ce44..0d770a2b9 100644 --- a/cmd/stindex/util.go +++ b/cmd/stindex/util.go @@ -12,7 +12,7 @@ import ( "path/filepath" "runtime" - "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/fs" ) func nulString(bs []byte) string { @@ -33,7 +33,7 @@ func defaultConfigDir() string { return filepath.Join(os.Getenv("AppData"), "Syncthing") case "darwin": - dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing") + dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing") if err != nil { log.Fatal(err) } @@ -43,7 +43,7 @@ func defaultConfigDir() string { if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" { return filepath.Join(xdgCfg, "syncthing") } - dir, err := osutil.ExpandTilde("~/.config/syncthing") + dir, err := fs.ExpandTilde("~/.config/syncthing") if err != nil { log.Fatal(err) } diff --git a/cmd/syncthing/gui.go b/cmd/syncthing/gui.go index c6bd3a965..b15881bfd 100644 --- a/cmd/syncthing/gui.go +++ b/cmd/syncthing/gui.go @@ -28,9 +28,9 @@ import ( "github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/discover" "github.com/syncthing/syncthing/lib/events" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/logger" "github.com/syncthing/syncthing/lib/model" - "github.com/syncthing/syncthing/lib/osutil" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/rand" "github.com/syncthing/syncthing/lib/stats" @@ -856,7 +856,7 @@ func (s *apiService) getSystemStatus(w http.ResponseWriter, r *http.Request) { var m runtime.MemStats runtime.ReadMemStats(&m) - tilde, _ := osutil.ExpandTilde("~") + tilde, _ := fs.ExpandTilde("~") res := make(map[string]interface{}) res["myID"] = myID.String() res["goroutines"] = runtime.NumGoroutine() @@ -1259,23 +1259,35 @@ func (s *apiService) getPeerCompletion(w http.ResponseWriter, r *http.Request) { func (s *apiService) getSystemBrowse(w http.ResponseWriter, r *http.Request) { qs := r.URL.Query() current := qs.Get("current") + // Default value or in case of error unmarshalling ends up being basic fs. + var fsType fs.FilesystemType + fsType.UnmarshalText([]byte(qs.Get("filesystem"))) + if current == "" { - if roots, err := osutil.GetFilesystemRoots(); err == nil { + filesystem := fs.NewFilesystem(fsType, "") + if roots, err := filesystem.Roots(); err == nil { sendJSON(w, roots) } else { http.Error(w, err.Error(), 500) } return } - search, _ := osutil.ExpandTilde(current) - pathSeparator := string(os.PathSeparator) + search, _ := fs.ExpandTilde(current) + pathSeparator := string(fs.PathSeparator) + if strings.HasSuffix(current, pathSeparator) && !strings.HasSuffix(search, pathSeparator) { search = search + pathSeparator } - subdirectories, _ := osutil.Glob(search + "*") + searchDir := filepath.Dir(search) + searchFile := filepath.Base(search) + + fs := fs.NewFilesystem(fsType, searchDir) + + subdirectories, _ := fs.Glob(searchFile + "*") + ret := make([]string, 0, len(subdirectories)) for _, subdirectory := range subdirectories { - info, err := os.Stat(subdirectory) + info, err := fs.Stat(subdirectory) if err == nil && info.IsDir() { ret = append(ret, subdirectory+pathSeparator) } diff --git a/cmd/syncthing/locations.go b/cmd/syncthing/locations.go index d0d5ab98d..916d9d126 100644 --- a/cmd/syncthing/locations.go +++ b/cmd/syncthing/locations.go @@ -13,7 +13,7 @@ import ( "strings" "time" - "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/fs" ) type locationEnum string @@ -65,7 +65,7 @@ func expandLocations() error { dir = strings.Replace(dir, "${"+varName+"}", value, -1) } var err error - dir, err = osutil.ExpandTilde(dir) + dir, err = fs.ExpandTilde(dir) if err != nil { return err } @@ -86,7 +86,7 @@ func defaultConfigDir() string { return filepath.Join(os.Getenv("AppData"), "Syncthing") case "darwin": - dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing") + dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing") if err != nil { l.Fatalln(err) } @@ -96,7 +96,7 @@ func defaultConfigDir() string { if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" { return filepath.Join(xdgCfg, "syncthing") } - dir, err := osutil.ExpandTilde("~/.config/syncthing") + dir, err := fs.ExpandTilde("~/.config/syncthing") if err != nil { l.Fatalln(err) } @@ -106,7 +106,7 @@ func defaultConfigDir() string { // homeDir returns the user's home directory, or dies trying. func homeDir() string { - home, err := osutil.ExpandTilde("~") + home, err := fs.ExpandTilde("~") if err != nil { l.Fatalln(err) } diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index 3accf6d48..66e238353 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -37,6 +37,7 @@ import ( "github.com/syncthing/syncthing/lib/dialer" "github.com/syncthing/syncthing/lib/discover" "github.com/syncthing/syncthing/lib/events" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/logger" "github.com/syncthing/syncthing/lib/model" "github.com/syncthing/syncthing/lib/osutil" @@ -444,7 +445,7 @@ func openGUI() { } func generate(generateDir string) { - dir, err := osutil.ExpandTilde(generateDir) + dir, err := fs.ExpandTilde(generateDir) if err != nil { l.Fatalln("generate:", err) } @@ -1085,7 +1086,7 @@ func defaultConfig(myName string) config.Configuration { if !noDefaultFolder { l.Infoln("Default folder created and/or linked to new config") - defaultFolder = config.NewFolderConfiguration("default", locations[locDefFolder]) + defaultFolder = config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, locations[locDefFolder]) defaultFolder.Label = "Default Folder" defaultFolder.RescanIntervalS = 60 defaultFolder.MinDiskFree = config.Size{Value: 1, Unit: "%"} @@ -1141,19 +1142,20 @@ func shutdown() { stop <- exitSuccess } -func ensureDir(dir string, mode os.FileMode) { - err := osutil.MkdirAll(dir, mode) +func ensureDir(dir string, mode fs.FileMode) { + fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir) + err := fs.MkdirAll(".", mode) if err != nil { l.Fatalln(err) } - if fi, err := os.Stat(dir); err == nil { + if fi, err := fs.Stat("."); err == nil { // Apprently the stat may fail even though the mkdirall passed. If it // does, we'll just assume things are in order and let other things // fail (like loading or creating the config...). currentMode := fi.Mode() & 0777 if currentMode != mode { - err := os.Chmod(dir, mode) + err := fs.Chmod(".", mode) // This can fail on crappy filesystems, nothing we can do about it. if err != nil { l.Warnln(err) @@ -1276,22 +1278,22 @@ func cleanConfigDirectory() { } for pat, dur := range patterns { - pat = filepath.Join(baseDirs["config"], pat) - files, err := osutil.Glob(pat) + fs := fs.NewFilesystem(fs.FilesystemTypeBasic, baseDirs["config"]) + files, err := fs.Glob(pat) if err != nil { l.Infoln("Cleaning:", err) continue } for _, file := range files { - info, err := osutil.Lstat(file) + info, err := fs.Lstat(file) if err != nil { l.Infoln("Cleaning:", err) continue } if time.Since(info.ModTime()) > dur { - if err = os.RemoveAll(file); err != nil { + if err = fs.RemoveAll(file); err != nil { l.Infoln("Cleaning:", err) } else { l.Infoln("Cleaned away old file", filepath.Base(file)) diff --git a/gui/default/syncthing/core/notifications.html b/gui/default/syncthing/core/notifications.html index 7fee92df1..061a26e04 100644 --- a/gui/default/syncthing/core/notifications.html +++ b/gui/default/syncthing/core/notifications.html @@ -40,4 +40,4 @@
- \ No newline at end of file + diff --git a/gui/default/syncthing/folder/editFolderModalView.html b/gui/default/syncthing/folder/editFolderModalView.html index 1a509078e..e176b6df9 100644 --- a/gui/default/syncthing/folder/editFolderModalView.html +++ b/gui/default/syncthing/folder/editFolderModalView.html @@ -180,7 +180,7 @@

- The first command line parameter is the folder path and the second parameter is the relative path in the folder. + See external versioner help for supported templated command line parameters. The path cannot be blank.

diff --git a/lib/config/config.go b/lib/config/config.go index 80cb8635b..359c87351 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -23,6 +23,7 @@ import ( "strconv" "strings" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/rand" "github.com/syncthing/syncthing/lib/upgrade" @@ -31,7 +32,7 @@ import ( const ( OldestHandledVersion = 10 - CurrentVersion = 21 + CurrentVersion = 22 MaxRescanIntervalS = 365 * 24 * 60 * 60 ) @@ -319,6 +320,9 @@ func (cfg *Configuration) clean() error { if cfg.Version == 20 { convertV20V21(cfg) } + if cfg.Version == 21 { + convertV21V22(cfg) + } // Build a list of available devices existingDevices := make(map[protocol.DeviceID]bool) @@ -368,23 +372,38 @@ func (cfg *Configuration) clean() error { return nil } +func convertV21V22(cfg *Configuration) { + for i := range cfg.Folders { + cfg.Folders[i].FilesystemType = fs.FilesystemTypeBasic + // Migrate to templated external versioner commands + if cfg.Folders[i].Versioning.Type == "external" { + cfg.Folders[i].Versioning.Params["command"] += " %FOLDER_PATH% %FILE_PATH%" + } + } + + cfg.Version = 22 +} + func convertV20V21(cfg *Configuration) { for _, folder := range cfg.Folders { + if folder.FilesystemType != fs.FilesystemTypeBasic { + continue + } switch folder.Versioning.Type { case "simple", "trashcan": // Clean out symlinks in the known place - cleanSymlinks(filepath.Join(folder.Path(), ".stversions")) + cleanSymlinks(folder.Filesystem(), ".stversions") case "staggered": versionDir := folder.Versioning.Params["versionsPath"] if versionDir == "" { // default place - cleanSymlinks(filepath.Join(folder.Path(), ".stversions")) + cleanSymlinks(folder.Filesystem(), ".stversions") } else if filepath.IsAbs(versionDir) { // absolute - cleanSymlinks(versionDir) + cleanSymlinks(fs.NewFilesystem(fs.FilesystemTypeBasic, versionDir), ".") } else { // relative to folder - cleanSymlinks(filepath.Join(folder.Path(), versionDir)) + cleanSymlinks(folder.Filesystem(), versionDir) } } } @@ -428,9 +447,7 @@ func convertV17V18(cfg *Configuration) { } func convertV16V17(cfg *Configuration) { - for i := range cfg.Folders { - cfg.Folders[i].Fsync = true - } + // Fsync = true removed cfg.Version = 17 } @@ -670,21 +687,21 @@ loop: return devices[0:count] } -func cleanSymlinks(dir string) { +func cleanSymlinks(filesystem fs.Filesystem, dir string) { if runtime.GOOS == "windows" { // We don't do symlinks on Windows. Additionally, there may // be things that look like symlinks that are not, which we // should leave alone. Deduplicated files, for example. return } - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + filesystem.Walk(dir, func(path string, info fs.FileInfo, err error) error { if err != nil { return err } - if info.Mode()&os.ModeSymlink != 0 { + if info.IsSymlink() { l.Infoln("Removing incorrectly versioned symlink", path) - os.Remove(path) - return filepath.SkipDir + filesystem.Remove(path) + return fs.SkipDir } return nil }) diff --git a/lib/config/config_test.go b/lib/config/config_test.go index 5a18e9818..0d498eaa7 100644 --- a/lib/config/config_test.go +++ b/lib/config/config_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/d4l3k/messagediff" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/protocol" ) @@ -103,7 +104,8 @@ func TestDeviceConfig(t *testing.T) { expectedFolders := []FolderConfiguration{ { ID: "test", - RawPath: "testdata", + FilesystemType: fs.FilesystemTypeBasic, + Path: "testdata", Devices: []FolderDeviceConfiguration{{DeviceID: device1}, {DeviceID: device4}}, Type: FolderTypeSendOnly, RescanIntervalS: 600, @@ -113,7 +115,6 @@ func TestDeviceConfig(t *testing.T) { AutoNormalize: true, MinDiskFree: Size{1, "%"}, MaxConflicts: -1, - Fsync: true, Versioning: VersioningConfiguration{ Params: map[string]string{}, }, @@ -121,15 +122,11 @@ func TestDeviceConfig(t *testing.T) { }, } - // The cachedPath will have been resolved to an absolute path, + // The cachedFilesystem will have been resolved to an absolute path, // depending on where the tests are running. Zero it out so we don't // fail based on that. for i := range cfg.Folders { - cfg.Folders[i].cachedPath = "" - } - - if runtime.GOOS != "windows" { - expectedFolders[0].RawPath += string(filepath.Separator) + cfg.Folders[i].cachedFilesystem = nil } expectedDevices := []DeviceConfiguration{ @@ -377,16 +374,17 @@ func TestVersioningConfig(t *testing.T) { } func TestIssue1262(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skipf("path gets converted to absolute as part of the filesystem initialization on linux") + } + cfg, err := Load("testdata/issue-1262.xml", device4) if err != nil { t.Fatal(err) } - actual := cfg.Folders()["test"].RawPath - expected := "e:/" - if runtime.GOOS == "windows" { - expected = `e:\` - } + actual := cfg.Folders()["test"].Filesystem().URI() + expected := `e:\` if actual != expected { t.Errorf("%q != %q", actual, expected) @@ -416,43 +414,12 @@ func TestIssue1750(t *testing.T) { } } -func TestWindowsPaths(t *testing.T) { - if runtime.GOOS != "windows" { - t.Skip("Not useful on non-Windows") - return - } - - folder := FolderConfiguration{ - RawPath: `e:\`, - } - - expected := `\\?\e:\` - actual := folder.Path() - if actual != expected { - t.Errorf("%q != %q", actual, expected) - } - - folder.RawPath = `\\192.0.2.22\network\share` - expected = folder.RawPath - actual = folder.Path() - if actual != expected { - t.Errorf("%q != %q", actual, expected) - } - - folder.RawPath = `relative\path` - expected = folder.RawPath - actual = folder.Path() - if actual == expected || !strings.HasPrefix(actual, "\\\\?\\") { - t.Errorf("%q == %q, expected absolutification", actual, expected) - } -} - func TestFolderPath(t *testing.T) { folder := FolderConfiguration{ - RawPath: "~/tmp", + Path: "~/tmp", } - realPath := folder.Path() + realPath := folder.Filesystem().URI() if !filepath.IsAbs(realPath) { t.Error(realPath, "should be absolute") } @@ -677,8 +644,8 @@ func TestEmptyFolderPaths(t *testing.T) { t.Fatal(err) } folder := wrapper.Folders()["f1"] - if folder.Path() != "" { - t.Errorf("Expected %q to be empty", folder.Path()) + if folder.cachedFilesystem != nil { + t.Errorf("Expected %q to be empty", folder.cachedFilesystem) } } diff --git a/lib/config/folderconfiguration.go b/lib/config/folderconfiguration.go index 171715e1a..0f9fe87b8 100644 --- a/lib/config/folderconfiguration.go +++ b/lib/config/folderconfiguration.go @@ -8,19 +8,17 @@ package config import ( "fmt" - "os" - "path/filepath" "runtime" - "strings" - "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/protocol" ) type FolderConfiguration struct { ID string `xml:"id,attr" json:"id"` Label string `xml:"label,attr" json:"label"` - RawPath string `xml:"path,attr" json:"path"` + FilesystemType fs.FilesystemType `xml:"filesystemType" json:"filesystemType"` + Path string `xml:"path,attr" json:"path"` Type FolderType `xml:"type,attr" json:"type"` Devices []FolderDeviceConfiguration `xml:"device" json:"devices"` RescanIntervalS int `xml:"rescanIntervalS,attr" json:"rescanIntervalS"` @@ -39,11 +37,10 @@ type FolderConfiguration struct { MaxConflicts int `xml:"maxConflicts" json:"maxConflicts"` DisableSparseFiles bool `xml:"disableSparseFiles" json:"disableSparseFiles"` DisableTempIndexes bool `xml:"disableTempIndexes" json:"disableTempIndexes"` - Fsync bool `xml:"fsync" json:"fsync"` Paused bool `xml:"paused" json:"paused"` WeakHashThresholdPct int `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash. - cachedPath string + cachedFilesystem fs.Filesystem DeprecatedReadOnly bool `xml:"ro,attr,omitempty" json:"-"` DeprecatedMinDiskFreePct float64 `xml:"minDiskFreePct,omitempty" json:"-"` @@ -54,10 +51,11 @@ type FolderDeviceConfiguration struct { IntroducedBy protocol.DeviceID `xml:"introducedBy,attr" json:"introducedBy"` } -func NewFolderConfiguration(id, path string) FolderConfiguration { +func NewFolderConfiguration(id string, fsType fs.FilesystemType, path string) FolderConfiguration { f := FolderConfiguration{ - ID: id, - RawPath: path, + ID: id, + FilesystemType: fsType, + Path: path, } f.prepare() return f @@ -71,53 +69,57 @@ func (f FolderConfiguration) Copy() FolderConfiguration { return c } -func (f FolderConfiguration) Path() string { +func (f FolderConfiguration) Filesystem() fs.Filesystem { // This is intentionally not a pointer method, because things like - // cfg.Folders["default"].Path() should be valid. - - if f.cachedPath == "" && f.RawPath != "" { - l.Infoln("bug: uncached path call (should only happen in tests)") - return f.cleanedPath() + // cfg.Folders["default"].Filesystem() should be valid. + if f.cachedFilesystem == nil && f.Path != "" { + l.Infoln("bug: uncached filesystem call (should only happen in tests)") + return fs.NewFilesystem(f.FilesystemType, f.Path) } - return f.cachedPath + return f.cachedFilesystem } func (f *FolderConfiguration) CreateMarker() error { if !f.HasMarker() { - marker := filepath.Join(f.Path(), ".stfolder") - fd, err := os.Create(marker) + fs := f.Filesystem() + fd, err := fs.Create(".stfolder") if err != nil { return err } fd.Close() - if err := osutil.SyncDir(filepath.Dir(marker)); err != nil { - l.Infof("fsync %q failed: %v", filepath.Dir(marker), err) + if dir, err := fs.Open("."); err == nil { + if serr := dir.Sync(); err != nil { + l.Infof("fsync %q failed: %v", ".", serr) + } + } else { + l.Infof("fsync %q failed: %v", ".", err) } - osutil.HideFile(marker) + fs.Hide(".stfolder") } return nil } func (f *FolderConfiguration) HasMarker() bool { - _, err := os.Stat(filepath.Join(f.Path(), ".stfolder")) + _, err := f.Filesystem().Stat(".stfolder") return err == nil } func (f *FolderConfiguration) CreateRoot() (err error) { // Directory permission bits. Will be filtered down to something // sane by umask on Unixes. - permBits := os.FileMode(0777) + permBits := fs.FileMode(0777) if runtime.GOOS == "windows" { // Windows has no umask so we must chose a safer set of bits to // begin with. permBits = 0700 } - if _, err = os.Stat(f.Path()); os.IsNotExist(err) { - if err = osutil.MkdirAll(f.Path(), permBits); err != nil { - l.Warnf("Creating directory for %v: %v", - f.Description(), err) + filesystem := f.Filesystem() + + if _, err = filesystem.Stat("."); fs.IsNotExist(err) { + if err = filesystem.MkdirAll(".", permBits); err != nil { + l.Warnf("Creating directory for %v: %v", f.Description(), err) } } @@ -140,24 +142,10 @@ func (f *FolderConfiguration) DeviceIDs() []protocol.DeviceID { } func (f *FolderConfiguration) prepare() { - if f.RawPath != "" { - // The reason it's done like this: - // C: -> C:\ -> C:\ (issue that this is trying to fix) - // C:\somedir -> C:\somedir\ -> C:\somedir - // C:\somedir\ -> C:\somedir\\ -> C:\somedir - // This way in the tests, we get away without OS specific separators - // in the test configs. - f.RawPath = filepath.Dir(f.RawPath + string(filepath.Separator)) - - // If we're not on Windows, we want the path to end with a slash to - // penetrate symlinks. On Windows, paths must not end with a slash. - if runtime.GOOS != "windows" && f.RawPath[len(f.RawPath)-1] != filepath.Separator { - f.RawPath = f.RawPath + string(filepath.Separator) - } + if f.Path != "" { + f.cachedFilesystem = fs.NewFilesystem(f.FilesystemType, f.Path) } - f.cachedPath = f.cleanedPath() - if f.RescanIntervalS > MaxRescanIntervalS { f.RescanIntervalS = MaxRescanIntervalS } else if f.RescanIntervalS < 0 { @@ -173,43 +161,6 @@ func (f *FolderConfiguration) prepare() { } } -func (f *FolderConfiguration) cleanedPath() string { - if f.RawPath == "" { - return "" - } - - cleaned := f.RawPath - - // Attempt tilde expansion; leave unchanged in case of error - if path, err := osutil.ExpandTilde(cleaned); err == nil { - cleaned = path - } - - // Attempt absolutification; leave unchanged in case of error - if !filepath.IsAbs(cleaned) { - // Abs() looks like a fairly expensive syscall on Windows, while - // IsAbs() is a whole bunch of string mangling. I think IsAbs() may be - // somewhat faster in the general case, hence the outer if... - if path, err := filepath.Abs(cleaned); err == nil { - cleaned = path - } - } - - // Attempt to enable long filename support on Windows. We may still not - // have an absolute path here if the previous steps failed. - if runtime.GOOS == "windows" && filepath.IsAbs(cleaned) && !strings.HasPrefix(f.RawPath, `\\`) { - return `\\?\` + cleaned - } - - // If we're not on Windows, we want the path to end with a slash to - // penetrate symlinks. On Windows, paths must not end with a slash. - if runtime.GOOS != "windows" && cleaned[len(cleaned)-1] != filepath.Separator { - cleaned = cleaned + string(filepath.Separator) - } - - return cleaned -} - type FolderDeviceConfigurationList []FolderDeviceConfiguration func (l FolderDeviceConfigurationList) Less(a, b int) bool { diff --git a/lib/config/testdata/v22.xml b/lib/config/testdata/v22.xml new file mode 100644 index 000000000..2a577fc50 --- /dev/null +++ b/lib/config/testdata/v22.xml @@ -0,0 +1,16 @@ + + + basic + + + 1 + -1 + true + + +
tcp://a
+ + +
tcp://b
+ + diff --git a/lib/config/testdata/versioningconfig.xml b/lib/config/testdata/versioningconfig.xml index b9c8c85d9..56bbe5c32 100644 --- a/lib/config/testdata/versioningconfig.xml +++ b/lib/config/testdata/versioningconfig.xml @@ -1,5 +1,5 @@ - - + + diff --git a/lib/db/set.go b/lib/db/set.go index 17cc4ae04..9bfbd0fbd 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -25,6 +25,7 @@ import ( type FileSet struct { sequence int64 // Our local sequence number folder string + fs fs.Filesystem db *Instance blockmap *BlockMap localSize sizeTracker @@ -113,10 +114,11 @@ func (s *sizeTracker) Size() Counts { return s.Counts } -func NewFileSet(folder string, db *Instance) *FileSet { +func NewFileSet(folder string, fs fs.Filesystem, db *Instance) *FileSet { var s = FileSet{ remoteSequence: make(map[protocol.DeviceID]int64), folder: folder, + fs: fs, db: db, blockmap: NewBlockMap(db, db.folderIdx.ID([]byte(folder))), updateMutex: sync.NewMutex(), @@ -303,7 +305,7 @@ func (s *FileSet) SetIndexID(device protocol.DeviceID, id protocol.IndexID) { func (s *FileSet) MtimeFS() *fs.MtimeFS { prefix := s.db.mtimesKey([]byte(s.folder)) kv := NewNamespacedKV(s.db, string(prefix)) - return fs.NewMtimeFS(fs.DefaultFilesystem, kv) + return fs.NewMtimeFS(s.fs, kv) } func (s *FileSet) ListDevices() []protocol.DeviceID { diff --git a/lib/db/set_test.go b/lib/db/set_test.go index 91bb1d775..d21e76c9b 100644 --- a/lib/db/set_test.go +++ b/lib/db/set_test.go @@ -15,6 +15,7 @@ import ( "github.com/d4l3k/messagediff" "github.com/syncthing/syncthing/lib/db" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/protocol" ) @@ -97,7 +98,7 @@ func (l fileList) String() string { func TestGlobalSet(t *testing.T) { ldb := db.OpenMemory() - m := db.NewFileSet("test", ldb) + m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) local0 := fileList{ protocol.FileInfo{Name: "a", Sequence: 1, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)}, @@ -312,7 +313,7 @@ func TestGlobalSet(t *testing.T) { func TestNeedWithInvalid(t *testing.T) { ldb := db.OpenMemory() - s := db.NewFileSet("test", ldb) + s := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) localHave := fileList{ protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)}, @@ -349,7 +350,7 @@ func TestNeedWithInvalid(t *testing.T) { func TestUpdateToInvalid(t *testing.T) { ldb := db.OpenMemory() - s := db.NewFileSet("test", ldb) + s := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) localHave := fileList{ protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)}, @@ -381,7 +382,7 @@ func TestUpdateToInvalid(t *testing.T) { func TestInvalidAvailability(t *testing.T) { ldb := db.OpenMemory() - s := db.NewFileSet("test", ldb) + s := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) remote0Have := fileList{ protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)}, @@ -419,7 +420,7 @@ func TestInvalidAvailability(t *testing.T) { func TestGlobalReset(t *testing.T) { ldb := db.OpenMemory() - m := db.NewFileSet("test", ldb) + m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) local := []protocol.FileInfo{ {Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}}, @@ -457,7 +458,7 @@ func TestGlobalReset(t *testing.T) { func TestNeed(t *testing.T) { ldb := db.OpenMemory() - m := db.NewFileSet("test", ldb) + m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) local := []protocol.FileInfo{ {Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}}, @@ -495,7 +496,7 @@ func TestNeed(t *testing.T) { func TestSequence(t *testing.T) { ldb := db.OpenMemory() - m := db.NewFileSet("test", ldb) + m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) local1 := []protocol.FileInfo{ {Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}}, @@ -525,7 +526,7 @@ func TestSequence(t *testing.T) { func TestListDropFolder(t *testing.T) { ldb := db.OpenMemory() - s0 := db.NewFileSet("test0", ldb) + s0 := db.NewFileSet("test0", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) local1 := []protocol.FileInfo{ {Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}}, {Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}}, @@ -533,7 +534,7 @@ func TestListDropFolder(t *testing.T) { } s0.Replace(protocol.LocalDeviceID, local1) - s1 := db.NewFileSet("test1", ldb) + s1 := db.NewFileSet("test1", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) local2 := []protocol.FileInfo{ {Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}}, {Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}}, @@ -575,7 +576,7 @@ func TestListDropFolder(t *testing.T) { func TestGlobalNeedWithInvalid(t *testing.T) { ldb := db.OpenMemory() - s := db.NewFileSet("test1", ldb) + s := db.NewFileSet("test1", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) rem0 := fileList{ protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)}, @@ -612,7 +613,7 @@ func TestGlobalNeedWithInvalid(t *testing.T) { func TestLongPath(t *testing.T) { ldb := db.OpenMemory() - s := db.NewFileSet("test", ldb) + s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) var b bytes.Buffer for i := 0; i < 100; i++ { @@ -642,7 +643,7 @@ func TestCommitted(t *testing.T) { ldb := db.OpenMemory() - s := db.NewFileSet("test", ldb) + s := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) local := []protocol.FileInfo{ {Name: string("file"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}}, @@ -688,7 +689,7 @@ func BenchmarkUpdateOneFile(b *testing.B) { os.RemoveAll("testdata/benchmarkupdate.db") }() - m := db.NewFileSet("test", ldb) + m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) m.Replace(protocol.LocalDeviceID, local0) l := local0[4:5] @@ -703,7 +704,7 @@ func BenchmarkUpdateOneFile(b *testing.B) { func TestIndexID(t *testing.T) { ldb := db.OpenMemory() - s := db.NewFileSet("test", ldb) + s := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) // The Index ID for some random device is zero by default. id := s.IndexID(remoteDevice0) diff --git a/lib/fs/basicfs.go b/lib/fs/basicfs.go index 3e9626d5c..2274b50cd 100644 --- a/lib/fs/basicfs.go +++ b/lib/fs/basicfs.go @@ -9,30 +9,156 @@ package fs import ( "errors" "os" + "path/filepath" + "runtime" + "strings" "time" + + "github.com/calmh/du" +) + +var ( + ErrInvalidFilename = errors.New("filename is invalid") + ErrNotRelative = errors.New("not a relative path") ) // The BasicFilesystem implements all aspects by delegating to package os. +// All paths are relative to the root and cannot (should not) escape the root directory. type BasicFilesystem struct { + root string } -func NewBasicFilesystem() *BasicFilesystem { - return new(BasicFilesystem) +func newBasicFilesystem(root string) *BasicFilesystem { + // The reason it's done like this: + // C: -> C:\ -> C:\ (issue that this is trying to fix) + // C:\somedir -> C:\somedir\ -> C:\somedir + // C:\somedir\ -> C:\somedir\\ -> C:\somedir + // This way in the tests, we get away without OS specific separators + // in the test configs. + root = filepath.Dir(root + string(filepath.Separator)) + + // Attempt tilde expansion; leave unchanged in case of error + if path, err := ExpandTilde(root); err == nil { + root = path + } + + // Attempt absolutification; leave unchanged in case of error + if !filepath.IsAbs(root) { + // Abs() looks like a fairly expensive syscall on Windows, while + // IsAbs() is a whole bunch of string mangling. I think IsAbs() may be + // somewhat faster in the general case, hence the outer if... + if path, err := filepath.Abs(root); err == nil { + root = path + } + } + + // Attempt to enable long filename support on Windows. We may still not + // have an absolute path here if the previous steps failed. + if runtime.GOOS == "windows" { + if filepath.IsAbs(root) && !strings.HasPrefix(root, `\\`) { + root = `\\?\` + root + } + // If we're not on Windows, we want the path to end with a slash to + // penetrate symlinks. On Windows, paths must not end with a slash. + } else if root[len(root)-1] != filepath.Separator { + root = root + string(filepath.Separator) + } + + return &BasicFilesystem{ + root: root, + } +} + +// rooted expands the relative path to the full path that is then used with os +// package. If the relative path somehow causes the final path to escape the root +// directoy, this returns an error, to prevent accessing files that are not in the +// shared directory. +func (f *BasicFilesystem) rooted(rel string) (string, error) { + // The root must not be empty. + if f.root == "" { + return "", ErrInvalidFilename + } + + pathSep := string(PathSeparator) + + // The expected prefix for the resulting path is the root, with a path + // separator at the end. + expectedPrefix := filepath.FromSlash(f.root) + if !strings.HasSuffix(expectedPrefix, pathSep) { + expectedPrefix += pathSep + } + + // The relative path should be clean from internal dotdots and similar + // funkyness. + rel = filepath.FromSlash(rel) + if filepath.Clean(rel) != rel { + return "", ErrInvalidFilename + } + + // It is not acceptable to attempt to traverse upwards. + switch rel { + case "..", pathSep: + return "", ErrNotRelative + } + if strings.HasPrefix(rel, ".."+pathSep) { + return "", ErrNotRelative + } + + if strings.HasPrefix(rel, pathSep+pathSep) { + // The relative path may pretend to be an absolute path within the + // root, but the double path separator on Windows implies something + // else. It would get cleaned by the Join below, but it's out of + // spec anyway. + return "", ErrNotRelative + } + + // The supposedly correct path is the one filepath.Join will return, as + // it does cleaning and so on. Check that one first to make sure no + // obvious escape attempts have been made. + joined := filepath.Join(f.root, rel) + if rel == "." && !strings.HasSuffix(joined, pathSep) { + joined += pathSep + } + if !strings.HasPrefix(joined, expectedPrefix) { + return "", ErrNotRelative + } + + return joined, nil +} + +func (f *BasicFilesystem) unrooted(path string) string { + return strings.TrimPrefix(strings.TrimPrefix(path, f.root), string(PathSeparator)) } func (f *BasicFilesystem) Chmod(name string, mode FileMode) error { + name, err := f.rooted(name) + if err != nil { + return err + } return os.Chmod(name, os.FileMode(mode)) } func (f *BasicFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error { + name, err := f.rooted(name) + if err != nil { + return err + } return os.Chtimes(name, atime, mtime) } func (f *BasicFilesystem) Mkdir(name string, perm FileMode) error { + name, err := f.rooted(name) + if err != nil { + return err + } return os.Mkdir(name, os.FileMode(perm)) } func (f *BasicFilesystem) Lstat(name string) (FileInfo, error) { + name, err := f.rooted(name) + if err != nil { + return nil, err + } fi, err := underlyingLstat(name) if err != nil { return nil, err @@ -41,14 +167,38 @@ func (f *BasicFilesystem) Lstat(name string) (FileInfo, error) { } func (f *BasicFilesystem) Remove(name string) error { + name, err := f.rooted(name) + if err != nil { + return err + } return os.Remove(name) } +func (f *BasicFilesystem) RemoveAll(name string) error { + name, err := f.rooted(name) + if err != nil { + return err + } + return os.RemoveAll(name) +} + func (f *BasicFilesystem) Rename(oldpath, newpath string) error { + oldpath, err := f.rooted(oldpath) + if err != nil { + return err + } + newpath, err = f.rooted(newpath) + if err != nil { + return err + } return os.Rename(oldpath, newpath) } func (f *BasicFilesystem) Stat(name string) (FileInfo, error) { + name, err := f.rooted(name) + if err != nil { + return nil, err + } fi, err := os.Stat(name) if err != nil { return nil, err @@ -57,7 +207,11 @@ func (f *BasicFilesystem) Stat(name string) (FileInfo, error) { } func (f *BasicFilesystem) DirNames(name string) ([]string, error) { - fd, err := os.OpenFile(name, os.O_RDONLY, 0777) + name, err := f.rooted(name) + if err != nil { + return nil, err + } + fd, err := os.OpenFile(name, OptReadOnly, 0777) if err != nil { return nil, err } @@ -72,19 +226,39 @@ func (f *BasicFilesystem) DirNames(name string) ([]string, error) { } func (f *BasicFilesystem) Open(name string) (File, error) { - fd, err := os.Open(name) + rootedName, err := f.rooted(name) if err != nil { return nil, err } - return fsFile{fd}, err + fd, err := os.Open(rootedName) + if err != nil { + return nil, err + } + return fsFile{fd, name}, err +} + +func (f *BasicFilesystem) OpenFile(name string, flags int, mode FileMode) (File, error) { + rootedName, err := f.rooted(name) + if err != nil { + return nil, err + } + fd, err := os.OpenFile(rootedName, flags, os.FileMode(mode)) + if err != nil { + return nil, err + } + return fsFile{fd, name}, err } func (f *BasicFilesystem) Create(name string) (File, error) { - fd, err := os.Create(name) + rootedName, err := f.rooted(name) if err != nil { return nil, err } - return fsFile{fd}, err + fd, err := os.Create(rootedName) + if err != nil { + return nil, err + } + return fsFile{fd, name}, err } func (f *BasicFilesystem) Walk(root string, walkFn WalkFunc) error { @@ -92,9 +266,47 @@ func (f *BasicFilesystem) Walk(root string, walkFn WalkFunc) error { return errors.New("not implemented") } +func (f *BasicFilesystem) Glob(pattern string) ([]string, error) { + pattern, err := f.rooted(pattern) + if err != nil { + return nil, err + } + files, err := filepath.Glob(pattern) + unrooted := make([]string, len(files)) + for i := range files { + unrooted[i] = f.unrooted(files[i]) + } + return unrooted, err +} + +func (f *BasicFilesystem) Usage(name string) (Usage, error) { + name, err := f.rooted(name) + if err != nil { + return Usage{}, err + } + u, err := du.Get(name) + return Usage{ + Free: u.FreeBytes, + Total: u.TotalBytes, + }, err +} + +func (f *BasicFilesystem) Type() FilesystemType { + return FilesystemTypeBasic +} + +func (f *BasicFilesystem) URI() string { + return strings.TrimPrefix(f.root, `\\?\`) +} + // fsFile implements the fs.File interface on top of an os.File type fsFile struct { *os.File + name string +} + +func (f fsFile) Name() string { + return f.name } func (f fsFile) Stat() (FileInfo, error) { @@ -105,6 +317,17 @@ func (f fsFile) Stat() (FileInfo, error) { return fsFileInfo{info}, nil } +func (f fsFile) Sync() error { + err := f.File.Sync() + // On Windows, fsyncing a directory returns a "handle is invalid" + // So we swallow that and let things go through in order not to have to add + // a separate way of syncing directories versus files. + if err != nil && (runtime.GOOS != "windows" || !strings.Contains(err.Error(), "handle is invalid")) { + return err + } + return nil +} + // fsFileInfo implements the fs.FileInfo interface on top of an os.FileInfo. type fsFileInfo struct { os.FileInfo diff --git a/lib/fs/basicfs_symlink_unix.go b/lib/fs/basicfs_symlink_unix.go deleted file mode 100644 index 6b854bf3d..000000000 --- a/lib/fs/basicfs_symlink_unix.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (C) 2016 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -// +build !windows - -package fs - -import "os" - -var symlinksSupported = true - -func DisableSymlinks() { - symlinksSupported = false -} - -func (BasicFilesystem) SymlinksSupported() bool { - return symlinksSupported -} - -func (BasicFilesystem) CreateSymlink(name, target string) error { - return os.Symlink(target, name) -} - -func (BasicFilesystem) ReadSymlink(path string) (string, error) { - return os.Readlink(path) -} diff --git a/lib/fs/basicfs_symlink_windows.go b/lib/fs/basicfs_symlink_windows.go deleted file mode 100644 index 5b85e1404..000000000 --- a/lib/fs/basicfs_symlink_windows.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -// +build windows - -package fs - -import "errors" - -var errNotSupported = errors.New("symlinks not supported") - -func DisableSymlinks() {} - -func (BasicFilesystem) SymlinksSupported() bool { - return false -} - -func (BasicFilesystem) ReadSymlink(path string) (string, error) { - return "", errNotSupported -} - -func (BasicFilesystem) CreateSymlink(path, target string) error { - return errNotSupported -} diff --git a/lib/fs/basicfs_test.go b/lib/fs/basicfs_test.go new file mode 100644 index 000000000..9d0c4c1ec --- /dev/null +++ b/lib/fs/basicfs_test.go @@ -0,0 +1,486 @@ +// Copyright (C) 2017 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package fs + +import ( + "io/ioutil" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "testing" + "time" +) + +func setup(t *testing.T) (Filesystem, string) { + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + return newBasicFilesystem(dir), dir +} + +func TestChmodFile(t *testing.T) { + fs, dir := setup(t) + path := filepath.Join(dir, "file") + defer os.RemoveAll(dir) + + defer os.Chmod(path, 0666) + + fd, err := os.Create(path) + if err != nil { + t.Error(err) + } + fd.Close() + + if err := os.Chmod(path, 0666); err != nil { + t.Error(err) + } + + if stat, err := os.Stat(path); err != nil || stat.Mode()&os.ModePerm != 0666 { + t.Errorf("wrong perm: %t %#o", err == nil, stat.Mode()&os.ModePerm) + } + + if err := fs.Chmod("file", 0444); err != nil { + t.Error(err) + } + + if stat, err := os.Stat(path); err != nil || stat.Mode()&os.ModePerm != 0444 { + t.Errorf("wrong perm: %t %#o", err == nil, stat.Mode()&os.ModePerm) + } +} + +func TestChmodDir(t *testing.T) { + fs, dir := setup(t) + path := filepath.Join(dir, "dir") + defer os.RemoveAll(dir) + + mode := os.FileMode(0755) + if runtime.GOOS == "windows" { + mode = os.FileMode(0777) + } + + defer os.Chmod(path, mode) + + if err := os.Mkdir(path, mode); err != nil { + t.Error(err) + } + + if stat, err := os.Stat(path); err != nil || stat.Mode()&os.ModePerm != mode { + t.Errorf("wrong perm: %t %#o", err == nil, stat.Mode()&os.ModePerm) + } + + if err := fs.Chmod("dir", 0555); err != nil { + t.Error(err) + } + + if stat, err := os.Stat(path); err != nil || stat.Mode()&os.ModePerm != 0555 { + t.Errorf("wrong perm: %t %#o", err == nil, stat.Mode()&os.ModePerm) + } +} + +func TestChtimes(t *testing.T) { + fs, dir := setup(t) + path := filepath.Join(dir, "file") + defer os.RemoveAll(dir) + fd, err := os.Create(path) + if err != nil { + t.Error(err) + } + fd.Close() + + mtime := time.Now().Add(-time.Hour) + + fs.Chtimes("file", mtime, mtime) + + stat, err := os.Stat(path) + if err != nil { + t.Error(err) + } + + diff := stat.ModTime().Sub(mtime) + if diff > 3*time.Second || diff < -3*time.Second { + t.Errorf("%s != %s", stat.Mode(), mtime) + } +} + +func TestCreate(t *testing.T) { + fs, dir := setup(t) + path := filepath.Join(dir, "file") + defer os.RemoveAll(dir) + + if _, err := os.Stat(path); err == nil { + t.Errorf("exists?") + } + + fd, err := fs.Create("file") + if err != nil { + t.Error(err) + } + fd.Close() + + if _, err := os.Stat(path); err != nil { + t.Error(err) + } +} + +func TestCreateSymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("windows not supported") + } + + fs, dir := setup(t) + path := filepath.Join(dir, "file") + defer os.RemoveAll(dir) + + if err := fs.CreateSymlink("blah", "file"); err != nil { + t.Error(err) + } + + if target, err := os.Readlink(path); err != nil || target != "blah" { + t.Error("target", target, "err", err) + } + + if err := os.Remove(path); err != nil { + t.Error(err) + } + + if err := fs.CreateSymlink(filepath.Join("..", "blah"), "file"); err != nil { + t.Error(err) + } + + if target, err := os.Readlink(path); err != nil || target != filepath.Join("..", "blah") { + t.Error("target", target, "err", err) + } +} + +func TestDirNames(t *testing.T) { + fs, dir := setup(t) + defer os.RemoveAll(dir) + + // Case differences + testCases := []string{ + "a", + "bC", + } + sort.Strings(testCases) + + for _, sub := range testCases { + if err := os.Mkdir(filepath.Join(dir, sub), 0777); err != nil { + t.Error(err) + } + } + + if dirs, err := fs.DirNames("."); err != nil || len(dirs) != len(testCases) { + t.Errorf("%s %s %s", err, dirs, testCases) + } else { + sort.Strings(dirs) + for i := range dirs { + if dirs[i] != testCases[i] { + t.Errorf("%s != %s", dirs[i], testCases[i]) + } + } + } +} + +func TestNames(t *testing.T) { + // Tests that all names are without the root directory. + fs, dir := setup(t) + defer os.RemoveAll(dir) + + expected := "file" + fd, err := fs.Create(expected) + if err != nil { + t.Error(err) + } + defer fd.Close() + + if fd.Name() != expected { + t.Errorf("incorrect %s != %s", fd.Name(), expected) + } + if stat, err := fd.Stat(); err != nil || stat.Name() != expected { + t.Errorf("incorrect %s != %s (%v)", stat.Name(), expected, err) + } + + if err := fs.Mkdir("dir", 0777); err != nil { + t.Error(err) + } + + expected = filepath.Join("dir", "file") + fd, err = fs.Create(expected) + if err != nil { + t.Error(err) + } + defer fd.Close() + + if fd.Name() != expected { + t.Errorf("incorrect %s != %s", fd.Name(), expected) + } + + // os.fd.Stat() returns just base, so do we. + if stat, err := fd.Stat(); err != nil || stat.Name() != filepath.Base(expected) { + t.Errorf("incorrect %s != %s (%v)", stat.Name(), filepath.Base(expected), err) + } +} + +func TestGlob(t *testing.T) { + // Tests that all names are without the root directory. + fs, dir := setup(t) + defer os.RemoveAll(dir) + + for _, dirToCreate := range []string{ + filepath.Join("a", "test", "b"), + filepath.Join("a", "best", "b"), + filepath.Join("a", "best", "c"), + } { + if err := fs.MkdirAll(dirToCreate, 0777); err != nil { + t.Error(err) + } + } + + testCases := []struct { + pattern string + matches []string + }{ + { + filepath.Join("a", "?est", "?"), + []string{ + filepath.Join("a", "test", "b"), + filepath.Join("a", "best", "b"), + filepath.Join("a", "best", "c"), + }, + }, + { + filepath.Join("a", "?est", "b"), + []string{ + filepath.Join("a", "test", "b"), + filepath.Join("a", "best", "b"), + }, + }, + { + filepath.Join("a", "best", "?"), + []string{ + filepath.Join("a", "best", "b"), + filepath.Join("a", "best", "c"), + }, + }, + } + + for _, testCase := range testCases { + results, err := fs.Glob(testCase.pattern) + sort.Strings(results) + sort.Strings(testCase.matches) + if err != nil { + t.Error(err) + } + if len(results) != len(testCase.matches) { + t.Errorf("result count mismatch") + } + for i := range testCase.matches { + if results[i] != testCase.matches[i] { + t.Errorf("%s != %s", results[i], testCase.matches[i]) + } + } + } +} + +func TestUsage(t *testing.T) { + fs, dir := setup(t) + defer os.RemoveAll(dir) + usage, err := fs.Usage(".") + if err != nil { + if runtime.GOOS == "netbsd" || runtime.GOOS == "openbsd" || runtime.GOOS == "solaris" { + t.Skip() + } + t.Errorf("Unexpected error: %s", err) + } + if usage.Free < 1 { + t.Error("Disk is full?", usage.Free) + } +} + +func TestWindowsPaths(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("Not useful on non-Windows") + return + } + + testCases := []struct { + input string + expectedRoot string + expectedURI string + }{ + {`e:\`, `\\?\e:\`, `e:\`}, + {`\\?\e:\`, `\\?\e:\`, `e:\`}, + {`\\192.0.2.22\network\share`, `\\192.0.2.22\network\share`, `\\192.0.2.22\network\share`}, + } + + for _, testCase := range testCases { + fs := newBasicFilesystem(testCase.input) + if fs.root != testCase.expectedRoot { + t.Errorf("root %q != %q", fs.root, testCase.expectedRoot) + } + if fs.URI() != testCase.expectedURI { + t.Errorf("uri %q != %q", fs.URI(), testCase.expectedURI) + } + } + + fs := newBasicFilesystem(`relative\path`) + if fs.root == `relative\path` || !strings.HasPrefix(fs.root, "\\\\?\\") { + t.Errorf("%q == %q, expected absolutification", fs.root, `relative\path`) + } +} + +func TestRooted(t *testing.T) { + type testcase struct { + root string + rel string + joined string + ok bool + } + cases := []testcase{ + // Valid cases + {"foo", "bar", "foo/bar", true}, + {"foo", "/bar", "foo/bar", true}, + {"foo/", "bar", "foo/bar", true}, + {"foo/", "/bar", "foo/bar", true}, + {"baz/foo", "bar", "baz/foo/bar", true}, + {"baz/foo", "/bar", "baz/foo/bar", true}, + {"baz/foo/", "bar", "baz/foo/bar", true}, + {"baz/foo/", "/bar", "baz/foo/bar", true}, + {"foo", "bar/baz", "foo/bar/baz", true}, + {"foo", "/bar/baz", "foo/bar/baz", true}, + {"foo/", "bar/baz", "foo/bar/baz", true}, + {"foo/", "/bar/baz", "foo/bar/baz", true}, + {"baz/foo", "bar/baz", "baz/foo/bar/baz", true}, + {"baz/foo", "/bar/baz", "baz/foo/bar/baz", true}, + {"baz/foo/", "bar/baz", "baz/foo/bar/baz", true}, + {"baz/foo/", "/bar/baz", "baz/foo/bar/baz", true}, + + // Not escape attempts, but oddly formatted relative paths. Disallowed. + {"foo", "./bar", "", false}, + {"baz/foo", "./bar", "", false}, + {"foo", "./bar/baz", "", false}, + {"baz/foo", "./bar/baz", "", false}, + {"baz/foo", "bar/../baz", "", false}, + {"baz/foo", "/bar/../baz", "", false}, + {"baz/foo", "./bar/../baz", "", false}, + {"baz/foo", "bar/../baz", "", false}, + {"baz/foo", "/bar/../baz", "", false}, + {"baz/foo", "./bar/../baz", "", false}, + + // Results in an allowed path, but does it by probing. Disallowed. + {"foo", "../foo", "", false}, + {"foo", "../foo/bar", "", false}, + {"baz/foo", "../foo/bar", "", false}, + {"baz/foo", "../../baz/foo/bar", "", false}, + {"baz/foo", "bar/../../foo/bar", "", false}, + {"baz/foo", "bar/../../../baz/foo/bar", "", false}, + + // Escape attempts. + {"foo", "", "", false}, + {"foo", "/", "", false}, + {"foo", "..", "", false}, + {"foo", "/..", "", false}, + {"foo", "../", "", false}, + {"foo", "../bar", "", false}, + {"foo", "../foobar", "", false}, + {"foo/", "../bar", "", false}, + {"foo/", "../foobar", "", false}, + {"baz/foo", "../bar", "", false}, + {"baz/foo", "../foobar", "", false}, + {"baz/foo/", "../bar", "", false}, + {"baz/foo/", "../foobar", "", false}, + {"baz/foo/", "bar/../../quux/baz", "", false}, + + // Empty root is a misconfiguration. + {"", "/foo", "", false}, + {"", "foo", "", false}, + {"", ".", "", false}, + {"", "..", "", false}, + {"", "/", "", false}, + {"", "", "", false}, + + // Root=/ is valid, and things should be verified as usual. + {"/", "foo", "/foo", true}, + {"/", "/foo", "/foo", true}, + {"/", "../foo", "", false}, + {"/", "..", "", false}, + {"/", "/", "", false}, + {"/", "", "", false}, + + // special case for filesystems to be able to MkdirAll('.') for example + {"/", ".", "/", true}, + } + + if runtime.GOOS == "windows" { + extraCases := []testcase{ + {`c:\`, `foo`, `c:\foo`, true}, + {`\\?\c:\`, `foo`, `\\?\c:\foo`, true}, + {`c:\`, `\foo`, `c:\foo`, true}, + {`\\?\c:\`, `\foo`, `\\?\c:\foo`, true}, + {`c:\`, `\\foo`, ``, false}, + {`c:\`, ``, ``, false}, + {`c:\`, `\`, ``, false}, + {`\\?\c:\`, `\\foo`, ``, false}, + {`\\?\c:\`, ``, ``, false}, + {`\\?\c:\`, `\`, ``, false}, + + // makes no sense, but will be treated simply as a bad filename + {`c:\foo`, `d:\bar`, `c:\foo\d:\bar`, true}, + + // special case for filesystems to be able to MkdirAll('.') for example + {`c:\`, `.`, `c:\`, true}, + {`\\?\c:\`, `.`, `\\?\c:\`, true}, + } + + for _, tc := range cases { + // Add case where root is backslashed, rel is forward slashed + extraCases = append(extraCases, testcase{ + root: filepath.FromSlash(tc.root), + rel: tc.rel, + joined: tc.joined, + ok: tc.ok, + }) + // and the opposite + extraCases = append(extraCases, testcase{ + root: tc.root, + rel: filepath.FromSlash(tc.rel), + joined: tc.joined, + ok: tc.ok, + }) + // and both backslashed + extraCases = append(extraCases, testcase{ + root: filepath.FromSlash(tc.root), + rel: filepath.FromSlash(tc.rel), + joined: tc.joined, + ok: tc.ok, + }) + } + + cases = append(cases, extraCases...) + } + + for _, tc := range cases { + fs := BasicFilesystem{root: tc.root} + res, err := fs.rooted(tc.rel) + if tc.ok { + if err != nil { + t.Errorf("Unexpected error for rooted(%q, %q): %v", tc.root, tc.rel, err) + continue + } + exp := filepath.FromSlash(tc.joined) + if res != exp { + t.Errorf("Unexpected result for rooted(%q, %q): %q != expected %q", tc.root, tc.rel, res, exp) + } + } else if err == nil { + t.Errorf("Unexpected pass for rooted(%q, %q) => %q", tc.root, tc.rel, res) + continue + } + } +} diff --git a/lib/fs/basicfs_unix.go b/lib/fs/basicfs_unix.go new file mode 100644 index 000000000..b3f46358d --- /dev/null +++ b/lib/fs/basicfs_unix.go @@ -0,0 +1,57 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +// +build !windows + +package fs + +import "os" + +func (BasicFilesystem) SymlinksSupported() bool { + return true +} + +func (f *BasicFilesystem) CreateSymlink(target, name string) error { + name, err := f.rooted(name) + if err != nil { + return err + } + return os.Symlink(target, name) +} + +func (f *BasicFilesystem) ReadSymlink(name string) (string, error) { + name, err := f.rooted(name) + if err != nil { + return "", err + } + return os.Readlink(name) +} + +func (f *BasicFilesystem) MkdirAll(name string, perm FileMode) error { + name, err := f.rooted(name) + if err != nil { + return err + } + return os.MkdirAll(name, os.FileMode(perm)) +} + +// Unhide is a noop on unix, as unhiding files requires renaming them. +// We still check that the relative path does not try to escape the root +func (f *BasicFilesystem) Unhide(name string) error { + _, err := f.rooted(name) + return err +} + +// Hide is a noop on unix, as hiding files requires renaming them. +// We still check that the relative path does not try to escape the root +func (f *BasicFilesystem) Hide(name string) error { + _, err := f.rooted(name) + return err +} + +func (f *BasicFilesystem) Roots() ([]string, error) { + return []string{"/"}, nil +} diff --git a/lib/fs/basicfs_windows.go b/lib/fs/basicfs_windows.go new file mode 100644 index 000000000..e660931d5 --- /dev/null +++ b/lib/fs/basicfs_windows.go @@ -0,0 +1,165 @@ +// Copyright (C) 2014 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +// +build windows + +package fs + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "syscall" + "unsafe" +) + +var errNotSupported = errors.New("symlinks not supported") + +func (BasicFilesystem) SymlinksSupported() bool { + return false +} + +func (BasicFilesystem) ReadSymlink(path string) (string, error) { + return "", errNotSupported +} + +func (BasicFilesystem) CreateSymlink(path, target string) error { + return errNotSupported +} + +// MkdirAll creates a directory named path, along with any necessary parents, +// and returns nil, or else returns an error. +// The permission bits perm are used for all directories that MkdirAll creates. +// If path is already a directory, MkdirAll does nothing and returns nil. +func (f *BasicFilesystem) MkdirAll(path string, perm FileMode) error { + path, err := f.rooted(path) + if err != nil { + return err + } + + return f.mkdirAll(path, os.FileMode(perm)) +} + +// Required due to https://github.com/golang/go/issues/10900 +func (f *BasicFilesystem) mkdirAll(path string, perm os.FileMode) error { + // Fast path: if we can tell whether path is a directory or file, stop with success or error. + dir, err := os.Stat(path) + if err == nil { + if dir.IsDir() { + return nil + } + return &os.PathError{ + Op: "mkdir", + Path: path, + Err: syscall.ENOTDIR, + } + } + + // Slow path: make sure parent exists and then call Mkdir for path. + i := len(path) + for i > 0 && IsPathSeparator(path[i-1]) { // Skip trailing path separator. + i-- + } + + j := i + for j > 0 && !IsPathSeparator(path[j-1]) { // Scan backward over element. + j-- + } + + if j > 1 { + // Create parent + parent := path[0 : j-1] + if parent != filepath.VolumeName(parent) { + err = os.MkdirAll(parent, perm) + if err != nil { + return err + } + } + } + + // Parent now exists; invoke Mkdir and use its result. + err = os.Mkdir(path, perm) + if err != nil { + // Handle arguments like "foo/." by + // double-checking that directory doesn't exist. + dir, err1 := os.Lstat(path) + if err1 == nil && dir.IsDir() { + return nil + } + return err + } + return nil +} + +func (f *BasicFilesystem) Unhide(name string) error { + name, err := f.rooted(name) + if err != nil { + return err + } + p, err := syscall.UTF16PtrFromString(name) + if err != nil { + return err + } + + attrs, err := syscall.GetFileAttributes(p) + if err != nil { + return err + } + + attrs &^= syscall.FILE_ATTRIBUTE_HIDDEN + return syscall.SetFileAttributes(p, attrs) +} + +func (f *BasicFilesystem) Hide(name string) error { + name, err := f.rooted(name) + if err != nil { + return err + } + p, err := syscall.UTF16PtrFromString(name) + if err != nil { + return err + } + + attrs, err := syscall.GetFileAttributes(p) + if err != nil { + return err + } + + attrs |= syscall.FILE_ATTRIBUTE_HIDDEN + return syscall.SetFileAttributes(p, attrs) +} + +func (f *BasicFilesystem) Roots() ([]string, error) { + kernel32, err := syscall.LoadDLL("kernel32.dll") + if err != nil { + return nil, err + } + getLogicalDriveStringsHandle, err := kernel32.FindProc("GetLogicalDriveStringsA") + if err != nil { + return nil, err + } + + buffer := [1024]byte{} + bufferSize := uint32(len(buffer)) + + hr, _, _ := getLogicalDriveStringsHandle.Call(uintptr(unsafe.Pointer(&bufferSize)), uintptr(unsafe.Pointer(&buffer))) + if hr == 0 { + return nil, fmt.Errorf("Syscall failed") + } + + var drives []string + parts := bytes.Split(buffer[:], []byte{0}) + for _, part := range parts { + if len(part) == 0 { + break + } + drives = append(drives, string(part)) + } + + return drives, nil +} diff --git a/lib/fs/debug.go b/lib/fs/debug.go new file mode 100644 index 000000000..52c3687e9 --- /dev/null +++ b/lib/fs/debug.go @@ -0,0 +1,22 @@ +// Copyright (C) 2015 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package fs + +import ( + "os" + "strings" + + "github.com/syncthing/syncthing/lib/logger" +) + +var ( + l = logger.DefaultLogger.NewFacility("filesystem", "Filesystem access") +) + +func init() { + l.SetDebug("filesystem", strings.Contains(os.Getenv("STTRACE"), "filesystem") || os.Getenv("STTRACE") == "all") +} diff --git a/lib/fs/errorfs.go b/lib/fs/errorfs.go new file mode 100644 index 000000000..8d38561c1 --- /dev/null +++ b/lib/fs/errorfs.go @@ -0,0 +1,41 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package fs + +import "time" + +type errorFilesystem struct { + err error + fsType FilesystemType + uri string +} + +func (fs *errorFilesystem) Chmod(name string, mode FileMode) error { return fs.err } +func (fs *errorFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error { return fs.err } +func (fs *errorFilesystem) Create(name string) (File, error) { return nil, fs.err } +func (fs *errorFilesystem) CreateSymlink(name, target string) error { return fs.err } +func (fs *errorFilesystem) DirNames(name string) ([]string, error) { return nil, fs.err } +func (fs *errorFilesystem) Lstat(name string) (FileInfo, error) { return nil, fs.err } +func (fs *errorFilesystem) Mkdir(name string, perm FileMode) error { return fs.err } +func (fs *errorFilesystem) MkdirAll(name string, perm FileMode) error { return fs.err } +func (fs *errorFilesystem) Open(name string) (File, error) { return nil, fs.err } +func (fs *errorFilesystem) OpenFile(string, int, FileMode) (File, error) { return nil, fs.err } +func (fs *errorFilesystem) ReadSymlink(name string) (string, error) { return "", fs.err } +func (fs *errorFilesystem) Remove(name string) error { return fs.err } +func (fs *errorFilesystem) RemoveAll(name string) error { return fs.err } +func (fs *errorFilesystem) Rename(oldname, newname string) error { return fs.err } +func (fs *errorFilesystem) Stat(name string) (FileInfo, error) { return nil, fs.err } +func (fs *errorFilesystem) SymlinksSupported() bool { return false } +func (fs *errorFilesystem) Walk(root string, walkFn WalkFunc) error { return fs.err } +func (fs *errorFilesystem) Unhide(name string) error { return fs.err } +func (fs *errorFilesystem) Hide(name string) error { return fs.err } +func (fs *errorFilesystem) Glob(pattern string) ([]string, error) { return nil, fs.err } +func (fs *errorFilesystem) SyncDir(name string) error { return fs.err } +func (fs *errorFilesystem) Roots() ([]string, error) { return nil, fs.err } +func (fs *errorFilesystem) Usage(name string) (Usage, error) { return Usage{}, fs.err } +func (fs *errorFilesystem) Type() FilesystemType { return fs.fsType } +func (fs *errorFilesystem) URI() string { return fs.uri } diff --git a/lib/fs/filesystem.go b/lib/fs/filesystem.go index 025f911d3..ccc7b5e8c 100644 --- a/lib/fs/filesystem.go +++ b/lib/fs/filesystem.go @@ -7,6 +7,7 @@ package fs import ( + "errors" "io" "os" "path/filepath" @@ -22,23 +23,38 @@ type Filesystem interface { DirNames(name string) ([]string, error) Lstat(name string) (FileInfo, error) Mkdir(name string, perm FileMode) error + MkdirAll(name string, perm FileMode) error Open(name string) (File, error) + OpenFile(name string, flags int, mode FileMode) (File, error) ReadSymlink(name string) (string, error) Remove(name string) error + RemoveAll(name string) error Rename(oldname, newname string) error Stat(name string) (FileInfo, error) SymlinksSupported() bool Walk(root string, walkFn WalkFunc) error + Hide(name string) error + Unhide(name string) error + Glob(pattern string) ([]string, error) + Roots() ([]string, error) + Usage(name string) (Usage, error) + Type() FilesystemType + URI() string } // The File interface abstracts access to a regular file, being a somewhat // smaller interface than os.File type File interface { - io.Reader - io.WriterAt io.Closer + io.Reader + io.ReaderAt + io.Seeker + io.Writer + io.WriterAt + Name() string Truncate(size int64) error Stat() (FileInfo, error) + Sync() error } // The FileInfo interface is almost the same as os.FileInfo, but with the @@ -59,12 +75,27 @@ type FileInfo interface { // FileMode is similar to os.FileMode type FileMode uint32 -// ModePerm is the equivalent of os.ModePerm -const ModePerm = FileMode(os.ModePerm) +// Usage represents filesystem space usage +type Usage struct { + Free int64 + Total int64 +} -// DefaultFilesystem is the fallback to use when nothing explicitly has -// been passed. -var DefaultFilesystem Filesystem = NewWalkFilesystem(NewBasicFilesystem()) +// Equivalents from os package. + +const ModePerm = FileMode(os.ModePerm) +const ModeSetgid = FileMode(os.ModeSetgid) +const ModeSetuid = FileMode(os.ModeSetuid) +const ModeSticky = FileMode(os.ModeSticky) +const PathSeparator = os.PathSeparator +const OptAppend = os.O_APPEND +const OptCreate = os.O_CREATE +const OptExclusive = os.O_EXCL +const OptReadOnly = os.O_RDONLY +const OptReadWrite = os.O_RDWR +const OptSync = os.O_SYNC +const OptTruncate = os.O_TRUNC +const OptWriteOnly = os.O_WRONLY // SkipDir is used as a return value from WalkFuncs to indicate that // the directory named in the call is to be skipped. It is not returned @@ -76,3 +107,29 @@ var IsExist = os.IsExist // IsNotExist is the equivalent of os.IsNotExist var IsNotExist = os.IsNotExist + +// IsPermission is the equivalent of os.IsPermission +var IsPermission = os.IsPermission + +// IsPathSeparator is the equivalent of os.IsPathSeparator +var IsPathSeparator = os.IsPathSeparator + +func NewFilesystem(fsType FilesystemType, uri string) Filesystem { + var fs Filesystem + switch fsType { + case FilesystemTypeBasic: + fs = NewWalkFilesystem(newBasicFilesystem(uri)) + default: + l.Debugln("Unknown filesystem", fsType, uri) + fs = &errorFilesystem{ + fsType: fsType, + uri: uri, + err: errors.New("filesystem with type " + fsType.String() + " does not exist."), + } + } + + if l.ShouldDebug("filesystem") { + fs = &logFilesystem{fs} + } + return fs +} diff --git a/lib/fs/logfs.go b/lib/fs/logfs.go new file mode 100644 index 000000000..1a2df6256 --- /dev/null +++ b/lib/fs/logfs.go @@ -0,0 +1,158 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package fs + +import ( + "fmt" + "path/filepath" + "runtime" + "time" +) + +type logFilesystem struct { + Filesystem +} + +func getCaller() string { + _, file, line, ok := runtime.Caller(2) + if !ok { + return "unknown" + } + return fmt.Sprintf("%s:%d", filepath.Base(file), line) +} + +func (fs *logFilesystem) Chmod(name string, mode FileMode) error { + err := fs.Filesystem.Chmod(name, mode) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Chmod", name, mode, err) + return err +} + +func (fs *logFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error { + err := fs.Filesystem.Chtimes(name, atime, mtime) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Chtimes", name, atime, mtime, err) + return err +} + +func (fs *logFilesystem) Create(name string) (File, error) { + file, err := fs.Filesystem.Create(name) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Create", name, file, err) + return file, err +} + +func (fs *logFilesystem) CreateSymlink(name, target string) error { + err := fs.Filesystem.CreateSymlink(name, target) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "CreateSymlink", name, target, err) + return err +} + +func (fs *logFilesystem) DirNames(name string) ([]string, error) { + names, err := fs.Filesystem.DirNames(name) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "DirNames", name, names, err) + return names, err +} + +func (fs *logFilesystem) Lstat(name string) (FileInfo, error) { + info, err := fs.Filesystem.Lstat(name) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Lstat", name, info, err) + return info, err +} + +func (fs *logFilesystem) Mkdir(name string, perm FileMode) error { + err := fs.Filesystem.Mkdir(name, perm) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Mkdir", name, perm, err) + return err +} + +func (fs *logFilesystem) MkdirAll(name string, perm FileMode) error { + err := fs.Filesystem.MkdirAll(name, perm) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "MkdirAll", name, perm, err) + return err +} + +func (fs *logFilesystem) Open(name string) (File, error) { + file, err := fs.Filesystem.Open(name) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Open", name, file, err) + return file, err +} + +func (fs *logFilesystem) OpenFile(name string, flags int, mode FileMode) (File, error) { + file, err := fs.Filesystem.OpenFile(name, flags, mode) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "OpenFile", name, flags, mode, file, err) + return file, err +} + +func (fs *logFilesystem) ReadSymlink(name string) (string, error) { + target, err := fs.Filesystem.ReadSymlink(name) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "ReadSymlink", name, target, err) + return target, err +} + +func (fs *logFilesystem) Remove(name string) error { + err := fs.Filesystem.Remove(name) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Remove", name, err) + return err +} + +func (fs *logFilesystem) RemoveAll(name string) error { + err := fs.Filesystem.RemoveAll(name) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "RemoveAll", name, err) + return err +} + +func (fs *logFilesystem) Rename(oldname, newname string) error { + err := fs.Filesystem.Rename(oldname, newname) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Rename", oldname, newname, err) + return err +} + +func (fs *logFilesystem) Stat(name string) (FileInfo, error) { + info, err := fs.Filesystem.Stat(name) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Stat", name, info, err) + return info, err +} + +func (fs *logFilesystem) SymlinksSupported() bool { + supported := fs.Filesystem.SymlinksSupported() + l.Debugln(getCaller(), fs.Type(), fs.URI(), "SymlinksSupported", supported) + return supported +} + +func (fs *logFilesystem) Walk(root string, walkFn WalkFunc) error { + err := fs.Filesystem.Walk(root, walkFn) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Walk", root, walkFn, err) + return err +} + +func (fs *logFilesystem) Unhide(name string) error { + err := fs.Filesystem.Unhide(name) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Unhide", name, err) + return err +} + +func (fs *logFilesystem) Hide(name string) error { + err := fs.Filesystem.Hide(name) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Hide", name, err) + return err +} + +func (fs *logFilesystem) Glob(name string) ([]string, error) { + names, err := fs.Filesystem.Glob(name) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Glob", name, names, err) + return names, err +} + +func (fs *logFilesystem) Roots() ([]string, error) { + roots, err := fs.Filesystem.Roots() + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Roots", roots, err) + return roots, err +} + +func (fs *logFilesystem) Usage(name string) (Usage, error) { + usage, err := fs.Filesystem.Usage(name) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Usage", name, usage, err) + return usage, err +} diff --git a/lib/fs/mtimefs.go b/lib/fs/mtimefs.go index ab6afcb31..1719b8cec 100644 --- a/lib/fs/mtimefs.go +++ b/lib/fs/mtimefs.go @@ -6,12 +6,7 @@ package fs -import ( - "os" - "time" - - "github.com/syncthing/syncthing/lib/osutil" -) +import "time" // The database is where we store the virtual mtimes type database interface { @@ -20,36 +15,34 @@ type database interface { Delete(key string) } -// variable so that we can mock it for testing -var osChtimes = os.Chtimes - // The MtimeFS is a filesystem with nanosecond mtime precision, regardless // of what shenanigans the underlying filesystem gets up to. A nil MtimeFS // just does the underlying operations with no additions. type MtimeFS struct { Filesystem - db database + chtimes func(string, time.Time, time.Time) error + db database } func NewMtimeFS(underlying Filesystem, db database) *MtimeFS { return &MtimeFS{ Filesystem: underlying, + chtimes: underlying.Chtimes, // for mocking it out in the tests db: db, } } func (f *MtimeFS) Chtimes(name string, atime, mtime time.Time) error { if f == nil { - return osChtimes(name, atime, mtime) + return f.chtimes(name, atime, mtime) } // Do a normal Chtimes call, don't care if it succeeds or not. - osChtimes(name, atime, mtime) + f.chtimes(name, atime, mtime) // Stat the file to see what happened. Here we *do* return an error, - // because it might be "does not exist" or similar. osutil.Lstat is the - // souped up version to account for Android breakage. - info, err := osutil.Lstat(name) + // because it might be "does not exist" or similar. + info, err := f.Filesystem.Lstat(name) if err != nil { return err } diff --git a/lib/fs/mtimefs_test.go b/lib/fs/mtimefs_test.go index 1e1d6fb7f..80514e1cb 100644 --- a/lib/fs/mtimefs_test.go +++ b/lib/fs/mtimefs_test.go @@ -25,22 +25,22 @@ func TestMtimeFS(t *testing.T) { // a random time with nanosecond precision testTime := time.Unix(1234567890, 123456789) - mtimefs := NewMtimeFS(DefaultFilesystem, make(mapStore)) + mtimefs := NewMtimeFS(newBasicFilesystem("."), make(mapStore)) // Do one Chtimes call that will go through to the normal filesystem - osChtimes = os.Chtimes + mtimefs.chtimes = os.Chtimes if err := mtimefs.Chtimes("testdata/exists0", testTime, testTime); err != nil { t.Error("Should not have failed:", err) } // Do one call that gets an error back from the underlying Chtimes - osChtimes = failChtimes + mtimefs.chtimes = failChtimes if err := mtimefs.Chtimes("testdata/exists1", testTime, testTime); err != nil { t.Error("Should not have failed:", err) } // Do one call that gets struck by an exceptionally evil Chtimes - osChtimes = evilChtimes + mtimefs.chtimes = evilChtimes if err := mtimefs.Chtimes("testdata/exists2", testTime, testTime); err != nil { t.Error("Should not have failed:", err) } diff --git a/lib/fs/types.go b/lib/fs/types.go new file mode 100644 index 000000000..b75a95991 --- /dev/null +++ b/lib/fs/types.go @@ -0,0 +1,36 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package fs + +type FilesystemType int + +const ( + FilesystemTypeBasic FilesystemType = iota // default is basic +) + +func (t FilesystemType) String() string { + switch t { + case FilesystemTypeBasic: + return "basic" + default: + return "unknown" + } +} + +func (t FilesystemType) MarshalText() ([]byte, error) { + return []byte(t.String()), nil +} + +func (t *FilesystemType) UnmarshalText(bs []byte) error { + switch string(bs) { + case "basic": + *t = FilesystemTypeBasic + default: + *t = FilesystemTypeBasic + } + return nil +} diff --git a/lib/fs/util.go b/lib/fs/util.go new file mode 100644 index 000000000..f0b6bf27c --- /dev/null +++ b/lib/fs/util.go @@ -0,0 +1,55 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package fs + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" +) + +var errNoHome = errors.New("no home directory found - set $HOME (or the platform equivalent)") + +func ExpandTilde(path string) (string, error) { + if path == "~" { + return getHomeDir() + } + + path = filepath.FromSlash(path) + if !strings.HasPrefix(path, fmt.Sprintf("~%c", PathSeparator)) { + return path, nil + } + + home, err := getHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, path[2:]), nil +} + +func getHomeDir() (string, error) { + var home string + + switch runtime.GOOS { + case "windows": + home = filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath")) + if home == "" { + home = os.Getenv("UserProfile") + } + default: + home = os.Getenv("HOME") + } + + if home == "" { + return "", errNoHome + } + + return home, nil +} diff --git a/lib/fs/walkfs.go b/lib/fs/walkfs.go index e545cabb3..9b4a7830d 100644 --- a/lib/fs/walkfs.go +++ b/lib/fs/walkfs.go @@ -28,16 +28,16 @@ import "path/filepath" // Walk skips the remaining files in the containing directory. type WalkFunc func(path string, info FileInfo, err error) error -type WalkFilesystem struct { +type walkFilesystem struct { Filesystem } -func NewWalkFilesystem(next Filesystem) *WalkFilesystem { - return &WalkFilesystem{next} +func NewWalkFilesystem(next Filesystem) Filesystem { + return &walkFilesystem{next} } // walk recursively descends path, calling walkFn. -func (f *WalkFilesystem) walk(path string, info FileInfo, walkFn WalkFunc) error { +func (f *walkFilesystem) walk(path string, info FileInfo, walkFn WalkFunc) error { err := walkFn(path, info, nil) if err != nil { if info.IsDir() && err == SkipDir { @@ -80,7 +80,7 @@ func (f *WalkFilesystem) walk(path string, info FileInfo, walkFn WalkFunc) error // order, which makes the output deterministic but means that for very // large directories Walk can be inefficient. // Walk does not follow symbolic links. -func (f *WalkFilesystem) Walk(root string, walkFn WalkFunc) error { +func (f *walkFilesystem) Walk(root string, walkFn WalkFunc) error { info, err := f.Lstat(root) if err != nil { return walkFn(root, nil, err) diff --git a/lib/ignore/ignore.go b/lib/ignore/ignore.go index 44adbc741..b245b09d7 100644 --- a/lib/ignore/ignore.go +++ b/lib/ignore/ignore.go @@ -12,13 +12,13 @@ import ( "crypto/md5" "fmt" "io" - "os" "path/filepath" "runtime" "strings" "time" "github.com/gobwas/glob" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/osutil" "github.com/syncthing/syncthing/lib/sync" ) @@ -77,6 +77,7 @@ type ChangeDetector interface { } type Matcher struct { + fs fs.Filesystem lines []string // exact lines read from .stignore patterns []Pattern // patterns including those from included files withCache bool @@ -105,8 +106,9 @@ func WithChangeDetector(cd ChangeDetector) Option { } } -func New(opts ...Option) *Matcher { +func New(fs fs.Filesystem, opts ...Option) *Matcher { m := &Matcher{ + fs: fs, stop: make(chan struct{}), mut: sync.NewMutex(), } @@ -114,7 +116,7 @@ func New(opts ...Option) *Matcher { opt(m) } if m.changeDetector == nil { - m.changeDetector = newModtimeChecker() + m.changeDetector = newModtimeChecker(fs) } if m.withCache { go m.clean(2 * time.Hour) @@ -130,7 +132,7 @@ func (m *Matcher) Load(file string) error { return nil } - fd, err := os.Open(file) + fd, err := m.fs.Open(file) if err != nil { m.parseLocked(&bytes.Buffer{}, file) return err @@ -156,7 +158,7 @@ func (m *Matcher) Parse(r io.Reader, file string) error { } func (m *Matcher) parseLocked(r io.Reader, file string) error { - lines, patterns, err := parseIgnoreFile(r, file, m.changeDetector) + lines, patterns, err := parseIgnoreFile(m.fs, r, file, m.changeDetector) // Error is saved and returned at the end. We process the patterns // (possibly blank) anyway. @@ -298,12 +300,12 @@ func hashPatterns(patterns []Pattern) string { return fmt.Sprintf("%x", h.Sum(nil)) } -func loadIgnoreFile(file string, cd ChangeDetector) ([]string, []Pattern, error) { +func loadIgnoreFile(fs fs.Filesystem, file string, cd ChangeDetector) ([]string, []Pattern, error) { if cd.Seen(file) { return nil, nil, fmt.Errorf("multiple include of ignore file %q", file) } - fd, err := os.Open(file) + fd, err := fs.Open(file) if err != nil { return nil, nil, err } @@ -316,10 +318,10 @@ func loadIgnoreFile(file string, cd ChangeDetector) ([]string, []Pattern, error) cd.Remember(file, info.ModTime()) - return parseIgnoreFile(fd, file, cd) + return parseIgnoreFile(fs, fd, file, cd) } -func parseIgnoreFile(fd io.Reader, currentFile string, cd ChangeDetector) ([]string, []Pattern, error) { +func parseIgnoreFile(fs fs.Filesystem, fd io.Reader, currentFile string, cd ChangeDetector) ([]string, []Pattern, error) { var lines []string var patterns []Pattern @@ -386,7 +388,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, cd ChangeDetector) ([]str } else if strings.HasPrefix(line, "#include ") { includeRel := line[len("#include "):] includeFile := filepath.Join(filepath.Dir(currentFile), includeRel) - _, includePatterns, err := loadIgnoreFile(includeFile, cd) + _, includePatterns, err := loadIgnoreFile(fs, includeFile, cd) if err != nil { return fmt.Errorf("include of %q: %v", includeRel, err) } @@ -450,7 +452,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, cd ChangeDetector) ([]str // path must be clean (i.e., in canonical shortest form). func IsInternal(file string) bool { internals := []string{".stfolder", ".stignore", ".stversions"} - pathSep := string(os.PathSeparator) + pathSep := string(fs.PathSeparator) for _, internal := range internals { if file == internal { return true @@ -463,8 +465,8 @@ func IsInternal(file string) bool { } // WriteIgnores is a convenience function to avoid code duplication -func WriteIgnores(path string, content []string) error { - fd, err := osutil.CreateAtomic(path) +func WriteIgnores(filesystem fs.Filesystem, path string, content []string) error { + fd, err := osutil.CreateAtomicFilesystem(filesystem, path) if err != nil { return err } @@ -476,18 +478,20 @@ func WriteIgnores(path string, content []string) error { if err := fd.Close(); err != nil { return err } - osutil.HideFile(path) + filesystem.Hide(path) return nil } // modtimeChecker is the default implementation of ChangeDetector type modtimeChecker struct { + fs fs.Filesystem modtimes map[string]time.Time } -func newModtimeChecker() *modtimeChecker { +func newModtimeChecker(fs fs.Filesystem) *modtimeChecker { return &modtimeChecker{ + fs: fs, modtimes: map[string]time.Time{}, } } @@ -507,7 +511,7 @@ func (c *modtimeChecker) Reset() { func (c *modtimeChecker) Changed() bool { for name, modtime := range c.modtimes { - info, err := os.Stat(name) + info, err := c.fs.Stat(name) if err != nil { return true } diff --git a/lib/ignore/ignore_test.go b/lib/ignore/ignore_test.go index 44dcebf9d..a1274b358 100644 --- a/lib/ignore/ignore_test.go +++ b/lib/ignore/ignore_test.go @@ -15,11 +15,14 @@ import ( "runtime" "testing" "time" + + "github.com/syncthing/syncthing/lib/fs" + "github.com/syncthing/syncthing/lib/osutil" ) func TestIgnore(t *testing.T) { - pats := New(WithCache(true)) - err := pats.Load("testdata/.stignore") + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), WithCache(true)) + err := pats.Load(".stignore") if err != nil { t.Fatal(err) } @@ -68,7 +71,7 @@ func TestExcludes(t *testing.T) { i*2 !ign2 ` - pats := New(WithCache(true)) + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := pats.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Fatal(err) @@ -113,7 +116,7 @@ func TestFlagOrder(t *testing.T) { (?i)(?d)(?d)!ign9 (?d)(?d)!ign10 ` - pats := New(WithCache(true)) + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := pats.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Fatal(err) @@ -148,7 +151,7 @@ func TestDeletables(t *testing.T) { ign7 (?i)ign8 ` - pats := New(WithCache(true)) + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := pats.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Fatal(err) @@ -187,7 +190,7 @@ func TestBadPatterns(t *testing.T) { } for _, pat := range badPatterns { - err := New(WithCache(true)).Parse(bytes.NewBufferString(pat), ".stignore") + err := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)).Parse(bytes.NewBufferString(pat), ".stignore") if err == nil { t.Errorf("No error for pattern %q", pat) } @@ -195,7 +198,7 @@ func TestBadPatterns(t *testing.T) { } func TestCaseSensitivity(t *testing.T) { - ign := New(WithCache(true)) + ign := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := ign.Parse(bytes.NewBufferString("test"), ".stignore") if err != nil { t.Error(err) @@ -225,29 +228,36 @@ func TestCaseSensitivity(t *testing.T) { } func TestCaching(t *testing.T) { - fd1, err := ioutil.TempFile("", "") + dir, err := ioutil.TempDir("", "") if err != nil { t.Fatal(err) } - fd2, err := ioutil.TempFile("", "") + fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir) + + fd1, err := osutil.TempFile(fs, "", "") + if err != nil { + t.Fatal(err) + } + + fd2, err := osutil.TempFile(fs, "", "") if err != nil { t.Fatal(err) } defer fd1.Close() defer fd2.Close() - defer os.Remove(fd1.Name()) - defer os.Remove(fd2.Name()) + defer fs.Remove(fd1.Name()) + defer fs.Remove(fd2.Name()) - _, err = fd1.WriteString("/x/\n#include " + filepath.Base(fd2.Name()) + "\n") + _, err = fd1.Write([]byte("/x/\n#include " + filepath.Base(fd2.Name()) + "\n")) if err != nil { t.Fatal(err) } - fd2.WriteString("/y/\n") + fd2.Write([]byte("/y/\n")) - pats := New(WithCache(true)) + pats := New(fs, WithCache(true)) err = pats.Load(fd1.Name()) if err != nil { t.Fatal(err) @@ -280,10 +290,10 @@ func TestCaching(t *testing.T) { // Modify the include file, expect empty cache. Ensure the timestamp on // the file changes. - fd2.WriteString("/z/\n") + fd2.Write([]byte("/z/\n")) fd2.Sync() fakeTime := time.Now().Add(5 * time.Second) - os.Chtimes(fd2.Name(), fakeTime, fakeTime) + fs.Chtimes(fd2.Name(), fakeTime, fakeTime) err = pats.Load(fd1.Name()) if err != nil { @@ -312,10 +322,10 @@ func TestCaching(t *testing.T) { // Modify the root file, expect cache to be invalidated - fd1.WriteString("/a/\n") + fd1.Write([]byte("/a/\n")) fd1.Sync() fakeTime = time.Now().Add(5 * time.Second) - os.Chtimes(fd1.Name(), fakeTime, fakeTime) + fs.Chtimes(fd1.Name(), fakeTime, fakeTime) err = pats.Load(fd1.Name()) if err != nil { @@ -354,7 +364,7 @@ func TestCommentsAndBlankLines(t *testing.T) { ` - pats := New(WithCache(true)) + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := pats.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Error(err) @@ -382,7 +392,7 @@ flamingo *.crow *.crow ` - pats := New() + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, ".")) err := pats.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { b.Error(err) @@ -411,20 +421,27 @@ flamingo *.crow ` // Caches per file, hence write the patterns to a file. - fd, err := ioutil.TempFile("", "") + dir, err := ioutil.TempDir("", "") if err != nil { b.Fatal(err) } - _, err = fd.WriteString(stignore) + fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir) + + fd, err := osutil.TempFile(fs, "", "") + if err != nil { + b.Fatal(err) + } + + _, err = fd.Write([]byte(stignore)) defer fd.Close() - defer os.Remove(fd.Name()) + defer fs.Remove(fd.Name()) if err != nil { b.Fatal(err) } // Load the patterns - pats := New(WithCache(true)) + pats := New(fs, WithCache(true)) err = pats.Load(fd.Name()) if err != nil { b.Fatal(err) @@ -445,22 +462,29 @@ flamingo } func TestCacheReload(t *testing.T) { - fd, err := ioutil.TempFile("", "") + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + + fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir) + + fd, err := osutil.TempFile(fs, "", "") if err != nil { t.Fatal(err) } defer fd.Close() - defer os.Remove(fd.Name()) + defer fs.Remove(fd.Name()) // Ignore file matches f1 and f2 - _, err = fd.WriteString("f1\nf2\n") + _, err = fd.Write([]byte("f1\nf2\n")) if err != nil { t.Fatal(err) } - pats := New(WithCache(true)) + pats := New(fs, WithCache(true)) err = pats.Load(fd.Name()) if err != nil { t.Fatal(err) @@ -488,13 +512,13 @@ func TestCacheReload(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = fd.WriteString("f1\nf3\n") + _, err = fd.Write([]byte("f1\nf3\n")) if err != nil { t.Fatal(err) } fd.Sync() fakeTime := time.Now().Add(5 * time.Second) - os.Chtimes(fd.Name(), fakeTime, fakeTime) + fs.Chtimes(fd.Name(), fakeTime, fakeTime) err = pats.Load(fd.Name()) if err != nil { @@ -515,7 +539,7 @@ func TestCacheReload(t *testing.T) { } func TestHash(t *testing.T) { - p1 := New(WithCache(true)) + p1 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := p1.Load("testdata/.stignore") if err != nil { t.Fatal(err) @@ -531,7 +555,7 @@ func TestHash(t *testing.T) { /ffile lost+found ` - p2 := New(WithCache(true)) + p2 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err = p2.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Fatal(err) @@ -546,7 +570,7 @@ func TestHash(t *testing.T) { /ffile lost+found ` - p3 := New(WithCache(true)) + p3 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err = p3.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Fatal(err) @@ -570,7 +594,7 @@ func TestHash(t *testing.T) { } func TestHashOfEmpty(t *testing.T) { - p1 := New(WithCache(true)) + p1 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := p1.Load("testdata/.stignore") if err != nil { t.Fatal(err) @@ -608,7 +632,7 @@ func TestWindowsPatterns(t *testing.T) { a/b c\d ` - pats := New(WithCache(true)) + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := pats.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Fatal(err) @@ -633,7 +657,7 @@ func TestAutomaticCaseInsensitivity(t *testing.T) { A/B c/d ` - pats := New(WithCache(true)) + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := pats.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Fatal(err) @@ -652,7 +676,7 @@ func TestCommas(t *testing.T) { foo,bar.txt {baz,quux}.txt ` - pats := New(WithCache(true)) + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := pats.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Fatal(err) @@ -683,7 +707,7 @@ func TestIssue3164(t *testing.T) { (?d)(?i)/foo (?d)(?i)**/bar ` - pats := New(WithCache(true)) + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := pats.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Fatal(err) @@ -719,7 +743,7 @@ func TestIssue3174(t *testing.T) { stignore := ` *ä* ` - pats := New(WithCache(true)) + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := pats.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Fatal(err) @@ -734,7 +758,7 @@ func TestIssue3639(t *testing.T) { stignore := ` foo/ ` - pats := New(WithCache(true)) + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := pats.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Fatal(err) @@ -767,7 +791,7 @@ func TestIssue3674(t *testing.T) { {"as/dc", true}, } - pats := New(WithCache(true)) + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := pats.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Fatal(err) @@ -799,7 +823,7 @@ func TestGobwasGlobIssue18(t *testing.T) { {"bbaa", false}, } - pats := New(WithCache(true)) + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := pats.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Fatal(err) @@ -859,7 +883,7 @@ func TestRoot(t *testing.T) { {"b", true}, } - pats := New(WithCache(true)) + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := pats.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Fatal(err) @@ -876,12 +900,12 @@ func TestRoot(t *testing.T) { func TestLines(t *testing.T) { stignore := ` #include testdata/excludes - + !/a /* ` - pats := New(WithCache(true)) + pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)) err := pats.Parse(bytes.NewBufferString(stignore), ".stignore") if err != nil { t.Fatal(err) diff --git a/lib/model/model.go b/lib/model/model.go index 4c1c77ac2..ce9ac5e30 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -14,7 +14,6 @@ import ( "fmt" "io" "net" - "os" "path/filepath" "reflect" "runtime" @@ -81,6 +80,7 @@ type Model struct { clientVersion string folderCfgs map[string]config.FolderConfiguration // folder -> cfg + folderFs map[string]fs.Filesystem // folder -> fs folderFiles map[string]*db.FileSet // folder -> files folderDevices folderDeviceSet // folder -> deviceIDs deviceFolders map[protocol.DeviceID][]string // deviceID -> folders @@ -99,21 +99,18 @@ type Model struct { pmut sync.RWMutex // protects the above } -type folderFactory func(*Model, config.FolderConfiguration, versioner.Versioner, *fs.MtimeFS) service +type folderFactory func(*Model, config.FolderConfiguration, versioner.Versioner, fs.Filesystem) service var ( folderFactories = make(map[config.FolderType]folderFactory, 0) ) var ( - errFolderPathEmpty = errors.New("folder path empty") errFolderPathMissing = errors.New("folder path missing") errFolderMarkerMissing = errors.New("folder marker missing") - errInvalidFilename = errors.New("filename is invalid") errDeviceUnknown = errors.New("unknown device") errDevicePaused = errors.New("device is paused") errDeviceIgnored = errors.New("device is ignored") - errNotRelative = errors.New("not a relative path") errFolderPaused = errors.New("folder is paused") errFolderMissing = errors.New("no such folder") errNetworkNotAllowed = errors.New("network not allowed") @@ -140,6 +137,7 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, clientName, clientVersi clientName: clientName, clientVersion: clientVersion, folderCfgs: make(map[string]config.FolderConfiguration), + folderFs: make(map[string]fs.Filesystem), folderFiles: make(map[string]*db.FileSet), folderDevices: make(folderDeviceSet), deviceFolders: make(map[protocol.DeviceID][]string), @@ -245,7 +243,7 @@ func (m *Model) startFolderLocked(folder string) config.FolderType { l.Fatalf("Requested versioning type %q that does not exist", cfg.Versioning.Type) } - ver = versionerFactory(folder, cfg.Path(), cfg.Versioning.Params) + ver = versionerFactory(folder, cfg.Filesystem(), cfg.Versioning.Params) if service, ok := ver.(suture.Service); ok { // The versioner implements the suture.Service interface, so // expects to be run in the background in addition to being called @@ -271,7 +269,12 @@ func (m *Model) warnAboutOverwritingProtectedFiles(folder string) { return } - folderLocation := m.folderCfgs[folder].Path() + // This is a bit of a hack. + ffs := m.folderCfgs[folder].Filesystem() + if ffs.Type() != fs.FilesystemTypeBasic { + return + } + folderLocation := ffs.URI() ignores := m.folderIgnores[folder] var filesAtRisk []string @@ -300,6 +303,10 @@ func (m *Model) AddFolder(cfg config.FolderConfiguration) { panic("cannot add empty folder id") } + if len(cfg.Path) == 0 { + panic("cannot add empty folder path") + } + m.fmut.Lock() m.addFolderLocked(cfg) m.fmut.Unlock() @@ -307,15 +314,16 @@ func (m *Model) AddFolder(cfg config.FolderConfiguration) { func (m *Model) addFolderLocked(cfg config.FolderConfiguration) { m.folderCfgs[cfg.ID] = cfg - m.folderFiles[cfg.ID] = db.NewFileSet(cfg.ID, m.db) + folderFs := cfg.Filesystem() + m.folderFiles[cfg.ID] = db.NewFileSet(cfg.ID, folderFs, m.db) for _, device := range cfg.Devices { m.folderDevices.set(device.DeviceID, cfg.ID) m.deviceFolders[device.DeviceID] = append(m.deviceFolders[device.DeviceID], cfg.ID) } - ignores := ignore.New(ignore.WithCache(m.cacheIgnoredFiles)) - if err := ignores.Load(filepath.Join(cfg.Path(), ".stignore")); err != nil && !os.IsNotExist(err) { + ignores := ignore.New(folderFs, ignore.WithCache(m.cacheIgnoredFiles)) + if err := ignores.Load(".stignore"); err != nil && !fs.IsNotExist(err) { l.Warnln("Loading ignores:", err) } m.folderIgnores[cfg.ID] = ignores @@ -327,8 +335,8 @@ func (m *Model) RemoveFolder(folder string) { // Delete syncthing specific files folderCfg := m.folderCfgs[folder] - folderPath := folderCfg.Path() - os.Remove(filepath.Join(folderPath, ".stfolder")) + fs := folderCfg.Filesystem() + fs.Remove(".stfolder") m.tearDownFolderLocked(folder) // Remove it from the database @@ -1139,16 +1147,10 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset } m.fmut.RLock() folderCfg := m.folderCfgs[folder] - folderPath := folderCfg.Path() folderIgnores := m.folderIgnores[folder] m.fmut.RUnlock() - fn, err := rootedJoinedPath(folderPath, name) - if err != nil { - // Request tries to escape! - l.Debugf("%v Invalid REQ(in) tries to escape: %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, len(buf)) - return protocol.ErrInvalid - } + folderFs := folderCfg.Filesystem() // Having passed the rootedJoinedPath check above, we know "name" is // acceptable relative to "folderPath" and in canonical form, so we can @@ -1164,7 +1166,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset return protocol.ErrNoSuchFile } - if err := osutil.TraversesSymlink(folderPath, filepath.Dir(name)); err != nil { + if err := osutil.TraversesSymlink(folderFs, filepath.Dir(name)); err != nil { l.Debugf("%v REQ(in) traversal check: %s - %s: %q / %q o=%d s=%d", m, err, deviceID, folder, name, offset, len(buf)) return protocol.ErrNoSuchFile } @@ -1172,29 +1174,29 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset // Only check temp files if the flag is set, and if we are set to advertise // the temp indexes. if fromTemporary && !folderCfg.DisableTempIndexes { - tempFn := filepath.Join(folderPath, ignore.TempName(name)) + tempFn := ignore.TempName(name) - if info, err := osutil.Lstat(tempFn); err != nil || !info.Mode().IsRegular() { + if info, err := folderFs.Lstat(tempFn); err != nil || !info.IsRegular() { // Reject reads for anything that doesn't exist or is something // other than a regular file. return protocol.ErrNoSuchFile } - if err := readOffsetIntoBuf(tempFn, offset, buf); err == nil { + if err := readOffsetIntoBuf(folderFs, tempFn, offset, buf); err == nil { return nil } // Fall through to reading from a non-temp file, just incase the temp // file has finished downloading. } - if info, err := osutil.Lstat(fn); err != nil || !info.Mode().IsRegular() { + if info, err := folderFs.Lstat(name); err != nil || !info.IsRegular() { // Reject reads for anything that doesn't exist or is something // other than a regular file. return protocol.ErrNoSuchFile } - err = readOffsetIntoBuf(fn, offset, buf) - if os.IsNotExist(err) { + err := readOffsetIntoBuf(folderFs, name, offset, buf) + if fs.IsNotExist(err) { return protocol.ErrNoSuchFile } else if err != nil { return protocol.ErrGeneric @@ -1259,9 +1261,8 @@ func (m *Model) GetIgnores(folder string) ([]string, []string, error) { } if cfg, ok := m.cfg.Folders()[folder]; ok { - matcher := ignore.New() - path := filepath.Join(cfg.Path(), ".stignore") - if err := matcher.Load(path); err != nil { + matcher := ignore.New(cfg.Filesystem()) + if err := matcher.Load(".stignore"); err != nil { return nil, nil, err } return matcher.Lines(), matcher.Patterns(), nil @@ -1276,7 +1277,7 @@ func (m *Model) SetIgnores(folder string, content []string) error { return fmt.Errorf("Folder %s does not exist", folder) } - if err := ignore.WriteIgnores(filepath.Join(cfg.Path(), ".stignore"), content); err != nil { + if err := ignore.WriteIgnores(cfg.Filesystem(), ".stignore", content); err != nil { l.Warnln("Saving .stignore:", err) return err } @@ -1610,8 +1611,6 @@ func (m *Model) updateLocals(folder string, fs []protocol.FileInfo) { } func (m *Model) diskChangeDetected(folderCfg config.FolderConfiguration, files []protocol.FileInfo, typeOfEvent events.EventType) { - path := strings.Replace(folderCfg.Path(), `\\?\`, "", 1) - for _, file := range files { objType := "file" action := "modified" @@ -1634,10 +1633,6 @@ func (m *Model) diskChangeDetected(folderCfg config.FolderConfiguration, files [ action = "deleted" } - // The full file path, adjusted to the local path separator character. Also - // for windows paths, strip unwanted chars from the front. - path := filepath.Join(path, filepath.FromSlash(file.Name)) - // Two different events can be fired here based on what EventType is passed into function events.Default.Log(typeOfEvent, map[string]string{ "folder": folderCfg.ID, @@ -1645,7 +1640,7 @@ func (m *Model) diskChangeDetected(folderCfg config.FolderConfiguration, files [ "label": folderCfg.Label, "action": action, "type": objType, - "path": path, + "path": filepath.FromSlash(file.Name), "modifiedBy": file.ModifiedBy.String(), }) } @@ -1738,20 +1733,17 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su // not relevant, we just want the dotdot escape detection here. For // historical reasons we may get paths that end in a slash. We // remove that first to allow the rootedJoinedPath to pass. - sub = strings.TrimRight(sub, string(os.PathSeparator)) - if _, err := rootedJoinedPath("root", sub); err != nil { - return errors.New("invalid subpath") - } + sub = strings.TrimRight(sub, string(fs.PathSeparator)) subDirs[i] = sub } m.fmut.Lock() - fs := m.folderFiles[folder] + fset := m.folderFiles[folder] folderCfg := m.folderCfgs[folder] ignores := m.folderIgnores[folder] runner, ok := m.folderRunners[folder] m.fmut.Unlock() - mtimefs := fs.MtimeFS() + mtimefs := fset.MtimeFS() // Check if the ignore patterns changed as part of scanning this folder. // If they did we should schedule a pull of the folder so that we @@ -1778,7 +1770,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su return err } - if err := ignores.Load(filepath.Join(folderCfg.Path(), ".stignore")); err != nil && !os.IsNotExist(err) { + if err := ignores.Load(".stignore"); err != nil && !fs.IsNotExist(err) { err = fmt.Errorf("loading ignores: %v", err) runner.setError(err) l.Infof("Stopping folder %s due to error: %s", folderCfg.Description(), err) @@ -1789,7 +1781,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su // directory, and don't scan subdirectories of things we've already // scanned. subDirs = unifySubs(subDirs, func(f string) bool { - _, ok := fs.Get(protocol.LocalDeviceID, f) + _, ok := fset.Get(protocol.LocalDeviceID, f) return ok }) @@ -1797,7 +1789,6 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su fchan, err := scanner.Walk(ctx, scanner.Config{ Folder: folderCfg.ID, - Dir: folderCfg.Path(), Subs: subDirs, Matcher: ignores, BlockSize: protocol.BlockSize, @@ -1860,7 +1851,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su for _, sub := range subDirs { var iterError error - fs.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool { + fset.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool { f := fi.(db.FileInfoTruncated) if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes { if err := m.CheckFolderHealth(folder); err != nil { @@ -1895,9 +1886,9 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su // The file is valid and not deleted. Lets check if it's // still here. - if _, err := mtimefs.Lstat(filepath.Join(folderCfg.Path(), f.Name)); err != nil { + if _, err := mtimefs.Lstat(f.Name); err != nil { // We don't specifically verify that the error is - // os.IsNotExist because there is a corner case when a + // fs.IsNotExist because there is a corner case when a // directory is suddenly transformed into a file. When that // happens, files that were in the directory (that is now a // file) are deleted but will return a confusing error ("not a @@ -2275,11 +2266,9 @@ func (m *Model) CheckFolderHealth(id string) error { // checkFolderPath returns nil if the folder path exists and has the marker file. func (m *Model) checkFolderPath(folder config.FolderConfiguration) error { - if folder.Path() == "" { - return errFolderPathEmpty - } + fs := folder.Filesystem() - if fi, err := os.Stat(folder.Path()); err != nil || !fi.IsDir() { + if fi, err := fs.Stat("."); err != nil || !fi.IsDir() { return errFolderPathMissing } @@ -2293,30 +2282,35 @@ func (m *Model) checkFolderPath(folder config.FolderConfiguration) error { // checkFolderFreeSpace returns nil if the folder has the required amount of // free space, or if folder free space checking is disabled. func (m *Model) checkFolderFreeSpace(folder config.FolderConfiguration) error { - return m.checkFreeSpace(folder.MinDiskFree, folder.Path()) + return m.checkFreeSpace(folder.MinDiskFree, folder.Filesystem()) } // checkHomeDiskFree returns nil if the home disk has the required amount of // free space, or if home disk free space checking is disabled. func (m *Model) checkHomeDiskFree() error { - return m.checkFreeSpace(m.cfg.Options().MinHomeDiskFree, m.cfg.ConfigPath()) + fs := fs.NewFilesystem(fs.FilesystemTypeBasic, filepath.Dir(m.cfg.ConfigPath())) + return m.checkFreeSpace(m.cfg.Options().MinHomeDiskFree, fs) } -func (m *Model) checkFreeSpace(req config.Size, path string) error { +func (m *Model) checkFreeSpace(req config.Size, fs fs.Filesystem) error { val := req.BaseValue() if val <= 0 { return nil } + usage, err := fs.Usage(".") + if err != nil { + return fmt.Errorf("failed to check available storage space") + } + if req.Percentage() { - free, err := osutil.DiskFreePercentage(path) - if err == nil && free < val { - return fmt.Errorf("insufficient space in %v: %f %% < %v", path, free, req) + freePct := (1 - float64(usage.Free)/float64(usage.Total)) * 100 + if err == nil && freePct < val { + return fmt.Errorf("insufficient space in %v %v: %f %% < %v", fs.Type(), fs.URI(), freePct, req) } } else { - free, err := osutil.DiskFreeBytes(path) - if err == nil && float64(free) < val { - return fmt.Errorf("insufficient space in %v: %v < %v", path, free, req) + if err == nil && float64(usage.Free) < val { + return fmt.Errorf("insufficient space in %v %v: %v < %v", fs.Type(), fs.URI(), usage.Free, req) } } @@ -2533,8 +2527,8 @@ func stringSliceWithout(ss []string, s string) []string { return ss } -func readOffsetIntoBuf(file string, offset int64, buf []byte) error { - fd, err := os.Open(file) +func readOffsetIntoBuf(fs fs.Filesystem, file string, offset int64, buf []byte) error { + fd, err := fs.Open(file) if err != nil { l.Debugln("readOffsetIntoBuf.Open", file, err) return err @@ -2585,7 +2579,7 @@ func simplifySortedPaths(subs []string) []string { next: for _, sub := range subs { for _, existing := range cleaned { - if sub == existing || strings.HasPrefix(sub, existing+string(os.PathSeparator)) { + if sub == existing || strings.HasPrefix(sub, existing+string(fs.PathSeparator)) { continue next } } @@ -2666,57 +2660,3 @@ func (s folderDeviceSet) sortedDevices(folder string) []protocol.DeviceID { sort.Sort(protocol.DeviceIDs(devs)) return devs } - -// rootedJoinedPath takes a root and a supposedly relative path inside that -// root and returns the joined path. An error is returned if the joined path -// is not in fact inside the root. -func rootedJoinedPath(root, rel string) (string, error) { - // The root must not be empty. - if root == "" { - return "", errInvalidFilename - } - - pathSep := string(os.PathSeparator) - - // The expected prefix for the resulting path is the root, with a path - // separator at the end. - expectedPrefix := filepath.FromSlash(root) - if !strings.HasSuffix(expectedPrefix, pathSep) { - expectedPrefix += pathSep - } - - // The relative path should be clean from internal dotdots and similar - // funkyness. - rel = filepath.FromSlash(rel) - if filepath.Clean(rel) != rel { - return "", errInvalidFilename - } - - // It is not acceptable to attempt to traverse upwards or refer to the - // root itself. - switch rel { - case ".", "..", pathSep: - return "", errNotRelative - } - if strings.HasPrefix(rel, ".."+pathSep) { - return "", errNotRelative - } - - if strings.HasPrefix(rel, pathSep+pathSep) { - // The relative path may pretend to be an absolute path within the - // root, but the double path separator on Windows implies something - // else. It would get cleaned by the Join below, but it's out of - // spec anyway. - return "", errNotRelative - } - - // The supposedly correct path is the one filepath.Join will return, as - // it does cleaning and so on. Check that one first to make sure no - // obvious escape attempts have been made. - joined := filepath.Join(root, rel) - if !strings.HasPrefix(joined, expectedPrefix) { - return "", errNotRelative - } - - return joined, nil -} diff --git a/lib/model/model_test.go b/lib/model/model_test.go index a8597ce9f..05dd6767b 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -25,8 +25,8 @@ import ( "github.com/d4l3k/messagediff" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/db" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/ignore" - "github.com/syncthing/syncthing/lib/osutil" "github.com/syncthing/syncthing/lib/protocol" srand "github.com/syncthing/syncthing/lib/rand" "github.com/syncthing/syncthing/lib/scanner" @@ -35,12 +35,14 @@ import ( var device1, device2 protocol.DeviceID var defaultConfig *config.Wrapper var defaultFolderConfig config.FolderConfiguration +var defaultFs fs.Filesystem func init() { device1, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR") device2, _ = protocol.DeviceIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY") + defaultFs = fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata") - defaultFolderConfig = config.NewFolderConfiguration("default", "testdata") + defaultFolderConfig = config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "testdata") defaultFolderConfig.Devices = []config.FolderDeviceConfiguration{{DeviceID: device1}} _defaultConfig := config.Configuration{ Folders: []config.FolderConfiguration{defaultFolderConfig}, @@ -513,14 +515,16 @@ func TestClusterConfig(t *testing.T) { } cfg.Folders = []config.FolderConfiguration{ { - ID: "folder1", + ID: "folder1", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, {DeviceID: device2}, }, }, { - ID: "folder2", + ID: "folder2", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, {DeviceID: device2}, @@ -622,13 +626,15 @@ func TestIntroducer(t *testing.T) { }, Folders: []config.FolderConfiguration{ { - ID: "folder1", + ID: "folder1", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, }, }, { - ID: "folder2", + ID: "folder2", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, }, @@ -671,14 +677,16 @@ func TestIntroducer(t *testing.T) { }, Folders: []config.FolderConfiguration{ { - ID: "folder1", + ID: "folder1", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, {DeviceID: device2, IntroducedBy: device1}, }, }, { - ID: "folder2", + ID: "folder2", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, }, @@ -726,14 +734,16 @@ func TestIntroducer(t *testing.T) { }, Folders: []config.FolderConfiguration{ { - ID: "folder1", + ID: "folder1", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, {DeviceID: device2, IntroducedBy: device1}, }, }, { - ID: "folder2", + ID: "folder2", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, {DeviceID: device2, IntroducedBy: device1}, @@ -771,14 +781,16 @@ func TestIntroducer(t *testing.T) { }, Folders: []config.FolderConfiguration{ { - ID: "folder1", + ID: "folder1", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, {DeviceID: device2, IntroducedBy: device1}, }, }, { - ID: "folder2", + ID: "folder2", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, {DeviceID: device2, IntroducedBy: device1}, @@ -816,14 +828,16 @@ func TestIntroducer(t *testing.T) { }, Folders: []config.FolderConfiguration{ { - ID: "folder1", + ID: "folder1", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, {DeviceID: device2, IntroducedBy: device1}, }, }, { - ID: "folder2", + ID: "folder2", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, }, @@ -872,14 +886,16 @@ func TestIntroducer(t *testing.T) { }, Folders: []config.FolderConfiguration{ { - ID: "folder1", + ID: "folder1", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, {DeviceID: device2, IntroducedBy: device1}, }, }, { - ID: "folder2", + ID: "folder2", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, {DeviceID: device2}, @@ -916,14 +932,16 @@ func TestIntroducer(t *testing.T) { }, Folders: []config.FolderConfiguration{ { - ID: "folder1", + ID: "folder1", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, {DeviceID: device2, IntroducedBy: device1}, }, }, { - ID: "folder2", + ID: "folder2", + Path: "testdata", Devices: []config.FolderDeviceConfiguration{ {DeviceID: device1}, {DeviceID: device2, IntroducedBy: protocol.LocalDeviceID}, @@ -1026,7 +1044,7 @@ func TestIgnores(t *testing.T) { // because we will be changing the files on disk often enough that the // mtimes will be unreliable to determine change status. m.fmut.Lock() - m.folderIgnores["default"] = ignore.New(ignore.WithCache(true), ignore.WithChangeDetector(newAlwaysChanged())) + m.folderIgnores["default"] = ignore.New(defaultFs, ignore.WithCache(true), ignore.WithChangeDetector(newAlwaysChanged())) m.fmut.Unlock() // Make sure the initial scan has finished (ScanFolders is blocking) @@ -1050,7 +1068,7 @@ func TestIgnores(t *testing.T) { } // Invalid path, marker should be missing, hence returns an error. - m.AddFolder(config.FolderConfiguration{ID: "fresh", RawPath: "XXX"}) + m.AddFolder(config.FolderConfiguration{ID: "fresh", Path: "XXX"}) _, _, err = m.GetIgnores("fresh") if err == nil { t.Error("No error") @@ -1069,14 +1087,14 @@ func TestIgnores(t *testing.T) { func TestROScanRecovery(t *testing.T) { ldb := db.OpenMemory() - set := db.NewFileSet("default", ldb) + set := db.NewFileSet("default", defaultFs, ldb) set.Update(protocol.LocalDeviceID, []protocol.FileInfo{ {Name: "dummyfile"}, }) fcfg := config.FolderConfiguration{ ID: "default", - RawPath: "testdata/rotestfolder", + Path: "testdata/rotestfolder", Type: config.FolderTypeSendOnly, RescanIntervalS: 1, } @@ -1089,7 +1107,7 @@ func TestROScanRecovery(t *testing.T) { }, }) - os.RemoveAll(fcfg.RawPath) + os.RemoveAll(fcfg.Path) m := NewModel(cfg, protocol.LocalDeviceID, "syncthing", "dev", ldb, nil) m.AddFolder(fcfg) @@ -1120,14 +1138,14 @@ func TestROScanRecovery(t *testing.T) { return } - os.Mkdir(fcfg.RawPath, 0700) + os.Mkdir(fcfg.Path, 0700) if err := waitFor("folder marker missing"); err != nil { t.Error(err) return } - fd, err := os.Create(filepath.Join(fcfg.RawPath, ".stfolder")) + fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder")) if err != nil { t.Error(err) return @@ -1139,14 +1157,14 @@ func TestROScanRecovery(t *testing.T) { return } - os.Remove(filepath.Join(fcfg.RawPath, ".stfolder")) + os.Remove(filepath.Join(fcfg.Path, ".stfolder")) if err := waitFor("folder marker missing"); err != nil { t.Error(err) return } - os.Remove(fcfg.RawPath) + os.Remove(fcfg.Path) if err := waitFor("folder path missing"); err != nil { t.Error(err) @@ -1156,14 +1174,14 @@ func TestROScanRecovery(t *testing.T) { func TestRWScanRecovery(t *testing.T) { ldb := db.OpenMemory() - set := db.NewFileSet("default", ldb) + set := db.NewFileSet("default", defaultFs, ldb) set.Update(protocol.LocalDeviceID, []protocol.FileInfo{ {Name: "dummyfile"}, }) fcfg := config.FolderConfiguration{ ID: "default", - RawPath: "testdata/rwtestfolder", + Path: "testdata/rwtestfolder", Type: config.FolderTypeSendReceive, RescanIntervalS: 1, } @@ -1176,7 +1194,7 @@ func TestRWScanRecovery(t *testing.T) { }, }) - os.RemoveAll(fcfg.RawPath) + os.RemoveAll(fcfg.Path) m := NewModel(cfg, protocol.LocalDeviceID, "syncthing", "dev", ldb, nil) m.AddFolder(fcfg) @@ -1207,14 +1225,14 @@ func TestRWScanRecovery(t *testing.T) { return } - os.Mkdir(fcfg.RawPath, 0700) + os.Mkdir(fcfg.Path, 0700) if err := waitFor("folder marker missing"); err != nil { t.Error(err) return } - fd, err := os.Create(filepath.Join(fcfg.RawPath, ".stfolder")) + fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder")) if err != nil { t.Error(err) return @@ -1226,14 +1244,14 @@ func TestRWScanRecovery(t *testing.T) { return } - os.Remove(filepath.Join(fcfg.RawPath, ".stfolder")) + os.Remove(filepath.Join(fcfg.Path, ".stfolder")) if err := waitFor("folder marker missing"); err != nil { t.Error(err) return } - os.Remove(fcfg.RawPath) + os.Remove(fcfg.Path) if err := waitFor("folder path missing"); err != nil { t.Error(err) @@ -1861,14 +1879,14 @@ func TestIssue3164(t *testing.T) { f := protocol.FileInfo{ Name: "issue3164", } - m := ignore.New() + m := ignore.New(defaultFs) if err := m.Parse(bytes.NewBufferString("(?d)oktodelete"), ""); err != nil { t.Fatal(err) } fl := sendReceiveFolder{ dbUpdates: make(chan dbUpdateJob, 1), - dir: "testdata", + fs: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), } fl.deleteDir(f, m) @@ -1955,7 +1973,7 @@ func TestIssue2782(t *testing.T) { if err := os.RemoveAll(testDir); err != nil { t.Skip(err) } - if err := osutil.MkdirAll(testDir+"/syncdir", 0755); err != nil { + if err := os.MkdirAll(testDir+"/syncdir", 0755); err != nil { t.Skip(err) } if err := ioutil.WriteFile(testDir+"/syncdir/file", []byte("hello, world\n"), 0644); err != nil { @@ -1968,7 +1986,7 @@ func TestIssue2782(t *testing.T) { db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "syncthing", "dev", db, nil) - m.AddFolder(config.NewFolderConfiguration("default", "~/"+testName+"/synclink/")) + m.AddFolder(config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "~/"+testName+"/synclink/")) m.StartFolder("default") m.ServeBackground() defer m.Stop() @@ -1985,7 +2003,7 @@ func TestIssue2782(t *testing.T) { func TestIndexesForUnknownDevicesDropped(t *testing.T) { dbi := db.OpenMemory() - files := db.NewFileSet("default", dbi) + files := db.NewFileSet("default", defaultFs, dbi) files.Replace(device1, genFiles(1)) files.Replace(device2, genFiles(1)) @@ -1998,7 +2016,7 @@ func TestIndexesForUnknownDevicesDropped(t *testing.T) { m.StartFolder("default") // Remote sequence is cached, hence need to recreated. - files = db.NewFileSet("default", dbi) + files = db.NewFileSet("default", defaultFs, dbi) if len(files.ListDevices()) != 1 { t.Error("Expected one device") @@ -2008,7 +2026,7 @@ func TestIndexesForUnknownDevicesDropped(t *testing.T) { func TestSharedWithClearedOnDisconnect(t *testing.T) { dbi := db.OpenMemory() - fcfg := config.NewFolderConfiguration("default", "testdata") + fcfg := config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "testdata") fcfg.Devices = []config.FolderDeviceConfiguration{ {DeviceID: device1}, {DeviceID: device2}, @@ -2247,7 +2265,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) { dbi := db.OpenMemory() - fcfg := config.NewFolderConfiguration("default", "testdata") + fcfg := config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "testdata") fcfg.Devices = []config.FolderDeviceConfiguration{ {DeviceID: device1}, {DeviceID: device2}, @@ -2335,151 +2353,6 @@ func TestNoRequestsFromPausedDevices(t *testing.T) { } } -func TestRootedJoinedPath(t *testing.T) { - type testcase struct { - root string - rel string - joined string - ok bool - } - cases := []testcase{ - // Valid cases - {"foo", "bar", "foo/bar", true}, - {"foo", "/bar", "foo/bar", true}, - {"foo/", "bar", "foo/bar", true}, - {"foo/", "/bar", "foo/bar", true}, - {"baz/foo", "bar", "baz/foo/bar", true}, - {"baz/foo", "/bar", "baz/foo/bar", true}, - {"baz/foo/", "bar", "baz/foo/bar", true}, - {"baz/foo/", "/bar", "baz/foo/bar", true}, - {"foo", "bar/baz", "foo/bar/baz", true}, - {"foo", "/bar/baz", "foo/bar/baz", true}, - {"foo/", "bar/baz", "foo/bar/baz", true}, - {"foo/", "/bar/baz", "foo/bar/baz", true}, - {"baz/foo", "bar/baz", "baz/foo/bar/baz", true}, - {"baz/foo", "/bar/baz", "baz/foo/bar/baz", true}, - {"baz/foo/", "bar/baz", "baz/foo/bar/baz", true}, - {"baz/foo/", "/bar/baz", "baz/foo/bar/baz", true}, - - // Not escape attempts, but oddly formatted relative paths. Disallowed. - {"foo", "./bar", "", false}, - {"baz/foo", "./bar", "", false}, - {"foo", "./bar/baz", "", false}, - {"baz/foo", "./bar/baz", "", false}, - {"baz/foo", "bar/../baz", "", false}, - {"baz/foo", "/bar/../baz", "", false}, - {"baz/foo", "./bar/../baz", "", false}, - {"baz/foo", "bar/../baz", "", false}, - {"baz/foo", "/bar/../baz", "", false}, - {"baz/foo", "./bar/../baz", "", false}, - - // Results in an allowed path, but does it by probing. Disallowed. - {"foo", "../foo", "", false}, - {"foo", "../foo/bar", "", false}, - {"baz/foo", "../foo/bar", "", false}, - {"baz/foo", "../../baz/foo/bar", "", false}, - {"baz/foo", "bar/../../foo/bar", "", false}, - {"baz/foo", "bar/../../../baz/foo/bar", "", false}, - - // Escape attempts. - {"foo", "", "", false}, - {"foo", "/", "", false}, - {"foo", "..", "", false}, - {"foo", "/..", "", false}, - {"foo", "../", "", false}, - {"foo", "../bar", "", false}, - {"foo", "../foobar", "", false}, - {"foo/", "../bar", "", false}, - {"foo/", "../foobar", "", false}, - {"baz/foo", "../bar", "", false}, - {"baz/foo", "../foobar", "", false}, - {"baz/foo/", "../bar", "", false}, - {"baz/foo/", "../foobar", "", false}, - {"baz/foo/", "bar/../../quux/baz", "", false}, - - // Empty root is a misconfiguration. - {"", "/foo", "", false}, - {"", "foo", "", false}, - {"", ".", "", false}, - {"", "..", "", false}, - {"", "/", "", false}, - {"", "", "", false}, - - // Root=/ is valid, and things should be verified as usual. - {"/", "foo", "/foo", true}, - {"/", "/foo", "/foo", true}, - {"/", "../foo", "", false}, - {"/", ".", "", false}, - {"/", "..", "", false}, - {"/", "/", "", false}, - {"/", "", "", false}, - } - - if runtime.GOOS == "windows" { - extraCases := []testcase{ - {`c:\`, `foo`, `c:\foo`, true}, - {`\\?\c:\`, `foo`, `\\?\c:\foo`, true}, - {`c:\`, `\foo`, `c:\foo`, true}, - {`\\?\c:\`, `\foo`, `\\?\c:\foo`, true}, - - {`c:\`, `\\foo`, ``, false}, - {`c:\`, ``, ``, false}, - {`c:\`, `.`, ``, false}, - {`c:\`, `\`, ``, false}, - {`\\?\c:\`, `\\foo`, ``, false}, - {`\\?\c:\`, ``, ``, false}, - {`\\?\c:\`, `.`, ``, false}, - {`\\?\c:\`, `\`, ``, false}, - - // makes no sense, but will be treated simply as a bad filename - {`c:\foo`, `d:\bar`, `c:\foo\d:\bar`, true}, - } - - for _, tc := range cases { - // Add case where root is backslashed, rel is forward slashed - extraCases = append(extraCases, testcase{ - root: filepath.FromSlash(tc.root), - rel: tc.rel, - joined: tc.joined, - ok: tc.ok, - }) - // and the opposite - extraCases = append(extraCases, testcase{ - root: tc.root, - rel: filepath.FromSlash(tc.rel), - joined: tc.joined, - ok: tc.ok, - }) - // and both backslashed - extraCases = append(extraCases, testcase{ - root: filepath.FromSlash(tc.root), - rel: filepath.FromSlash(tc.rel), - joined: tc.joined, - ok: tc.ok, - }) - } - - cases = append(cases, extraCases...) - } - - for _, tc := range cases { - res, err := rootedJoinedPath(tc.root, tc.rel) - if tc.ok { - if err != nil { - t.Errorf("Unexpected error for rootedJoinedPath(%q, %q): %v", tc.root, tc.rel, err) - continue - } - exp := filepath.FromSlash(tc.joined) - if res != exp { - t.Errorf("Unexpected result for rootedJoinedPath(%q, %q): %q != expected %q", tc.root, tc.rel, res, exp) - } - } else if err == nil { - t.Errorf("Unexpected pass for rootedJoinedPath(%q, %q) => %q", tc.root, tc.rel, res) - continue - } - } -} - func addFakeConn(m *Model, dev protocol.DeviceID) *fakeConnection { fc := &fakeConnection{id: dev, model: m} m.AddConnection(fc, protocol.HelloResult{}) diff --git a/lib/model/requests_test.go b/lib/model/requests_test.go index d440b20f4..bfd0bc49f 100644 --- a/lib/model/requests_test.go +++ b/lib/model/requests_test.go @@ -18,6 +18,7 @@ import ( "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/db" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/protocol" ) @@ -214,7 +215,7 @@ func TestRequestVersioningSymlinkAttack(t *testing.T) { // deleted symlink to escape cfg := defaultConfig.RawCopy() - cfg.Folders[0] = config.NewFolderConfiguration("default", "_tmpfolder") + cfg.Folders[0] = config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "_tmpfolder") cfg.Folders[0].PullerSleepS = 1 cfg.Folders[0].Devices = []config.FolderDeviceConfiguration{ {DeviceID: device1}, @@ -287,7 +288,7 @@ func TestRequestVersioningSymlinkAttack(t *testing.T) { func setupModelWithConnection() (*Model, *fakeConnection) { cfg := defaultConfig.RawCopy() - cfg.Folders[0] = config.NewFolderConfiguration("default", "_tmpfolder") + cfg.Folders[0] = config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "_tmpfolder") cfg.Folders[0].PullerSleepS = 1 cfg.Folders[0].Devices = []config.FolderDeviceConfiguration{ {DeviceID: device1}, diff --git a/lib/model/rofolder.go b/lib/model/rofolder.go index 50af80074..72dadd35e 100644 --- a/lib/model/rofolder.go +++ b/lib/model/rofolder.go @@ -24,7 +24,7 @@ type sendOnlyFolder struct { config.FolderConfiguration } -func newSendOnlyFolder(model *Model, cfg config.FolderConfiguration, _ versioner.Versioner, _ *fs.MtimeFS) service { +func newSendOnlyFolder(model *Model, cfg config.FolderConfiguration, _ versioner.Versioner, _ fs.Filesystem) service { ctx, cancel := context.WithCancel(context.Background()) return &sendOnlyFolder{ diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go index a8c24c29c..7b06b2454 100644 --- a/lib/model/rwfolder.go +++ b/lib/model/rwfolder.go @@ -11,7 +11,6 @@ import ( "errors" "fmt" "math/rand" - "os" "path/filepath" "runtime" "sort" @@ -51,7 +50,7 @@ type copyBlocksState struct { } // Which filemode bits to preserve -const retainBits = os.ModeSetgid | os.ModeSetuid | os.ModeSticky +const retainBits = fs.ModeSetgid | fs.ModeSetuid | fs.ModeSticky var ( activity = newDeviceActivity() @@ -84,8 +83,7 @@ type sendReceiveFolder struct { folder config.FolderConfiguration - mtimeFS *fs.MtimeFS - dir string + fs fs.Filesystem versioner versioner.Versioner sleep time.Duration pause time.Duration @@ -99,7 +97,7 @@ type sendReceiveFolder struct { errorsMut sync.Mutex } -func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, mtimeFS *fs.MtimeFS) service { +func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service { ctx, cancel := context.WithCancel(context.Background()) f := &sendReceiveFolder{ @@ -113,8 +111,7 @@ func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver vers }, FolderConfiguration: cfg, - mtimeFS: mtimeFS, - dir: cfg.Path(), + fs: fs, versioner: ver, queue: newJobQueue(), @@ -434,7 +431,7 @@ func (f *sendReceiveFolder) pullerIteration(ignores *ignore.Matcher) int { for _, fi := range processDirectly { // Verify that the thing we are handling lives inside a directory, // and not a symlink or empty space. - if err := osutil.TraversesSymlink(f.dir, filepath.Dir(fi.Name)); err != nil { + if err := osutil.TraversesSymlink(f.fs, filepath.Dir(fi.Name)); err != nil { f.newError(fi.Name, err) continue } @@ -523,7 +520,7 @@ nextFile: // Verify that the thing we are handling lives inside a directory, // and not a symlink or empty space. - if err := osutil.TraversesSymlink(f.dir, filepath.Dir(fi.Name)); err != nil { + if err := osutil.TraversesSymlink(f.fs, filepath.Dir(fi.Name)); err != nil { f.newError(fi.Name, err) continue } @@ -610,12 +607,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo) { }) }() - realName, err := rootedJoinedPath(f.dir, file.Name) - if err != nil { - f.newError(file.Name, err) - return - } - mode := os.FileMode(file.Permissions & 0777) + mode := fs.FileMode(file.Permissions & 0777) if f.ignorePermissions(file) { mode = 0777 } @@ -625,13 +617,13 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo) { l.Debugf("need dir\n\t%v\n\t%v", file, curFile) } - info, err := f.mtimeFS.Lstat(realName) + info, err := f.fs.Lstat(file.Name) switch { // There is already something under that name, but it's a file/link. // Most likely a file/link is getting replaced with a directory. // Remove the file/link and fall through to directory creation. case err == nil && (!info.IsDir() || info.IsSymlink()): - err = osutil.InWritableDir(os.Remove, realName) + err = osutil.InWritableDir(f.fs.Remove, f.fs, file.Name) if err != nil { l.Infof("Puller (folder %q, dir %q): %v", f.folderID, file.Name, err) f.newError(file.Name, err) @@ -640,28 +632,28 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo) { fallthrough // The directory doesn't exist, so we create it with the right // mode bits from the start. - case err != nil && os.IsNotExist(err): + case err != nil && fs.IsNotExist(err): // We declare a function that acts on only the path name, so // we can pass it to InWritableDir. We use a regular Mkdir and // not MkdirAll because the parent should already exist. mkdir := func(path string) error { - err = os.Mkdir(path, mode) + err = f.fs.Mkdir(path, mode) if err != nil || f.ignorePermissions(file) { return err } // Stat the directory so we can check its permissions. - info, err := f.mtimeFS.Lstat(path) + info, err := f.fs.Lstat(path) if err != nil { return err } // Mask for the bits we want to preserve and add them in to the // directories permissions. - return os.Chmod(path, mode|(os.FileMode(info.Mode())&retainBits)) + return f.fs.Chmod(path, mode|(info.Mode()&retainBits)) } - if err = osutil.InWritableDir(mkdir, realName); err == nil { + if err = osutil.InWritableDir(mkdir, f.fs, file.Name); err == nil { f.dbUpdates <- dbUpdateJob{file, dbUpdateHandleDir} } else { l.Infof("Puller (folder %q, dir %q): %v", f.folderID, file.Name, err) @@ -681,7 +673,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo) { // It's OK to change mode bits on stuff within non-writable directories. if f.ignorePermissions(file) { f.dbUpdates <- dbUpdateJob{file, dbUpdateHandleDir} - } else if err := os.Chmod(realName, mode|(os.FileMode(info.Mode())&retainBits)); err == nil { + } else if err := f.fs.Chmod(file.Name, mode|(fs.FileMode(info.Mode())&retainBits)); err == nil { f.dbUpdates <- dbUpdateJob{file, dbUpdateHandleDir} } else { l.Infof("Puller (folder %q, dir %q): %v", f.folderID, file.Name, err) @@ -712,12 +704,6 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo) { }) }() - realName, err := rootedJoinedPath(f.dir, file.Name) - if err != nil { - f.newError(file.Name, err) - return - } - if shouldDebug() { curFile, _ := f.model.CurrentFolderFile(f.folderID, file.Name) l.Debugf("need symlink\n\t%v\n\t%v", file, curFile) @@ -732,11 +718,11 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo) { return } - if _, err = f.mtimeFS.Lstat(realName); err == nil { + if _, err = f.fs.Lstat(file.Name); err == nil { // There is already something under that name. Remove it to replace // with the symlink. This also handles the "change symlink type" // path. - err = osutil.InWritableDir(os.Remove, realName) + err = osutil.InWritableDir(f.fs.Remove, f.fs, file.Name) if err != nil { l.Infof("Puller (folder %q, dir %q): %v", f.folderID, file.Name, err) f.newError(file.Name, err) @@ -747,10 +733,10 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo) { // We declare a function that acts on only the path name, so // we can pass it to InWritableDir. createLink := func(path string) error { - return os.Symlink(file.SymlinkTarget, path) + return f.fs.CreateSymlink(file.SymlinkTarget, path) } - if err = osutil.InWritableDir(createLink, realName); err == nil { + if err = osutil.InWritableDir(createLink, f.fs, file.Name); err == nil { f.dbUpdates <- dbUpdateJob{file, dbUpdateHandleSymlink} } else { l.Infof("Puller (folder %q, dir %q): %v", f.folderID, file.Name, err) @@ -781,31 +767,21 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, matcher *ignore.Ma }) }() - realName, err := rootedJoinedPath(f.dir, file.Name) - if err != nil { - f.newError(file.Name, err) - return - } - // Delete any temporary files lying around in the directory - dir, _ := os.Open(realName) - if dir != nil { - files, _ := dir.Readdirnames(-1) - for _, dirFile := range files { - fullDirFile := filepath.Join(file.Name, dirFile) - if ignore.IsTemporary(dirFile) || (matcher != nil && - matcher.Match(fullDirFile).IsDeletable()) { - os.RemoveAll(filepath.Join(f.dir, fullDirFile)) - } + + files, _ := f.fs.DirNames(file.Name) + for _, dirFile := range files { + fullDirFile := filepath.Join(file.Name, dirFile) + if ignore.IsTemporary(dirFile) || (matcher != nil && matcher.Match(fullDirFile).IsDeletable()) { + f.fs.RemoveAll(fullDirFile) } - dir.Close() } - err = osutil.InWritableDir(os.Remove, realName) - if err == nil || os.IsNotExist(err) { + err = osutil.InWritableDir(f.fs.Remove, f.fs, file.Name) + if err == nil || fs.IsNotExist(err) { // It was removed or it doesn't exist to start with f.dbUpdates <- dbUpdateJob{file, dbUpdateDeleteDir} - } else if _, serr := f.mtimeFS.Lstat(realName); serr != nil && !os.IsPermission(serr) { + } else if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) { // We get an error just looking at the directory, and it's not a // permission problem. Lets assume the error is in fact some variant // of "file does not exist" (possibly expressed as some parent being a @@ -840,12 +816,6 @@ func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo) { }) }() - realName, err := rootedJoinedPath(f.dir, file.Name) - if err != nil { - f.newError(file.Name, err) - return - } - cur, ok := f.model.CurrentFolderFile(f.folderID, file.Name) if ok && f.inConflict(cur.Version, file.Version) { // There is a conflict here. Move the file to a conflict copy instead @@ -854,17 +824,17 @@ func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo) { file.Version = file.Version.Merge(cur.Version) err = osutil.InWritableDir(func(name string) error { return f.moveForConflict(name, file.ModifiedBy.String()) - }, realName) + }, f.fs, file.Name) } else if f.versioner != nil && !cur.IsSymlink() { - err = osutil.InWritableDir(f.versioner.Archive, realName) + err = osutil.InWritableDir(f.versioner.Archive, f.fs, file.Name) } else { - err = osutil.InWritableDir(os.Remove, realName) + err = osutil.InWritableDir(f.fs.Remove, f.fs, file.Name) } - if err == nil || os.IsNotExist(err) { + if err == nil || fs.IsNotExist(err) { // It was removed or it doesn't exist to start with f.dbUpdates <- dbUpdateJob{file, dbUpdateDeleteFile} - } else if _, serr := f.mtimeFS.Lstat(realName); serr != nil && !os.IsPermission(serr) { + } else if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) { // We get an error just looking at the file, and it's not a permission // problem. Lets assume the error is in fact some variant of "file // does not exist" (possibly expressed as some parent being a file and @@ -915,24 +885,13 @@ func (f *sendReceiveFolder) renameFile(source, target protocol.FileInfo) { l.Debugln(f, "taking rename shortcut", source.Name, "->", target.Name) - from, err := rootedJoinedPath(f.dir, source.Name) - if err != nil { - f.newError(source.Name, err) - return - } - to, err := rootedJoinedPath(f.dir, target.Name) - if err != nil { - f.newError(target.Name, err) - return - } - if f.versioner != nil { - err = osutil.Copy(from, to) + err = osutil.Copy(f.fs, source.Name, target.Name) if err == nil { - err = osutil.InWritableDir(f.versioner.Archive, from) + err = osutil.InWritableDir(f.versioner.Archive, f.fs, source.Name) } } else { - err = osutil.TryRename(from, to) + err = osutil.TryRename(f.fs, source.Name, target.Name) } if err == nil { @@ -955,7 +914,7 @@ func (f *sendReceiveFolder) renameFile(source, target protocol.FileInfo) { // get rid of. Attempt to delete it instead so that we make *some* // progress. The target is unhandled. - err = osutil.InWritableDir(os.Remove, from) + err = osutil.InWritableDir(f.fs.Remove, f.fs, source.Name) if err != nil { l.Infof("Puller (folder %q, file %q): delete %q after failed rename: %v", f.folderID, target.Name, source.Name, err) f.newError(target.Name, err) @@ -1041,26 +1000,16 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c return } - // Figure out the absolute filenames we need once and for all - tempName, err := rootedJoinedPath(f.dir, ignore.TempName(file.Name)) - if err != nil { - f.newError(file.Name, err) - return - } - realName, err := rootedJoinedPath(f.dir, file.Name) - if err != nil { - f.newError(file.Name, err) - return - } + tempName := ignore.TempName(file.Name) if hasCurFile && !curFile.IsDirectory() && !curFile.IsSymlink() { // Check that the file on disk is what we expect it to be according to // the database. If there's a mismatch here, there might be local // changes that we don't know about yet and we should scan before // touching the file. If we can't stat the file we'll just pull it. - if info, err := f.mtimeFS.Lstat(realName); err == nil { + if info, err := f.fs.Lstat(file.Name); err == nil { if !info.ModTime().Equal(curFile.ModTime()) || info.Size() != curFile.Size { - l.Debugln("file modified but not rescanned; not pulling:", realName) + l.Debugln("file modified but not rescanned; not pulling:", file.Name) // Scan() is synchronous (i.e. blocks until the scan is // completed and returns an error), but a scan can't happen // while we're in the puller routine. Request the scan in the @@ -1082,7 +1031,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c // Check for an old temporary file which might have some blocks we could // reuse. - tempBlocks, err := scanner.HashFile(f.ctx, fs.DefaultFilesystem, tempName, protocol.BlockSize, nil, false) + tempBlocks, err := scanner.HashFile(f.ctx, f.fs, tempName, protocol.BlockSize, nil, false) if err == nil { // Check for any reusable blocks in the temp file tempCopyBlocks, _ := scanner.BlockDiff(tempBlocks, file.Blocks) @@ -1110,7 +1059,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c // Otherwise, discard the file ourselves in order for the // sharedpuller not to panic when it fails to exclusively create a // file which already exists - osutil.InWritableDir(os.Remove, tempName) + osutil.InWritableDir(f.fs.Remove, f.fs, tempName) } } else { // Copy the blocks, as we don't want to shuffle them on the FileInfo @@ -1119,8 +1068,8 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c } if f.MinDiskFree.BaseValue() > 0 { - if free, err := osutil.DiskFreeBytes(f.dir); err == nil && free < blocksSize { - l.Warnf(`Folder "%s": insufficient disk space in %s for %s: have %.2f MiB, need %.2f MiB`, f.folderID, f.dir, file.Name, float64(free)/1024/1024, float64(blocksSize)/1024/1024) + if usage, err := f.fs.Usage("."); err == nil && usage.Free < blocksSize { + l.Warnf(`Folder "%s": insufficient disk space in %s for %s: have %.2f MiB, need %.2f MiB`, f.folderID, f.fs.URI(), file.Name, float64(usage.Free)/1024/1024, float64(blocksSize)/1024/1024) f.newError(file.Name, errors.New("insufficient space")) return } @@ -1141,9 +1090,10 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c s := sharedPullerState{ file: file, + fs: f.fs, folder: f.folderID, tempName: tempName, - realName: realName, + realName: file.Name, copyTotal: len(blocks), copyNeeded: len(blocks), reused: len(reused), @@ -1170,20 +1120,15 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c // shortcutFile sets file mode and modification time, when that's the only // thing that has changed. func (f *sendReceiveFolder) shortcutFile(file protocol.FileInfo) error { - realName, err := rootedJoinedPath(f.dir, file.Name) - if err != nil { - f.newError(file.Name, err) - return err - } if !f.ignorePermissions(file) { - if err := os.Chmod(realName, os.FileMode(file.Permissions&0777)); err != nil { + if err := f.fs.Chmod(file.Name, fs.FileMode(file.Permissions&0777)); err != nil { l.Infof("Puller (folder %q, file %q): shortcut: chmod: %v", f.folderID, file.Name, err) f.newError(file.Name, err) return err } } - f.mtimeFS.Chtimes(realName, file.ModTime(), file.ModTime()) // never fails + f.fs.Chtimes(file.Name, file.ModTime(), file.ModTime()) // never fails // This may have been a conflict. We should merge the version vectors so // that our clock doesn't move backwards. @@ -1211,15 +1156,16 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch f.model.progressEmitter.Register(state.sharedPullerState) } - folderRoots := make(map[string]string) + folderFilesystems := make(map[string]fs.Filesystem) var folders []string f.model.fmut.RLock() for folder, cfg := range f.model.folderCfgs { - folderRoots[folder] = cfg.Path() + folderFilesystems[folder] = cfg.Filesystem() folders = append(folders, folder) } f.model.fmut.RUnlock() + var file fs.File var weakHashFinder *weakhash.Finder if weakhash.Enabled { @@ -1237,9 +1183,12 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch } if len(hashesToFind) > 0 { - weakHashFinder, err = weakhash.NewFinder(state.realName, protocol.BlockSize, hashesToFind) - if err != nil { - l.Debugln("weak hasher", err) + file, err = f.fs.Open(state.file.Name) + if err == nil { + weakHashFinder, err = weakhash.NewFinder(file, protocol.BlockSize, hashesToFind) + if err != nil { + l.Debugln("weak hasher", err) + } } } else { l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name) @@ -1289,12 +1238,9 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch } if !found { - found = f.model.finder.Iterate(folders, block.Hash, func(folder, file string, index int32) bool { - inFile, err := rootedJoinedPath(folderRoots[folder], file) - if err != nil { - return false - } - fd, err := os.Open(inFile) + found = f.model.finder.Iterate(folders, block.Hash, func(folder, path string, index int32) bool { + fs := folderFilesystems[folder] + fd, err := fs.Open(path) if err != nil { return false } @@ -1308,8 +1254,8 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch hash, err := scanner.VerifyBuffer(buf, block) if err != nil { if hash != nil { - l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, file, index, block.Hash, hash) - err = f.model.finder.Fix(folder, file, index, block.Hash, hash) + l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, path, index, block.Hash, hash) + err = f.model.finder.Fix(folder, path, index, block.Hash, hash) if err != nil { l.Warnln("finder fix:", err) } @@ -1323,7 +1269,7 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch if err != nil { state.fail("dst write", err) } - if file == state.file.Name { + if path == state.file.Name { state.copiedFromOrigin() } return true @@ -1345,7 +1291,12 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch state.copyDone(block) } } - weakHashFinder.Close() + if file != nil { + // os.File used to return invalid argument if nil. + // fs.File panics as it's an interface. + file.Close() + } + out <- state.sharedPullerState } } @@ -1426,12 +1377,12 @@ func (f *sendReceiveFolder) pullerRoutine(in <-chan pullBlockState, out chan<- * func (f *sendReceiveFolder) performFinish(state *sharedPullerState) error { // Set the correct permission bits on the new file if !f.ignorePermissions(state.file) { - if err := os.Chmod(state.tempName, os.FileMode(state.file.Permissions&0777)); err != nil { + if err := f.fs.Chmod(state.tempName, fs.FileMode(state.file.Permissions&0777)); err != nil { return err } } - if stat, err := f.mtimeFS.Lstat(state.realName); err == nil { + if stat, err := f.fs.Lstat(state.file.Name); err == nil { // There is an old file or directory already in place. We need to // handle that. @@ -1445,7 +1396,7 @@ func (f *sendReceiveFolder) performFinish(state *sharedPullerState) error { // and future hard ignores before attempting a directory delete. // Should share code with f.deletDir(). - if err = osutil.InWritableDir(os.Remove, state.realName); err != nil { + if err = osutil.InWritableDir(f.fs.Remove, f.fs, state.file.Name); err != nil { return err } @@ -1458,7 +1409,7 @@ func (f *sendReceiveFolder) performFinish(state *sharedPullerState) error { state.file.Version = state.file.Version.Merge(state.version) err = osutil.InWritableDir(func(name string) error { return f.moveForConflict(name, state.file.ModifiedBy.String()) - }, state.realName) + }, f.fs, state.file.Name) if err != nil { return err } @@ -1468,7 +1419,7 @@ func (f *sendReceiveFolder) performFinish(state *sharedPullerState) error { // file before we replace it. Archiving a non-existent file is not // an error. - if err = f.versioner.Archive(state.realName); err != nil { + if err = f.versioner.Archive(state.file.Name); err != nil { return err } } @@ -1476,12 +1427,12 @@ func (f *sendReceiveFolder) performFinish(state *sharedPullerState) error { // Replace the original content with the new one. If it didn't work, // leave the temp file in place for reuse. - if err := osutil.TryRename(state.tempName, state.realName); err != nil { + if err := osutil.TryRename(f.fs, state.tempName, state.file.Name); err != nil { return err } // Set the correct timestamp on the new file - f.mtimeFS.Chtimes(state.realName, state.file.ModTime(), state.file.ModTime()) // never fails + f.fs.Chtimes(state.file.Name, state.file.ModTime(), state.file.ModTime()) // never fails // Record the updated file in the index f.dbUpdates <- dbUpdateJob{state.file, dbUpdateHandleFile} @@ -1540,26 +1491,7 @@ func (f *sendReceiveFolder) dbUpdaterRoutine() { tick := time.NewTicker(maxBatchTime) defer tick.Stop() - var changedFiles []string - var changedDirs []string - if f.Fsync { - changedFiles = make([]string, 0, maxBatchSize) - changedDirs = make([]string, 0, maxBatchSize) - } - - syncFilesOnce := func(files []string, syncFn func(string) error) { - sort.Strings(files) - var lastFile string - for _, file := range files { - if lastFile == file { - continue - } - lastFile = file - if err := syncFn(file); err != nil { - l.Infof("fsync %q failed: %v", file, err) - } - } - } + changedDirs := make(map[string]struct{}) handleBatch := func() { found := false @@ -1567,20 +1499,16 @@ func (f *sendReceiveFolder) dbUpdaterRoutine() { for _, job := range batch { files = append(files, job.file) - if f.Fsync { - // collect changed files and dirs - switch job.jobType { - case dbUpdateHandleFile, dbUpdateShortcutFile: - changedFiles = append(changedFiles, filepath.Join(f.dir, job.file.Name)) - case dbUpdateHandleDir: - changedDirs = append(changedDirs, filepath.Join(f.dir, job.file.Name)) - case dbUpdateHandleSymlink: - // fsyncing symlinks is only supported by MacOS, ignore - } - if job.jobType != dbUpdateShortcutFile { - changedDirs = append(changedDirs, filepath.Dir(filepath.Join(f.dir, job.file.Name))) - } + + switch job.jobType { + case dbUpdateHandleFile, dbUpdateShortcutFile: + changedDirs[filepath.Dir(job.file.Name)] = struct{}{} + case dbUpdateHandleDir: + changedDirs[job.file.Name] = struct{}{} + case dbUpdateHandleSymlink: + // fsyncing symlinks is only supported by MacOS, ignore } + if job.file.IsInvalid() || (job.file.IsDirectory() && !job.file.IsSymlink()) { continue } @@ -1593,12 +1521,18 @@ func (f *sendReceiveFolder) dbUpdaterRoutine() { lastFile = job.file } - if f.Fsync { - // sync files and dirs to disk - syncFilesOnce(changedFiles, osutil.SyncFile) - changedFiles = changedFiles[:0] - syncFilesOnce(changedDirs, osutil.SyncDir) - changedDirs = changedDirs[:0] + // sync directories + for dir := range changedDirs { + delete(changedDirs, dir) + fd, err := f.fs.Open(dir) + if err != nil { + l.Infof("fsync %q failed: %v", dir, err) + continue + } + if err := fd.Sync(); err != nil { + l.Infof("fsync %q failed: %v", dir, err) + } + fd.Close() } // All updates to file/folder objects that originated remotely @@ -1669,14 +1603,14 @@ func removeAvailability(availabilities []Availability, availability Availability func (f *sendReceiveFolder) moveForConflict(name string, lastModBy string) error { if strings.Contains(filepath.Base(name), ".sync-conflict-") { l.Infoln("Conflict for", name, "which is already a conflict copy; not copying again.") - if err := os.Remove(name); err != nil && !os.IsNotExist(err) { + if err := f.fs.Remove(name); err != nil && !fs.IsNotExist(err) { return err } return nil } if f.MaxConflicts == 0 { - if err := os.Remove(name); err != nil && !os.IsNotExist(err) { + if err := f.fs.Remove(name); err != nil && !fs.IsNotExist(err) { return err } return nil @@ -1685,8 +1619,8 @@ func (f *sendReceiveFolder) moveForConflict(name string, lastModBy string) error ext := filepath.Ext(name) withoutExt := name[:len(name)-len(ext)] newName := withoutExt + time.Now().Format(".sync-conflict-20060102-150405-") + lastModBy + ext - err := os.Rename(name, newName) - if os.IsNotExist(err) { + err := f.fs.Rename(name, newName) + if fs.IsNotExist(err) { // We were supposed to move a file away but it does not exist. Either // the user has already moved it away, or the conflict was between a // remote modification and a local delete. In either way it does not @@ -1694,11 +1628,11 @@ func (f *sendReceiveFolder) moveForConflict(name string, lastModBy string) error err = nil } if f.MaxConflicts > -1 { - matches, gerr := osutil.Glob(withoutExt + ".sync-conflict-????????-??????*" + ext) + matches, gerr := f.fs.Glob(withoutExt + ".sync-conflict-????????-??????*" + ext) if gerr == nil && len(matches) > f.MaxConflicts { sort.Sort(sort.Reverse(sort.StringSlice(matches))) for _, match := range matches[f.MaxConflicts:] { - gerr = os.Remove(match) + gerr = f.fs.Remove(match) if gerr != nil { l.Debugln(f, "removing extra conflict", gerr) } @@ -1772,7 +1706,7 @@ func fileValid(file db.FileIntf) error { return errSymlinksUnsupported case runtime.GOOS == "windows" && windowsInvalidFilename(file.FileName()): - return errInvalidFilename + return fs.ErrInvalidFilename } return nil @@ -1821,7 +1755,7 @@ func (l byComponentCount) Swap(a, b int) { func componentCount(name string) int { count := 0 for _, codepoint := range name { - if codepoint == os.PathSeparator { + if codepoint == fs.PathSeparator { count++ } } diff --git a/lib/model/rwfolder_test.go b/lib/model/rwfolder_test.go index 10b48795e..8db5c6773 100644 --- a/lib/model/rwfolder_test.go +++ b/lib/model/rwfolder_test.go @@ -87,8 +87,7 @@ func setUpSendReceiveFolder(model *Model) *sendReceiveFolder { ctx: context.TODO(), }, - mtimeFS: fs.NewMtimeFS(fs.DefaultFilesystem, db.NewNamespacedKV(model.db, "mtime")), - dir: "testdata", + fs: fs.NewMtimeFS(fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), db.NewNamespacedKV(model.db, "mtime")), queue: newJobQueue(), errors: make(map[string]string), errorsMut: sync.NewMutex(), @@ -246,7 +245,7 @@ func TestCopierFinder(t *testing.T) { } // Verify that the fetched blocks have actually been written to the temp file - blks, err := scanner.HashFile(context.TODO(), fs.DefaultFilesystem, tempFile, protocol.BlockSize, nil, false) + blks, err := scanner.HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, "."), tempFile, protocol.BlockSize, nil, false) if err != nil { t.Log(err) } diff --git a/lib/model/sharedpullerstate.go b/lib/model/sharedpullerstate.go index 2e3947434..c29bedab1 100644 --- a/lib/model/sharedpullerstate.go +++ b/lib/model/sharedpullerstate.go @@ -8,10 +8,10 @@ package model import ( "io" - "os" "path/filepath" "time" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sync" ) @@ -21,6 +21,7 @@ import ( type sharedPullerState struct { // Immutable, does not require locking file protocol.FileInfo // The new file (desired end state) + fs fs.Filesystem folder string tempName string realName string @@ -32,7 +33,7 @@ type sharedPullerState struct { // Mutable, must be locked for access err error // The first error we hit - fd *os.File // The fd of the temp file + fd fs.File // The fd of the temp file copyTotal int // Total number of copy actions for the whole job pullTotal int // Total number of pull actions for the whole job copyOrigin int // Number of blocks copied from the original file @@ -92,8 +93,8 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) { // osutil.InWritableDir except we need to do more stuff so we duplicate it // here. dir := filepath.Dir(s.tempName) - if info, err := os.Stat(dir); err != nil { - if os.IsNotExist(err) { + if info, err := s.fs.Stat(dir); err != nil { + if fs.IsNotExist(err) { // XXX: This works around a bug elsewhere, a race condition when // things are deleted while being synced. However that happens, we // end up with a directory for "foo" with the delete bit, but a @@ -103,7 +104,7 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) { // next scan it'll be found and the delete bit on it is removed. // The user can then clean up as they like... l.Infoln("Resurrecting directory", dir) - if err := os.MkdirAll(dir, 0755); err != nil { + if err := s.fs.MkdirAll(dir, 0755); err != nil { s.failLocked("resurrect dir", err) return nil, err } @@ -112,10 +113,10 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) { return nil, err } } else if info.Mode()&0200 == 0 { - err := os.Chmod(dir, 0755) + err := s.fs.Chmod(dir, 0755) if !s.ignorePerms && err == nil { defer func() { - err := os.Chmod(dir, info.Mode().Perm()) + err := s.fs.Chmod(dir, info.Mode()&fs.ModePerm) if err != nil { panic(err) } @@ -128,7 +129,7 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) { // permissions will be set to the final value later, but in the meantime // we don't want to have a temporary file with looser permissions than // the final outcome. - mode := os.FileMode(s.file.Permissions) | 0600 + mode := fs.FileMode(s.file.Permissions) | 0600 if s.ignorePerms { // When ignorePerms is set we use a very permissive mode and let the // system umask filter it. @@ -137,9 +138,9 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) { // Attempt to create the temp file // RDWR because of issue #2994. - flags := os.O_RDWR + flags := fs.OptReadWrite if s.reused == 0 { - flags |= os.O_CREATE | os.O_EXCL + flags |= fs.OptCreate | fs.OptExclusive } else if !s.ignorePerms { // With sufficiently bad luck when exiting or crashing, we may have // had time to chmod the temp file to read only state but not yet @@ -151,12 +152,12 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) { // already and make no modification, as we would otherwise override // what the umask dictates. - if err := os.Chmod(s.tempName, mode); err != nil { + if err := s.fs.Chmod(s.tempName, mode); err != nil { s.failLocked("dst create chmod", err) return nil, err } } - fd, err := os.OpenFile(s.tempName, flags, mode) + fd, err := s.fs.OpenFile(s.tempName, flags, mode) if err != nil { s.failLocked("dst create", err) return nil, err @@ -180,7 +181,7 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) { } // sourceFile opens the existing source file for reading -func (s *sharedPullerState) sourceFile() (*os.File, error) { +func (s *sharedPullerState) sourceFile() (fs.File, error) { s.mut.Lock() defer s.mut.Unlock() @@ -190,7 +191,7 @@ func (s *sharedPullerState) sourceFile() (*os.File, error) { } // Attempt to open the existing file - fd, err := os.Open(s.realName) + fd, err := s.fs.Open(s.realName) if err != nil { s.failLocked("src open", err) return nil, err @@ -292,9 +293,12 @@ func (s *sharedPullerState) finalClose() (bool, error) { } if s.fd != nil { + // This is our error if we weren't errored before. Otherwise we + // keep the earlier error. + if fsyncErr := s.fd.Sync(); fsyncErr != nil && s.err == nil { + s.err = fsyncErr + } if closeErr := s.fd.Close(); closeErr != nil && s.err == nil { - // This is our error if we weren't errored before. Otherwise we - // keep the earlier error. s.err = closeErr } s.fd = nil diff --git a/lib/model/sharedpullerstate_test.go b/lib/model/sharedpullerstate_test.go index 3aad91022..c375324d2 100644 --- a/lib/model/sharedpullerstate_test.go +++ b/lib/model/sharedpullerstate_test.go @@ -10,12 +10,14 @@ import ( "os" "testing" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/sync" ) func TestSourceFileOK(t *testing.T) { s := sharedPullerState{ - realName: "testdata/foo", + fs: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), + realName: "foo", mut: sync.NewRWMutex(), } @@ -47,6 +49,7 @@ func TestSourceFileOK(t *testing.T) { func TestSourceFileBad(t *testing.T) { s := sharedPullerState{ + fs: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), realName: "nonexistent", mut: sync.NewRWMutex(), } @@ -73,7 +76,8 @@ func TestReadOnlyDir(t *testing.T) { }() s := sharedPullerState{ - tempName: "testdata/read_only_dir/.temp_name", + fs: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), + tempName: "read_only_dir/.temp_name", mut: sync.NewRWMutex(), } diff --git a/lib/osutil/atomic.go b/lib/osutil/atomic.go index 165fdf768..30b5e3bf7 100644 --- a/lib/osutil/atomic.go +++ b/lib/osutil/atomic.go @@ -8,10 +8,10 @@ package osutil import ( "errors" - "io/ioutil" - "os" "path/filepath" "runtime" + + "github.com/syncthing/syncthing/lib/fs" ) var ( @@ -25,7 +25,8 @@ var ( // returned on Close, so a lazy user can ignore errors until Close. type AtomicWriter struct { path string - next *os.File + next fs.File + fs fs.Filesystem err error } @@ -33,11 +34,19 @@ type AtomicWriter struct { // instead of the given name. The file is created with secure (0600) // permissions. func CreateAtomic(path string) (*AtomicWriter, error) { + fs := fs.NewFilesystem(fs.FilesystemTypeBasic, filepath.Dir(path)) + return CreateAtomicFilesystem(fs, filepath.Base(path)) +} + +// CreateAtomicFilesystem is like os.Create, except a temporary file name is used +// instead of the given name. The file is created with secure (0600) +// permissions. +func CreateAtomicFilesystem(filesystem fs.Filesystem, path string) (*AtomicWriter, error) { // The security of this depends on the tempfile having secure // permissions, 0600, from the beginning. This is what ioutil.TempFile // does. We have a test that verifies that that is the case, should this // ever change in the standard library in the future. - fd, err := ioutil.TempFile(filepath.Dir(path), TempPrefix) + fd, err := TempFile(filesystem, filepath.Dir(path), TempPrefix) if err != nil { return nil, err } @@ -45,6 +54,7 @@ func CreateAtomic(path string) (*AtomicWriter, error) { w := &AtomicWriter{ path: path, next: fd, + fs: filesystem, } return w, nil @@ -71,7 +81,7 @@ func (w *AtomicWriter) Close() error { } // Try to not leave temp file around, but ignore error. - defer os.Remove(w.next.Name()) + defer w.fs.Remove(w.next.Name()) if err := w.next.Sync(); err != nil { w.err = err @@ -88,17 +98,21 @@ func (w *AtomicWriter) Close() error { // either. Return this error because it may be more informative. On non- // Windows we want the atomic rename behavior so we don't attempt remove. if runtime.GOOS == "windows" { - if err := os.Remove(w.path); err != nil && !os.IsNotExist(err) { + if err := w.fs.Remove(w.path); err != nil && !fs.IsNotExist(err) { return err } } - if err := os.Rename(w.next.Name(), w.path); err != nil { + if err := w.fs.Rename(w.next.Name(), w.path); err != nil { w.err = err return err } - SyncDir(filepath.Dir(w.next.Name())) + // fsync the directory too + if fd, err := w.fs.Open(filepath.Dir(w.next.Name())); err == nil { + fd.Sync() + fd.Close() + } // Set w.err to return appropriately for any future operations. w.err = ErrClosed diff --git a/lib/osutil/fsroots_unix.go b/lib/osutil/fsroots_unix.go deleted file mode 100644 index 39422298d..000000000 --- a/lib/osutil/fsroots_unix.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (C) 2016 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -// +build !windows - -package osutil - -func GetFilesystemRoots() ([]string, error) { - return []string{"/"}, nil -} diff --git a/lib/osutil/fsroots_windows.go b/lib/osutil/fsroots_windows.go deleted file mode 100644 index 144ba0a30..000000000 --- a/lib/osutil/fsroots_windows.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (C) 2016 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -// +build windows - -package osutil - -import ( - "bytes" - "fmt" - "syscall" - "unsafe" -) - -func GetFilesystemRoots() ([]string, error) { - kernel32, err := syscall.LoadDLL("kernel32.dll") - if err != nil { - return nil, err - } - getLogicalDriveStringsHandle, err := kernel32.FindProc("GetLogicalDriveStringsA") - if err != nil { - return nil, err - } - - buffer := [1024]byte{} - bufferSize := uint32(len(buffer)) - - hr, _, _ := getLogicalDriveStringsHandle.Call(uintptr(unsafe.Pointer(&bufferSize)), uintptr(unsafe.Pointer(&buffer))) - if hr == 0 { - return nil, fmt.Errorf("Syscall failed") - } - - var drives []string - parts := bytes.Split(buffer[:], []byte{0}) - for _, part := range parts { - if len(part) == 0 { - break - } - drives = append(drives, string(part)) - } - - return drives, nil -} diff --git a/lib/osutil/glob_unix.go b/lib/osutil/glob_unix.go deleted file mode 100644 index 8a5731781..000000000 --- a/lib/osutil/glob_unix.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (C) 2015 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -// +build !windows - -package osutil - -import ( - "path/filepath" -) - -func Glob(pattern string) (matches []string, err error) { - return filepath.Glob(pattern) -} diff --git a/lib/osutil/glob_windows.go b/lib/osutil/glob_windows.go deleted file mode 100644 index 385f60793..000000000 --- a/lib/osutil/glob_windows.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (C) 2015 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -// +build windows - -package osutil - -import ( - "os" - "path/filepath" - "runtime" - "sort" - "strings" -) - -// Glob implements filepath.Glob, but works with Windows long path prefixes. -// Deals with https://github.com/golang/go/issues/10577 -func Glob(pattern string) (matches []string, err error) { - if !hasMeta(pattern) { - if _, err = os.Lstat(pattern); err != nil { - return nil, nil - } - return []string{pattern}, nil - } - - dir, file := filepath.Split(filepath.Clean(pattern)) - switch dir { - case "": - dir = "." - case string(filepath.Separator): - // nothing - default: - if runtime.GOOS != "windows" || len(dir) < 2 || dir[len(dir)-2] != ':' { - dir = dir[0 : len(dir)-1] // chop off trailing separator, if it's not after the drive letter - } - } - - if !hasMeta(dir) { - return glob(dir, file, nil) - } - - var m []string - m, err = Glob(dir) - if err != nil { - return - } - for _, d := range m { - matches, err = glob(d, file, matches) - if err != nil { - return - } - } - return -} - -func hasMeta(path string) bool { - // Strip off Windows long path prefix if it exists. - if strings.HasPrefix(path, "\\\\?\\") { - path = path[4:] - } - // TODO(niemeyer): Should other magic characters be added here? - return strings.IndexAny(path, "*?[") >= 0 -} - -func glob(dir, pattern string, matches []string) (m []string, e error) { - m = matches - fi, err := os.Stat(dir) - if err != nil { - return - } - if !fi.IsDir() { - return - } - d, err := os.Open(dir) - if err != nil { - return - } - defer d.Close() - - names, _ := d.Readdirnames(-1) - sort.Strings(names) - - for _, n := range names { - matched, err := filepath.Match(pattern, n) - if err != nil { - return m, err - } - if matched { - m = append(m, filepath.Join(dir, n)) - } - } - return -} diff --git a/lib/osutil/glob_windows_test.go b/lib/osutil/glob_windows_test.go deleted file mode 100644 index ea62e3370..000000000 --- a/lib/osutil/glob_windows_test.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -// +build windows - -package osutil_test - -import ( - "testing" - - "github.com/syncthing/syncthing/lib/osutil" -) - -func TestGlob(t *testing.T) { - testcases := []string{ - `C:\*`, - `\\?\C:\*`, - `\\?\C:\Users`, - `\\?\\\?\C:\Users`, - } - for _, tc := range testcases { - if _, err := osutil.Glob(tc); err != nil { - t.Fatalf("pattern %s failed: %v", tc, err) - } - } -} diff --git a/lib/osutil/hidden_unix.go b/lib/osutil/hidden_unix.go index 584c05bf5..6e772c38a 100644 --- a/lib/osutil/hidden_unix.go +++ b/lib/osutil/hidden_unix.go @@ -8,12 +8,4 @@ package osutil -func HideFile(path string) error { - return nil -} - -func ShowFile(path string) error { - return nil -} - func HideConsole() {} diff --git a/lib/osutil/hidden_windows.go b/lib/osutil/hidden_windows.go index 28db3ed34..00fd4972f 100644 --- a/lib/osutil/hidden_windows.go +++ b/lib/osutil/hidden_windows.go @@ -10,36 +10,6 @@ package osutil import "syscall" -func HideFile(path string) error { - p, err := syscall.UTF16PtrFromString(path) - if err != nil { - return err - } - - attrs, err := syscall.GetFileAttributes(p) - if err != nil { - return err - } - - attrs |= syscall.FILE_ATTRIBUTE_HIDDEN - return syscall.SetFileAttributes(p, attrs) -} - -func ShowFile(path string) error { - p, err := syscall.UTF16PtrFromString(path) - if err != nil { - return err - } - - attrs, err := syscall.GetFileAttributes(p) - if err != nil { - return err - } - - attrs &^= syscall.FILE_ATTRIBUTE_HIDDEN - return syscall.SetFileAttributes(p, attrs) -} - func HideConsole() { getConsoleWindow := syscall.NewLazyDLL("kernel32.dll").NewProc("GetConsoleWindow") showWindow := syscall.NewLazyDLL("user32.dll").NewProc("ShowWindow") diff --git a/lib/osutil/lstat_broken.go b/lib/osutil/lstat_broken.go deleted file mode 100644 index 02a711692..000000000 --- a/lib/osutil/lstat_broken.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (C) 2015 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -// +build linux android - -package osutil - -import ( - "os" - "syscall" - "time" -) - -// Lstat is like os.Lstat, except lobotomized for Android. See -// https://forum.syncthing.net/t/2395 -func Lstat(name string) (fi os.FileInfo, err error) { - for i := 0; i < 10; i++ { // We have to draw the line somewhere - fi, err = os.Lstat(name) - if err, ok := err.(*os.PathError); ok && err.Err == syscall.EINTR { - time.Sleep(time.Duration(i+1) * time.Millisecond) - continue - } - return - } - return -} diff --git a/lib/osutil/lstat_ok.go b/lib/osutil/lstat_ok.go deleted file mode 100644 index c73f75a43..000000000 --- a/lib/osutil/lstat_ok.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (C) 2015 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -// +build !linux,!android - -package osutil - -import "os" - -func Lstat(name string) (fi os.FileInfo, err error) { - return os.Lstat(name) -} diff --git a/lib/osutil/mkdirall.go b/lib/osutil/mkdirall.go deleted file mode 100644 index d96aa0aee..000000000 --- a/lib/osutil/mkdirall.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (C) 2015 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -// +build !windows - -package osutil - -import ( - "os" -) - -func MkdirAll(path string, perm os.FileMode) error { - return os.MkdirAll(path, perm) -} diff --git a/lib/osutil/mkdirall_windows.go b/lib/osutil/mkdirall_windows.go deleted file mode 100644 index d4f738646..000000000 --- a/lib/osutil/mkdirall_windows.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Modified by Zillode to fix https://github.com/syncthing/syncthing/issues/1822 -// Sync with https://github.com/golang/go/blob/master/src/os/path.go -// See https://github.com/golang/go/issues/10900 - -package osutil - -import ( - "os" - "path/filepath" - "syscall" -) - -// MkdirAll creates a directory named path, along with any necessary parents, -// and returns nil, or else returns an error. -// The permission bits perm are used for all directories that MkdirAll creates. -// If path is already a directory, MkdirAll does nothing and returns nil. -func MkdirAll(path string, perm os.FileMode) error { - // Fast path: if we can tell whether path is a directory or file, stop with success or error. - dir, err := os.Stat(path) - if err == nil { - if dir.IsDir() { - return nil - } - return &os.PathError{ - Op: "mkdir", - Path: path, - Err: syscall.ENOTDIR, - } - } - - // Slow path: make sure parent exists and then call Mkdir for path. - i := len(path) - for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator. - i-- - } - - j := i - for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element. - j-- - } - - if j > 1 { - // Create parent - parent := path[0 : j-1] - if parent != filepath.VolumeName(parent) { - err = MkdirAll(parent, perm) - if err != nil { - return err - } - } - } - - // Parent now exists; invoke Mkdir and use its result. - err = os.Mkdir(path, perm) - if err != nil { - // Handle arguments like "foo/." by - // double-checking that directory doesn't exist. - dir, err1 := os.Lstat(path) - if err1 == nil && dir.IsDir() { - return nil - } - return err - } - return nil -} diff --git a/lib/osutil/net.go b/lib/osutil/net.go new file mode 100644 index 000000000..fe7f36ab1 --- /dev/null +++ b/lib/osutil/net.go @@ -0,0 +1,56 @@ +// Copyright (C) 2015 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package osutil + +import ( + "bytes" + "net" +) + +// ResolveInterfaceAddresses returns available addresses of the given network +// type for a given interface. +func ResolveInterfaceAddresses(network, nameOrMac string) []string { + intf, err := net.InterfaceByName(nameOrMac) + if err == nil { + return interfaceAddresses(network, intf) + } + + mac, err := net.ParseMAC(nameOrMac) + if err != nil { + return []string{nameOrMac} + } + + intfs, err := net.Interfaces() + if err != nil { + return []string{nameOrMac} + } + + for _, intf := range intfs { + if bytes.Equal(intf.HardwareAddr, mac) { + return interfaceAddresses(network, &intf) + } + } + + return []string{nameOrMac} +} + +func interfaceAddresses(network string, intf *net.Interface) []string { + var out []string + addrs, err := intf.Addrs() + if err != nil { + return out + } + + for _, addr := range addrs { + ipnet, ok := addr.(*net.IPNet) + if ok && (network == "tcp" || (network == "tcp4" && len(ipnet.IP) == net.IPv4len) || (network == "tcp6" && len(ipnet.IP) == net.IPv6len)) { + out = append(out, ipnet.IP.String()) + } + } + + return out +} diff --git a/lib/osutil/osutil.go b/lib/osutil/osutil.go index aa8a1fdde..889509f3d 100644 --- a/lib/osutil/osutil.go +++ b/lib/osutil/osutil.go @@ -9,19 +9,16 @@ package osutil import ( "errors" - "fmt" "io" "os" "path/filepath" "runtime" "strings" - "github.com/calmh/du" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/sync" ) -var errNoHome = errors.New("no home directory found - set $HOME (or the platform equivalent)") - // Try to keep this entire operation atomic-like. We shouldn't be doing this // often enough that there is any contention on this lock. var renameLock = sync.NewMutex() @@ -29,12 +26,12 @@ var renameLock = sync.NewMutex() // TryRename renames a file, leaving source file intact in case of failure. // Tries hard to succeed on various systems by temporarily tweaking directory // permissions and removing the destination file when necessary. -func TryRename(from, to string) error { +func TryRename(filesystem fs.Filesystem, from, to string) error { renameLock.Lock() defer renameLock.Unlock() - return withPreparedTarget(from, to, func() error { - return os.Rename(from, to) + return withPreparedTarget(filesystem, from, to, func() error { + return filesystem.Rename(from, to) }) } @@ -43,28 +40,28 @@ func TryRename(from, to string) error { // for situations like committing a temp file to it's final location. // Tries hard to succeed on various systems by temporarily tweaking directory // permissions and removing the destination file when necessary. -func Rename(from, to string) error { +func Rename(filesystem fs.Filesystem, from, to string) error { // Don't leave a dangling temp file in case of rename error if !(runtime.GOOS == "windows" && strings.EqualFold(from, to)) { - defer os.Remove(from) + defer filesystem.Remove(from) } - return TryRename(from, to) + return TryRename(filesystem, from, to) } // Copy copies the file content from source to destination. // Tries hard to succeed on various systems by temporarily tweaking directory // permissions and removing the destination file when necessary. -func Copy(from, to string) (err error) { - return withPreparedTarget(from, to, func() error { - return copyFileContents(from, to) +func Copy(filesystem fs.Filesystem, from, to string) (err error) { + return withPreparedTarget(filesystem, from, to, func() error { + return copyFileContents(filesystem, from, to) }) } // InWritableDir calls fn(path), while making sure that the directory // containing `path` is writable for the duration of the call. -func InWritableDir(fn func(string) error, path string) error { +func InWritableDir(fn func(string) error, fs fs.Filesystem, path string) error { dir := filepath.Dir(path) - info, err := os.Stat(dir) + info, err := fs.Stat(dir) if err != nil { return err } @@ -75,10 +72,10 @@ func InWritableDir(fn func(string) error, path string) error { // A non-writeable directory (for this user; we assume that's the // relevant part). Temporarily change the mode so we can delete the // file or directory inside it. - err = os.Chmod(dir, 0755) + err = fs.Chmod(dir, 0755) if err == nil { defer func() { - err = os.Chmod(dir, info.Mode()) + err = fs.Chmod(dir, info.Mode()) if err != nil { // We managed to change the permission bits like a // millisecond ago, so it'd be bizarre if we couldn't @@ -92,59 +89,22 @@ func InWritableDir(fn func(string) error, path string) error { return fn(path) } -func ExpandTilde(path string) (string, error) { - if path == "~" { - return getHomeDir() - } - - path = filepath.FromSlash(path) - if !strings.HasPrefix(path, fmt.Sprintf("~%c", os.PathSeparator)) { - return path, nil - } - - home, err := getHomeDir() - if err != nil { - return "", err - } - return filepath.Join(home, path[2:]), nil -} - -func getHomeDir() (string, error) { - var home string - - switch runtime.GOOS { - case "windows": - home = filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath")) - if home == "" { - home = os.Getenv("UserProfile") - } - default: - home = os.Getenv("HOME") - } - - if home == "" { - return "", errNoHome - } - - return home, nil -} - // Tries hard to succeed on various systems by temporarily tweaking directory // permissions and removing the destination file when necessary. -func withPreparedTarget(from, to string, f func() error) error { +func withPreparedTarget(filesystem fs.Filesystem, from, to string, f func() error) error { // Make sure the destination directory is writeable toDir := filepath.Dir(to) - if info, err := os.Stat(toDir); err == nil && info.IsDir() && info.Mode()&0200 == 0 { - os.Chmod(toDir, 0755) - defer os.Chmod(toDir, info.Mode()) + if info, err := filesystem.Stat(toDir); err == nil && info.IsDir() && info.Mode()&0200 == 0 { + filesystem.Chmod(toDir, 0755) + defer filesystem.Chmod(toDir, info.Mode()) } // On Windows, make sure the destination file is writeable (or we can't delete it) if runtime.GOOS == "windows" { - os.Chmod(to, 0666) + filesystem.Chmod(to, 0666) if !strings.EqualFold(from, to) { - err := os.Remove(to) - if err != nil && !os.IsNotExist(err) { + err := filesystem.Remove(to) + if err != nil && !fs.IsNotExist(err) { return err } } @@ -156,13 +116,13 @@ func withPreparedTarget(from, to string, f func() error) error { // by dst. The file will be created if it does not already exist. If the // destination file exists, all it's contents will be replaced by the contents // of the source file. -func copyFileContents(src, dst string) (err error) { - in, err := os.Open(src) +func copyFileContents(filesystem fs.Filesystem, src, dst string) (err error) { + in, err := filesystem.Open(src) if err != nil { return } defer in.Close() - out, err := os.Create(dst) + out, err := filesystem.Create(dst) if err != nil { return } @@ -193,13 +153,3 @@ func init() { func IsWindowsExecutable(path string) bool { return execExts[strings.ToLower(filepath.Ext(path))] } - -func DiskFreeBytes(path string) (free int64, err error) { - u, err := du.Get(path) - return u.FreeBytes, err -} - -func DiskFreePercentage(path string) (freePct float64, err error) { - u, err := du.Get(path) - return (float64(u.FreeBytes) / float64(u.TotalBytes)) * 100, err -} diff --git a/lib/osutil/osutil_test.go b/lib/osutil/osutil_test.go index 4fc6c5797..82e41ffe2 100644 --- a/lib/osutil/osutil_test.go +++ b/lib/osutil/osutil_test.go @@ -11,6 +11,7 @@ import ( "runtime" "testing" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/osutil" ) @@ -21,6 +22,8 @@ func TestInWriteableDir(t *testing.T) { } defer os.RemoveAll("testdata") + fs := fs.NewFilesystem(fs.FilesystemTypeBasic, ".") + os.Mkdir("testdata", 0700) os.Mkdir("testdata/rw", 0700) os.Mkdir("testdata/ro", 0500) @@ -36,35 +39,35 @@ func TestInWriteableDir(t *testing.T) { // These should succeed - err = osutil.InWritableDir(create, "testdata/file") + err = osutil.InWritableDir(create, fs, "testdata/file") if err != nil { t.Error("testdata/file:", err) } - err = osutil.InWritableDir(create, "testdata/rw/foo") + err = osutil.InWritableDir(create, fs, "testdata/rw/foo") if err != nil { t.Error("testdata/rw/foo:", err) } - err = osutil.InWritableDir(os.Remove, "testdata/rw/foo") + err = osutil.InWritableDir(os.Remove, fs, "testdata/rw/foo") if err != nil { t.Error("testdata/rw/foo:", err) } - err = osutil.InWritableDir(create, "testdata/ro/foo") + err = osutil.InWritableDir(create, fs, "testdata/ro/foo") if err != nil { t.Error("testdata/ro/foo:", err) } - err = osutil.InWritableDir(os.Remove, "testdata/ro/foo") + err = osutil.InWritableDir(os.Remove, fs, "testdata/ro/foo") if err != nil { t.Error("testdata/ro/foo:", err) } // These should not - err = osutil.InWritableDir(create, "testdata/nonexistent/foo") + err = osutil.InWritableDir(create, fs, "testdata/nonexistent/foo") if err == nil { t.Error("testdata/nonexistent/foo returned nil error") } - err = osutil.InWritableDir(create, "testdata/file/foo") + err = osutil.InWritableDir(create, fs, "testdata/file/foo") if err == nil { t.Error("testdata/file/foo returned nil error") } @@ -101,8 +104,10 @@ func TestInWritableDirWindowsRemove(t *testing.T) { create("testdata/windows/ro/readonly") os.Chmod("testdata/windows/ro/readonly", 0500) + fs := fs.NewFilesystem(fs.FilesystemTypeBasic, ".") + for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} { - err := osutil.InWritableDir(os.Remove, path) + err := osutil.InWritableDir(os.Remove, fs, path) if err != nil { t.Errorf("Unexpected error %s: %s", path, err) } @@ -174,6 +179,8 @@ func TestInWritableDirWindowsRename(t *testing.T) { create("testdata/windows/ro/readonly") os.Chmod("testdata/windows/ro/readonly", 0500) + fs := fs.NewFilesystem(fs.FilesystemTypeBasic, ".") + for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} { err := os.Rename(path, path+"new") if err == nil { @@ -183,11 +190,11 @@ func TestInWritableDirWindowsRename(t *testing.T) { } rename := func(path string) error { - return osutil.Rename(path, path+"new") + return osutil.Rename(fs, path, path+"new") } for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} { - err := osutil.InWritableDir(rename, path) + err := osutil.InWritableDir(rename, fs, path) if err != nil { t.Errorf("Unexpected error %s: %s", path, err) } @@ -197,18 +204,3 @@ func TestInWritableDirWindowsRename(t *testing.T) { } } } - -func TestDiskUsage(t *testing.T) { - free, err := osutil.DiskFreePercentage(".") - if err != nil { - if runtime.GOOS == "netbsd" || - runtime.GOOS == "openbsd" || - runtime.GOOS == "solaris" { - t.Skip() - } - t.Errorf("Unexpected error: %s", err) - } - if free < 1 { - t.Error("Disk is full?", free) - } -} diff --git a/lib/osutil/sync.go b/lib/osutil/sync.go deleted file mode 100644 index 9b5bd2ce8..000000000 --- a/lib/osutil/sync.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (C) 2016 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -package osutil - -import ( - "os" - "runtime" -) - -func SyncFile(path string) error { - flag := 0 - if runtime.GOOS == "windows" { - flag = os.O_WRONLY - } - fd, err := os.OpenFile(path, flag, 0) - if err != nil { - return err - } - defer fd.Close() - // MacOS and Windows do not flush the disk cache - return fd.Sync() -} - -func SyncDir(path string) error { - if runtime.GOOS == "windows" { - // not supported by Windows - return nil - } - return SyncFile(path) -} diff --git a/lib/osutil/tempfile.go b/lib/osutil/tempfile.go new file mode 100644 index 000000000..5e9fc2027 --- /dev/null +++ b/lib/osutil/tempfile.go @@ -0,0 +1,63 @@ +// Copyright (C) 2015 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package osutil + +import ( + "os" + "path/filepath" + "strconv" + "sync" + "time" + + "github.com/syncthing/syncthing/lib/fs" +) + +var rand uint32 +var randmu sync.Mutex + +func reseed() uint32 { + return uint32(time.Now().UnixNano() + int64(os.Getpid())) +} + +func nextSuffix() string { + randmu.Lock() + r := rand + if r == 0 { + r = reseed() + } + r = r*1664525 + 1013904223 // constants from Numerical Recipes + rand = r + randmu.Unlock() + return strconv.Itoa(int(1e9 + r%1e9))[1:] +} + +// TempFile creates a new temporary file in the directory dir +// with a name beginning with prefix, opens the file for reading +// and writing, and returns the resulting *os.File. +// If dir is the empty string, TempFile uses the default directory +// for temporary files (see os.TempDir). +// Multiple programs calling TempFile simultaneously +// will not choose the same file. The caller can use f.Name() +// to find the pathname of the file. It is the caller's responsibility +// to remove the file when no longer needed. +func TempFile(filesystem fs.Filesystem, dir, prefix string) (f fs.File, err error) { + nconflict := 0 + for i := 0; i < 10000; i++ { + name := filepath.Join(dir, prefix+nextSuffix()) + f, err = filesystem.OpenFile(name, fs.OptReadWrite|fs.OptCreate|fs.OptExclusive, 0600) + if fs.IsExist(err) { + if nconflict++; nconflict > 10 { + randmu.Lock() + rand = reseed() + randmu.Unlock() + } + continue + } + break + } + return +} diff --git a/lib/osutil/traversessymlink.go b/lib/osutil/traversessymlink.go index 36ff98e49..52d199c37 100644 --- a/lib/osutil/traversessymlink.go +++ b/lib/osutil/traversessymlink.go @@ -8,9 +8,10 @@ package osutil import ( "fmt" - "os" "path/filepath" "strings" + + "github.com/syncthing/syncthing/lib/fs" ) // TraversesSymlinkError is an error indicating symlink traversal @@ -34,9 +35,10 @@ func (e NotADirectoryError) Error() string { // TraversesSymlink returns an error if base and any path component of name up to and // including filepath.Join(base, name) traverses a symlink. // Base and name must both be clean and name must be relative to base. -func TraversesSymlink(base, name string) error { +func TraversesSymlink(filesystem fs.Filesystem, name string) error { + base := "." path := base - info, err := Lstat(path) + info, err := filesystem.Lstat(path) if err != nil { return err } @@ -51,17 +53,17 @@ func TraversesSymlink(base, name string) error { return nil } - parts := strings.Split(name, string(os.PathSeparator)) + parts := strings.Split(name, string(fs.PathSeparator)) for _, part := range parts { path = filepath.Join(path, part) - info, err := Lstat(path) + info, err := filesystem.Lstat(path) if err != nil { - if os.IsNotExist(err) { + if fs.IsNotExist(err) { return nil } return err } - if info.Mode()&os.ModeSymlink != 0 { + if info.IsSymlink() { return &TraversesSymlinkError{ path: strings.TrimPrefix(path, base), } diff --git a/lib/osutil/traversessymlink_test.go b/lib/osutil/traversessymlink_test.go index 6676006b1..c1ec67a8a 100644 --- a/lib/osutil/traversessymlink_test.go +++ b/lib/osutil/traversessymlink_test.go @@ -12,17 +12,20 @@ import ( "os" "testing" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/osutil" ) func TestTraversesSymlink(t *testing.T) { os.RemoveAll("testdata") defer os.RemoveAll("testdata") - os.MkdirAll("testdata/a/b/c", 0755) - os.Symlink("b", "testdata/a/l") + + fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata") + fs.MkdirAll("a/b/c", 0755) + fs.CreateSymlink("b", "a/l") // a/l -> b, so a/l/c should resolve by normal stat - info, err := osutil.Lstat("testdata/a/l/c") + info, err := fs.Lstat("a/l/c") if err != nil { t.Fatal("unexpected error", err) } @@ -52,7 +55,7 @@ func TestTraversesSymlink(t *testing.T) { } for _, tc := range cases { - if res := osutil.TraversesSymlink("testdata", tc.name); tc.traverses == (res == nil) { + if res := osutil.TraversesSymlink(fs, tc.name); tc.traverses == (res == nil) { t.Errorf("TraversesSymlink(%q) = %v, should be %v", tc.name, res, tc.traverses) } } @@ -63,10 +66,11 @@ var traversesSymlinkResult error func BenchmarkTraversesSymlink(b *testing.B) { os.RemoveAll("testdata") defer os.RemoveAll("testdata") - os.MkdirAll("testdata/a/b/c", 0755) + fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata") + fs.MkdirAll("a/b/c", 0755) for i := 0; i < b.N; i++ { - traversesSymlinkResult = osutil.TraversesSymlink("testdata", "a/b/c") + traversesSymlinkResult = osutil.TraversesSymlink(fs, "a/b/c") } b.ReportAllocs() diff --git a/lib/scanner/blockqueue.go b/lib/scanner/blockqueue.go index f164c9f17..e1ef4be1f 100644 --- a/lib/scanner/blockqueue.go +++ b/lib/scanner/blockqueue.go @@ -9,7 +9,6 @@ package scanner import ( "context" "errors" - "path/filepath" "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/protocol" @@ -64,7 +63,6 @@ func HashFile(ctx context.Context, fs fs.Filesystem, path string, blockSize int, // is closed and all items handled. type parallelHasher struct { fs fs.Filesystem - dir string blockSize int workers int outbox chan<- protocol.FileInfo @@ -75,10 +73,9 @@ type parallelHasher struct { wg sync.WaitGroup } -func newParallelHasher(ctx context.Context, fs fs.Filesystem, dir string, blockSize, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) { +func newParallelHasher(ctx context.Context, fs fs.Filesystem, blockSize, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) { ph := ¶llelHasher{ fs: fs, - dir: dir, blockSize: blockSize, workers: workers, outbox: outbox, @@ -111,7 +108,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) { panic("Bug. Asked to hash a directory or a deleted file.") } - blocks, err := HashFile(ctx, ph.fs, filepath.Join(ph.dir, f.Name), ph.blockSize, ph.counter, ph.useWeakHashes) + blocks, err := HashFile(ctx, ph.fs, f.Name, ph.blockSize, ph.counter, ph.useWeakHashes) if err != nil { l.Debugln("hash error:", f.Name, err) continue diff --git a/lib/scanner/infinitefs_test.go b/lib/scanner/infinitefs_test.go index 4976c464f..06d9d717c 100644 --- a/lib/scanner/infinitefs_test.go +++ b/lib/scanner/infinitefs_test.go @@ -19,6 +19,7 @@ import ( ) type infiniteFS struct { + fs.Filesystem width int // number of files and directories per level depth int // number of tree levels to simulate filesize int64 // size of each file in bytes @@ -50,18 +51,6 @@ func (i infiniteFS) Open(name string) (fs.File, error) { return &fakeFile{name, i.filesize, 0}, nil } -func (infiniteFS) Chmod(name string, mode fs.FileMode) error { return errNotSupp } -func (infiniteFS) Chtimes(name string, atime time.Time, mtime time.Time) error { return errNotSupp } -func (infiniteFS) Create(name string) (fs.File, error) { return nil, errNotSupp } -func (infiniteFS) CreateSymlink(name, target string) error { return errNotSupp } -func (infiniteFS) Mkdir(name string, perm fs.FileMode) error { return errNotSupp } -func (infiniteFS) ReadSymlink(name string) (string, error) { return "", errNotSupp } -func (infiniteFS) Remove(name string) error { return errNotSupp } -func (infiniteFS) Rename(oldname, newname string) error { return errNotSupp } -func (infiniteFS) Stat(name string) (fs.FileInfo, error) { return nil, errNotSupp } -func (infiniteFS) SymlinksSupported() bool { return false } -func (infiniteFS) Walk(root string, walkFn fs.WalkFunc) error { return errNotSupp } - type fakeInfo struct { name string size int64 @@ -71,7 +60,7 @@ func (f fakeInfo) Name() string { return f.name } func (f fakeInfo) Mode() fs.FileMode { return 0755 } func (f fakeInfo) Size() int64 { return f.size } func (f fakeInfo) ModTime() time.Time { return time.Unix(1234567890, 0) } -func (f fakeInfo) IsDir() bool { return strings.Contains(filepath.Base(f.name), "dir") } +func (f fakeInfo) IsDir() bool { return strings.Contains(filepath.Base(f.name), "dir") || f.name == "." } func (f fakeInfo) IsRegular() bool { return !f.IsDir() } func (f fakeInfo) IsSymlink() bool { return false } @@ -81,6 +70,10 @@ type fakeFile struct { readOffset int64 } +func (f *fakeFile) Name() string { + return f.name +} + func (f *fakeFile) Read(bs []byte) (int, error) { remaining := f.size - f.readOffset if remaining == 0 { @@ -98,6 +91,10 @@ func (f *fakeFile) Stat() (fs.FileInfo, error) { return fakeInfo{f.name, f.size}, nil } -func (f *fakeFile) WriteAt(bs []byte, offs int64) (int, error) { return 0, errNotSupp } -func (f *fakeFile) Close() error { return nil } -func (f *fakeFile) Truncate(size int64) error { return errNotSupp } +func (f *fakeFile) Write([]byte) (int, error) { return 0, errNotSupp } +func (f *fakeFile) WriteAt([]byte, int64) (int, error) { return 0, errNotSupp } +func (f *fakeFile) Close() error { return nil } +func (f *fakeFile) Truncate(size int64) error { return errNotSupp } +func (f *fakeFile) ReadAt([]byte, int64) (int, error) { return 0, errNotSupp } +func (f *fakeFile) Seek(int64, int) (int64, error) { return 0, errNotSupp } +func (f *fakeFile) Sync() error { return nil } diff --git a/lib/scanner/walk.go b/lib/scanner/walk.go index 1b7996891..c5430c6c8 100644 --- a/lib/scanner/walk.go +++ b/lib/scanner/walk.go @@ -9,7 +9,6 @@ package scanner import ( "context" "errors" - "path/filepath" "runtime" "sync/atomic" "time" @@ -42,8 +41,6 @@ func init() { type Config struct { // Folder for which the walker has been created Folder string - // Dir is the base directory for the walk - Dir string // Limit walking to these paths within Dir, or no limit if Sub is empty Subs []string // BlockSize controls the size of the block used when hashing. @@ -86,7 +83,7 @@ func Walk(ctx context.Context, cfg Config) (chan protocol.FileInfo, error) { w.CurrentFiler = noCurrentFiler{} } if w.Filesystem == nil { - w.Filesystem = fs.DefaultFilesystem + panic("no filesystem specified") } return w.walk(ctx) @@ -99,7 +96,7 @@ type walker struct { // Walk returns the list of files found in the local folder by scanning the // file system. Files are blockwise hashed. func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) { - l.Debugln("Walk", w.Dir, w.Subs, w.BlockSize, w.Matcher) + l.Debugln("Walk", w.Subs, w.BlockSize, w.Matcher) if err := w.checkDir(); err != nil { return nil, err @@ -113,10 +110,10 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) { go func() { hashFiles := w.walkAndHashFiles(ctx, toHashChan, finishedChan) if len(w.Subs) == 0 { - w.Filesystem.Walk(w.Dir, hashFiles) + w.Filesystem.Walk(".", hashFiles) } else { for _, sub := range w.Subs { - w.Filesystem.Walk(filepath.Join(w.Dir, sub), hashFiles) + w.Filesystem.Walk(sub, hashFiles) } } close(toHashChan) @@ -125,7 +122,7 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) { // We're not required to emit scan progress events, just kick off hashers, // and feed inputs directly from the walker. if w.ProgressTickIntervalS < 0 { - newParallelHasher(ctx, w.Filesystem, w.Dir, w.BlockSize, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes) + newParallelHasher(ctx, w.Filesystem, w.BlockSize, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes) return finishedChan, nil } @@ -156,7 +153,7 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) { done := make(chan struct{}) progress := newByteCounter() - newParallelHasher(ctx, w.Filesystem, w.Dir, w.BlockSize, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes) + newParallelHasher(ctx, w.Filesystem, w.BlockSize, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes) // A routine which actually emits the FolderScanProgress events // every w.ProgressTicker ticks, until the hasher routines terminate. @@ -166,13 +163,13 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) { for { select { case <-done: - l.Debugln("Walk progress done", w.Dir, w.Subs, w.BlockSize, w.Matcher) + l.Debugln("Walk progress done", w.Folder, w.Subs, w.BlockSize, w.Matcher) ticker.Stop() return case <-ticker.C: current := progress.Total() rate := progress.Rate() - l.Debugf("Walk %s %s current progress %d/%d at %.01f MiB/s (%d%%)", w.Dir, w.Subs, current, total, rate/1024/1024, current*100/total) + l.Debugf("Walk %s %s current progress %d/%d at %.01f MiB/s (%d%%)", w.Folder, w.Subs, current, total, rate/1024/1024, current*100/total) events.Default.Log(events.FolderScanProgress, map[string]interface{}{ "folder": w.Folder, "current": current, @@ -203,7 +200,7 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) { func (w *walker) walkAndHashFiles(ctx context.Context, fchan, dchan chan protocol.FileInfo) fs.WalkFunc { now := time.Now() - return func(absPath string, info fs.FileInfo, err error) error { + return func(path string, info fs.FileInfo, err error) error { select { case <-ctx.Done(): return ctx.Err() @@ -219,58 +216,52 @@ func (w *walker) walkAndHashFiles(ctx context.Context, fchan, dchan chan protoco } if err != nil { - l.Debugln("error:", absPath, info, err) + l.Debugln("error:", path, info, err) return skip } - relPath, err := filepath.Rel(w.Dir, absPath) - if err != nil { - l.Debugln("rel error:", absPath, err) - return skip - } - - if relPath == "." { + if path == "." { return nil } - info, err = w.Filesystem.Lstat(absPath) + info, err = w.Filesystem.Lstat(path) // An error here would be weird as we've already gotten to this point, but act on it nonetheless if err != nil { return skip } - if ignore.IsTemporary(relPath) { - l.Debugln("temporary:", relPath) + if ignore.IsTemporary(path) { + l.Debugln("temporary:", path) if info.IsRegular() && info.ModTime().Add(w.TempLifetime).Before(now) { - w.Filesystem.Remove(absPath) - l.Debugln("removing temporary:", relPath, info.ModTime()) + w.Filesystem.Remove(path) + l.Debugln("removing temporary:", path, info.ModTime()) } return nil } - if ignore.IsInternal(relPath) { - l.Debugln("ignored (internal):", relPath) + if ignore.IsInternal(path) { + l.Debugln("ignored (internal):", path) return skip } - if w.Matcher.Match(relPath).IsIgnored() { - l.Debugln("ignored (patterns):", relPath) + if w.Matcher.Match(path).IsIgnored() { + l.Debugln("ignored (patterns):", path) return skip } - if !utf8.ValidString(relPath) { - l.Warnf("File name %q is not in UTF8 encoding; skipping.", relPath) + if !utf8.ValidString(path) { + l.Warnf("File name %q is not in UTF8 encoding; skipping.", path) return skip } - relPath, shouldSkip := w.normalizePath(absPath, relPath) + path, shouldSkip := w.normalizePath(path) if shouldSkip { return skip } switch { case info.IsSymlink(): - if err := w.walkSymlink(ctx, absPath, relPath, dchan); err != nil { + if err := w.walkSymlink(ctx, path, dchan); err != nil { return err } if info.IsDir() { @@ -280,10 +271,10 @@ func (w *walker) walkAndHashFiles(ctx context.Context, fchan, dchan chan protoco return nil case info.IsDir(): - err = w.walkDir(ctx, relPath, info, dchan) + err = w.walkDir(ctx, path, info, dchan) case info.IsRegular(): - err = w.walkRegular(ctx, relPath, info, fchan) + err = w.walkRegular(ctx, path, info, fchan) } return err @@ -375,7 +366,7 @@ func (w *walker) walkDir(ctx context.Context, relPath string, info fs.FileInfo, // walkSymlink returns nil or an error, if the error is of the nature that // it should stop the entire walk. -func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan chan protocol.FileInfo) error { +func (w *walker) walkSymlink(ctx context.Context, relPath string, dchan chan protocol.FileInfo) error { // Symlinks are not supported on Windows. We ignore instead of returning // an error. if runtime.GOOS == "windows" { @@ -387,9 +378,9 @@ func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan // checking that their existing blocks match with the blocks in // the index. - target, err := w.Filesystem.ReadSymlink(absPath) + target, err := w.Filesystem.ReadSymlink(relPath) if err != nil { - l.Debugln("readlink error:", absPath, err) + l.Debugln("readlink error:", relPath, err) return nil } @@ -413,7 +404,7 @@ func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan SymlinkTarget: target, } - l.Debugln("symlink changedb:", absPath, f) + l.Debugln("symlink changedb:", relPath, f) select { case dchan <- f: @@ -426,55 +417,58 @@ func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan // normalizePath returns the normalized relative path (possibly after fixing // it on disk), or skip is true. -func (w *walker) normalizePath(absPath, relPath string) (normPath string, skip bool) { +func (w *walker) normalizePath(path string) (normPath string, skip bool) { if runtime.GOOS == "darwin" { // Mac OS X file names should always be NFD normalized. - normPath = norm.NFD.String(relPath) + normPath = norm.NFD.String(path) } else { // Every other OS in the known universe uses NFC or just plain // doesn't bother to define an encoding. In our case *we* do care, // so we enforce NFC regardless. - normPath = norm.NFC.String(relPath) + normPath = norm.NFC.String(path) } - if relPath != normPath { + if path != normPath { // The file name was not normalized. if !w.AutoNormalize { // We're not authorized to do anything about it, so complain and skip. - l.Warnf("File name %q is not in the correct UTF8 normalization form; skipping.", relPath) + l.Warnf("File name %q is not in the correct UTF8 normalization form; skipping.", path) return "", true } // We will attempt to normalize it. - normalizedPath := filepath.Join(w.Dir, normPath) - if _, err := w.Filesystem.Lstat(normalizedPath); fs.IsNotExist(err) { + if _, err := w.Filesystem.Lstat(normPath); fs.IsNotExist(err) { // Nothing exists with the normalized filename. Good. - if err = w.Filesystem.Rename(absPath, normalizedPath); err != nil { - l.Infof(`Error normalizing UTF8 encoding of file "%s": %v`, relPath, err) + if err = w.Filesystem.Rename(path, normPath); err != nil { + l.Infof(`Error normalizing UTF8 encoding of file "%s": %v`, path, err) return "", true } - l.Infof(`Normalized UTF8 encoding of file name "%s".`, relPath) + l.Infof(`Normalized UTF8 encoding of file name "%s".`, path) } else { // There is something already in the way at the normalized // file name. - l.Infof(`File "%s" has UTF8 encoding conflict with another file; ignoring.`, relPath) + l.Infof(`File "%s" path has UTF8 encoding conflict with another file; ignoring.`, path) return "", true } } - return normPath, false + return path, false } func (w *walker) checkDir() error { - if info, err := w.Filesystem.Lstat(w.Dir); err != nil { + info, err := w.Filesystem.Lstat(".") + if err != nil { return err - } else if !info.IsDir() { - return errors.New(w.Dir + ": not a directory") - } else { - l.Debugln("checkDir", w.Dir, info) } + + if !info.IsDir() { + return errors.New(w.Filesystem.URI() + ": not a directory") + } + + l.Debugln("checkDir", w.Filesystem.Type(), w.Filesystem.URI(), info) + return nil } diff --git a/lib/scanner/walk_test.go b/lib/scanner/walk_test.go index 852d29615..871accd83 100644 --- a/lib/scanner/walk_test.go +++ b/lib/scanner/walk_test.go @@ -23,7 +23,6 @@ import ( "github.com/d4l3k/messagediff" "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/ignore" - "github.com/syncthing/syncthing/lib/osutil" "github.com/syncthing/syncthing/lib/protocol" "golang.org/x/text/unicode/norm" ) @@ -54,18 +53,18 @@ func init() { } func TestWalkSub(t *testing.T) { - ignores := ignore.New() + ignores := ignore.New(fs.NewFilesystem(fs.FilesystemTypeBasic, ".")) err := ignores.Load("testdata/.stignore") if err != nil { t.Fatal(err) } fchan, err := Walk(context.TODO(), Config{ - Dir: "testdata", - Subs: []string{"dir2"}, - BlockSize: 128 * 1024, - Matcher: ignores, - Hashers: 2, + Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), + Subs: []string{"dir2"}, + BlockSize: 128 * 1024, + Matcher: ignores, + Hashers: 2, }) var files []protocol.FileInfo for f := range fchan { @@ -90,7 +89,7 @@ func TestWalkSub(t *testing.T) { } func TestWalk(t *testing.T) { - ignores := ignore.New() + ignores := ignore.New(fs.NewFilesystem(fs.FilesystemTypeBasic, ".")) err := ignores.Load("testdata/.stignore") if err != nil { t.Fatal(err) @@ -98,10 +97,10 @@ func TestWalk(t *testing.T) { t.Log(ignores) fchan, err := Walk(context.TODO(), Config{ - Dir: "testdata", - BlockSize: 128 * 1024, - Matcher: ignores, - Hashers: 2, + Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), + BlockSize: 128 * 1024, + Matcher: ignores, + Hashers: 2, }) if err != nil { @@ -122,9 +121,9 @@ func TestWalk(t *testing.T) { func TestWalkError(t *testing.T) { _, err := Walk(context.TODO(), Config{ - Dir: "testdata-missing", - BlockSize: 128 * 1024, - Hashers: 2, + Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata-missing"), + BlockSize: 128 * 1024, + Hashers: 2, }) if err == nil { @@ -132,8 +131,8 @@ func TestWalkError(t *testing.T) { } _, err = Walk(context.TODO(), Config{ - Dir: "testdata/bar", - BlockSize: 128 * 1024, + Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata/bar"), + BlockSize: 128 * 1024, }) if err == nil { @@ -220,9 +219,11 @@ func TestNormalization(t *testing.T) { numValid := len(tests) - numInvalid + fs := fs.NewFilesystem(fs.FilesystemTypeBasic, ".") + for _, s1 := range tests { // Create a directory for each of the interesting strings above - if err := osutil.MkdirAll(filepath.Join("testdata/normalization", s1), 0755); err != nil { + if err := fs.MkdirAll(filepath.Join("testdata/normalization", s1), 0755); err != nil { t.Fatal(err) } @@ -231,10 +232,10 @@ func TestNormalization(t *testing.T) { // file names. Ensure that the file doesn't exist when it's // created. This detects and fails if there's file name // normalization stuff at the filesystem level. - if fd, err := os.OpenFile(filepath.Join("testdata/normalization", s1, s2), os.O_CREATE|os.O_EXCL, 0644); err != nil { + if fd, err := fs.OpenFile(filepath.Join("testdata/normalization", s1, s2), os.O_CREATE|os.O_EXCL, 0644); err != nil { t.Fatal(err) } else { - fd.WriteString("test") + fd.Write([]byte("test")) fd.Close() } } @@ -245,11 +246,11 @@ func TestNormalization(t *testing.T) { // make sure it all gets done. In production, things will be correct // eventually... - _, err := walkDir("testdata/normalization") + _, err := walkDir(fs, "testdata/normalization") if err != nil { t.Fatal(err) } - tmp, err := walkDir("testdata/normalization") + tmp, err := walkDir(fs, "testdata/normalization") if err != nil { t.Fatal(err) } @@ -299,8 +300,8 @@ func TestWalkSymlinkUnix(t *testing.T) { // Scan it fchan, err := Walk(context.TODO(), Config{ - Dir: "_symlinks", - BlockSize: 128 * 1024, + Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"), + BlockSize: 128 * 1024, }) if err != nil { @@ -344,8 +345,8 @@ func TestWalkSymlinkWindows(t *testing.T) { // Scan it fchan, err := Walk(context.TODO(), Config{ - Dir: "_symlinks", - BlockSize: 128 * 1024, + Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"), + BlockSize: 128 * 1024, }) if err != nil { @@ -364,9 +365,10 @@ func TestWalkSymlinkWindows(t *testing.T) { } } -func walkDir(dir string) ([]protocol.FileInfo, error) { +func walkDir(fs fs.Filesystem, dir string) ([]protocol.FileInfo, error) { fchan, err := Walk(context.TODO(), Config{ - Dir: dir, + Filesystem: fs, + Subs: []string{dir}, BlockSize: 128 * 1024, AutoNormalize: true, Hashers: 2, @@ -435,7 +437,7 @@ func BenchmarkHashFile(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - if _, err := HashFile(context.TODO(), fs.DefaultFilesystem, testdataName, protocol.BlockSize, nil, true); err != nil { + if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.BlockSize, nil, true); err != nil { b.Fatal(err) } } @@ -467,15 +469,17 @@ func TestStopWalk(t *testing.T) { // many directories. It'll take a while to scan, giving us time to // cancel it and make sure the scan stops. - fs := fs.NewWalkFilesystem(&infiniteFS{100, 100, 1e6}) + // Use an errorFs as the backing fs for the rest of the interface + // The way we get it is a bit hacky tho. + errorFs := fs.NewFilesystem(fs.FilesystemType(-1), ".") + fs := fs.NewWalkFilesystem(&infiniteFS{errorFs, 100, 100, 1e6}) const numHashers = 4 ctx, cancel := context.WithCancel(context.Background()) fchan, err := Walk(ctx, Config{ - Dir: "testdir", + Filesystem: fs, BlockSize: 128 * 1024, Hashers: numHashers, - Filesystem: fs, ProgressTickIntervalS: -1, // Don't attempt to build the full list of files before starting to scan... }) diff --git a/lib/versioner/external.go b/lib/versioner/external.go index 40c7f5c0e..3503f4d89 100644 --- a/lib/versioner/external.go +++ b/lib/versioner/external.go @@ -10,10 +10,11 @@ import ( "errors" "os" "os/exec" - "path/filepath" "strings" - "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/fs" + + "github.com/kballard/go-shellquote" ) func init() { @@ -23,15 +24,15 @@ func init() { type External struct { command string - folderPath string + filesystem fs.Filesystem } -func NewExternal(folderID, folderPath string, params map[string]string) Versioner { +func NewExternal(folderID string, filesystem fs.Filesystem, params map[string]string) Versioner { command := params["command"] s := External{ command: command, - folderPath: folderPath, + filesystem: filesystem, } l.Debugf("instantiated %#v", s) @@ -41,29 +42,41 @@ func NewExternal(folderID, folderPath string, params map[string]string) Versione // Archive moves the named file away to a version archive. If this function // returns nil, the named file does not exist any more (has been archived). func (v External) Archive(filePath string) error { - info, err := osutil.Lstat(filePath) - if os.IsNotExist(err) { + info, err := v.filesystem.Lstat(filePath) + if fs.IsNotExist(err) { l.Debugln("not archiving nonexistent file", filePath) return nil } else if err != nil { return err } - if info.Mode()&os.ModeSymlink != 0 { + if info.IsSymlink() { panic("bug: attempting to version a symlink") } l.Debugln("archiving", filePath) - inFolderPath, err := filepath.Rel(v.folderPath, filePath) - if err != nil { - return err - } - if v.command == "" { return errors.New("Versioner: command is empty, please enter a valid command") } - cmd := exec.Command(v.command, v.folderPath, inFolderPath) + words, err := shellquote.Split(v.command) + if err != nil { + return errors.New("Versioner: command is invalid: " + err.Error()) + } + + context := map[string]string{ + "%FOLDER_FILESYSTEM%": v.filesystem.Type().String(), + "%FOLDER_PATH%": v.filesystem.URI(), + "%FILE_PATH%": filePath, + } + + for i, word := range words { + if replacement, ok := context[word]; ok { + words[i] = replacement + } + } + + cmd := exec.Command(words[0], words[1:]...) env := os.Environ() // filter STGUIAUTH and STGUIAPIKEY from environment variables filteredEnv := []string{} @@ -73,13 +86,14 @@ func (v External) Archive(filePath string) error { } } cmd.Env = filteredEnv - err = cmd.Run() + combinedOutput, err := cmd.CombinedOutput() + l.Debugln("external command output:", string(combinedOutput)) if err != nil { return err } // return error if the file was not removed - if _, err = osutil.Lstat(filePath); os.IsNotExist(err) { + if _, err = v.filesystem.Lstat(filePath); fs.IsNotExist(err) { return nil } return errors.New("Versioner: file was not removed by external script") diff --git a/lib/versioner/external_test.go b/lib/versioner/external_test.go index 0ecd9760a..937cca32e 100644 --- a/lib/versioner/external_test.go +++ b/lib/versioner/external_test.go @@ -12,6 +12,8 @@ import ( "path/filepath" "runtime" "testing" + + "github.com/syncthing/syncthing/lib/fs" ) func TestExternalNoCommand(t *testing.T) { @@ -28,8 +30,8 @@ func TestExternalNoCommand(t *testing.T) { // The versioner should fail due to missing command. e := External{ + filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "."), command: "nonexistent command", - folderPath: "testdata/folder path", } if err := e.Archive(file); err == nil { t.Error("Command should have failed") @@ -43,12 +45,12 @@ func TestExternalNoCommand(t *testing.T) { } func TestExternal(t *testing.T) { - cmd := "./_external_test/external.sh" + cmd := "./_external_test/external.sh %FOLDER_PATH% %FILE_PATH%" if runtime.GOOS == "windows" { - cmd = `.\_external_test\external.bat` + cmd = `.\\_external_test\\external.bat %FOLDER_PATH% %FILE_PATH%` } - file := "testdata/folder path/dir (parens)/long filename (parens).txt" + file := filepath.Join("testdata", "folder path", "dir (parens)", "/long filename (parens).txt") prepForRemoval(t, file) defer os.RemoveAll("testdata") @@ -61,8 +63,8 @@ func TestExternal(t *testing.T) { // The versioner should run successfully. e := External{ + filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "."), command: cmd, - folderPath: "testdata/folder path", } if err := e.Archive(file); err != nil { t.Fatal(err) diff --git a/lib/versioner/simple.go b/lib/versioner/simple.go index d325234a5..8a4cd629c 100644 --- a/lib/versioner/simple.go +++ b/lib/versioner/simple.go @@ -7,10 +7,10 @@ package versioner import ( - "os" "path/filepath" "strconv" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/osutil" "github.com/syncthing/syncthing/lib/util" ) @@ -21,19 +21,19 @@ func init() { } type Simple struct { - keep int - folderPath string + keep int + fs fs.Filesystem } -func NewSimple(folderID, folderPath string, params map[string]string) Versioner { +func NewSimple(folderID string, fs fs.Filesystem, params map[string]string) Versioner { keep, err := strconv.Atoi(params["keep"]) if err != nil { keep = 5 // A reasonable default } s := Simple{ - keep: keep, - folderPath: folderPath, + keep: keep, + fs: fs, } l.Debugf("instantiated %#v", s) @@ -43,24 +43,24 @@ func NewSimple(folderID, folderPath string, params map[string]string) Versioner // Archive moves the named file away to a version archive. If this function // returns nil, the named file does not exist any more (has been archived). func (v Simple) Archive(filePath string) error { - fileInfo, err := osutil.Lstat(filePath) - if os.IsNotExist(err) { + info, err := v.fs.Lstat(filePath) + if fs.IsNotExist(err) { l.Debugln("not archiving nonexistent file", filePath) return nil } else if err != nil { return err } - if fileInfo.Mode()&os.ModeSymlink != 0 { + if info.IsSymlink() { panic("bug: attempting to version a symlink") } - versionsDir := filepath.Join(v.folderPath, ".stversions") - _, err = os.Stat(versionsDir) + versionsDir := ".stversions" + _, err = v.fs.Stat(versionsDir) if err != nil { - if os.IsNotExist(err) { - l.Debugln("creating versions dir", versionsDir) - osutil.MkdirAll(versionsDir, 0755) - osutil.HideFile(versionsDir) + if fs.IsNotExist(err) { + l.Debugln("creating versions dir .stversions") + v.fs.Mkdir(versionsDir, 0755) + v.fs.Hide(versionsDir) } else { return err } @@ -69,28 +69,25 @@ func (v Simple) Archive(filePath string) error { l.Debugln("archiving", filePath) file := filepath.Base(filePath) - inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath)) - if err != nil { - return err - } + inFolderPath := filepath.Dir(filePath) dir := filepath.Join(versionsDir, inFolderPath) - err = osutil.MkdirAll(dir, 0755) - if err != nil && !os.IsExist(err) { + err = v.fs.MkdirAll(dir, 0755) + if err != nil && !fs.IsExist(err) { return err } - ver := taggedFilename(file, fileInfo.ModTime().Format(TimeFormat)) + ver := taggedFilename(file, info.ModTime().Format(TimeFormat)) dst := filepath.Join(dir, ver) l.Debugln("moving to", dst) - err = osutil.Rename(filePath, dst) + err = osutil.Rename(v.fs, filePath, dst) if err != nil { return err } // Glob according to the new file~timestamp.ext pattern. pattern := filepath.Join(dir, taggedFilename(file, TimeGlob)) - newVersions, err := osutil.Glob(pattern) + newVersions, err := v.fs.Glob(pattern) if err != nil { l.Warnln("globbing:", err, "for", pattern) return nil @@ -98,7 +95,7 @@ func (v Simple) Archive(filePath string) error { // Also according to the old file.ext~timestamp pattern. pattern = filepath.Join(dir, file+"~"+TimeGlob) - oldVersions, err := osutil.Glob(pattern) + oldVersions, err := v.fs.Glob(pattern) if err != nil { l.Warnln("globbing:", err, "for", pattern) return nil @@ -111,7 +108,7 @@ func (v Simple) Archive(filePath string) error { if len(versions) > v.keep { for _, toRemove := range versions[:len(versions)-v.keep] { l.Debugln("cleaning out", toRemove) - err = os.Remove(toRemove) + err = v.fs.Remove(toRemove) if err != nil { l.Warnln("removing old version:", err) } diff --git a/lib/versioner/simple_test.go b/lib/versioner/simple_test.go index 43b62571b..24e0031b0 100644 --- a/lib/versioner/simple_test.go +++ b/lib/versioner/simple_test.go @@ -9,10 +9,11 @@ package versioner import ( "io/ioutil" "math" - "os" "path/filepath" "testing" "time" + + "github.com/syncthing/syncthing/lib/fs" ) func TestTaggedFilename(t *testing.T) { @@ -53,29 +54,28 @@ func TestSimpleVersioningVersionCount(t *testing.T) { } dir, err := ioutil.TempDir("", "") - defer os.RemoveAll(dir) + //defer os.RemoveAll(dir) if err != nil { t.Error(err) } - v := NewSimple("", dir, map[string]string{"keep": "2"}) - versionDir := filepath.Join(dir, ".stversions") + fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir) - path := filepath.Join(dir, "test") + v := NewSimple("", fs, map[string]string{"keep": "2"}) + + path := "test" for i := 1; i <= 3; i++ { - f, err := os.Create(path) + f, err := fs.Create(path) if err != nil { t.Error(err) } f.Close() - v.Archive(path) - - d, err := os.Open(versionDir) - if err != nil { + if err := v.Archive(path); err != nil { t.Error(err) } - n, err := d.Readdirnames(-1) + + n, err := fs.DirNames(".stversions") if err != nil { t.Error(err) } @@ -83,7 +83,6 @@ func TestSimpleVersioningVersionCount(t *testing.T) { if float64(len(n)) != math.Min(float64(i), 2) { t.Error("Wrong count") } - d.Close() time.Sleep(time.Second) } diff --git a/lib/versioner/staggered.go b/lib/versioner/staggered.go index 4af97e32c..1f69b6bdf 100644 --- a/lib/versioner/staggered.go +++ b/lib/versioner/staggered.go @@ -12,7 +12,7 @@ import ( "strconv" "time" - "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/sync" "github.com/syncthing/syncthing/lib/util" ) @@ -28,9 +28,9 @@ type Interval struct { } type Staggered struct { - versionsPath string cleanInterval int64 - folderPath string + folderFs fs.Filesystem + versionsFs fs.Filesystem interval [4]Interval mutex sync.Mutex @@ -38,7 +38,7 @@ type Staggered struct { testCleanDone chan struct{} } -func NewStaggered(folderID, folderPath string, params map[string]string) Versioner { +func NewStaggered(folderID string, folderFs fs.Filesystem, params map[string]string) Versioner { maxAge, err := strconv.ParseInt(params["maxAge"], 10, 0) if err != nil { maxAge = 31536000 // Default: ~1 year @@ -49,22 +49,20 @@ func NewStaggered(folderID, folderPath string, params map[string]string) Version } // Use custom path if set, otherwise .stversions in folderPath - var versionsDir string + var versionsFs fs.Filesystem if params["versionsPath"] == "" { - versionsDir = filepath.Join(folderPath, ".stversions") - l.Debugln("using default dir .stversions") + versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), ".stversions")) } else if filepath.IsAbs(params["versionsPath"]) { - l.Debugln("using dir", params["versionsPath"]) - versionsDir = params["versionsPath"] + versionsFs = fs.NewFilesystem(folderFs.Type(), params["versionsPath"]) } else { - versionsDir = filepath.Join(folderPath, params["versionsPath"]) - l.Debugln("using dir", versionsDir) + versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), params["versionsPath"])) } + l.Debugln("%s folder using %s (%s) staggered versioner dir", folderID, versionsFs.URI(), versionsFs.Type()) s := &Staggered{ - versionsPath: versionsDir, cleanInterval: cleanInterval, - folderPath: folderPath, + folderFs: folderFs, + versionsFs: versionsFs, interval: [4]Interval{ {30, 3600}, // first hour -> 30 sec between versions {3600, 86400}, // next day -> 1 h between versions @@ -102,12 +100,12 @@ func (v *Staggered) Stop() { } func (v *Staggered) clean() { - l.Debugln("Versioner clean: Waiting for lock on", v.versionsPath) + l.Debugln("Versioner clean: Waiting for lock on", v.versionsFs) v.mutex.Lock() defer v.mutex.Unlock() - l.Debugln("Versioner clean: Cleaning", v.versionsPath) + l.Debugln("Versioner clean: Cleaning", v.versionsFs) - if _, err := os.Stat(v.versionsPath); os.IsNotExist(err) { + if _, err := v.versionsFs.Stat("."); fs.IsNotExist(err) { // There is no need to clean a nonexistent dir. return } @@ -115,14 +113,14 @@ func (v *Staggered) clean() { versionsPerFile := make(map[string][]string) filesPerDir := make(map[string]int) - err := filepath.Walk(v.versionsPath, func(path string, f os.FileInfo, err error) error { + err := v.versionsFs.Walk(".", func(path string, f fs.FileInfo, err error) error { if err != nil { return err } - if f.Mode().IsDir() && f.Mode()&os.ModeSymlink == 0 { + if f.IsDir() && !f.IsSymlink() { filesPerDir[path] = 0 - if path != v.versionsPath { + if path != "." { dir := filepath.Dir(path) filesPerDir[dir]++ } @@ -155,25 +153,20 @@ func (v *Staggered) clean() { continue } - if path == v.versionsPath { - l.Debugln("Cleaner: versions dir is empty, don't delete", path) - continue - } - l.Debugln("Cleaner: deleting empty directory", path) - err = os.Remove(path) + err = v.versionsFs.Remove(path) if err != nil { l.Warnln("Versioner: can't remove directory", path, err) } } - l.Debugln("Cleaner: Finished cleaning", v.versionsPath) + l.Debugln("Cleaner: Finished cleaning", v.versionsFs) } func (v *Staggered) expire(versions []string) { l.Debugln("Versioner: Expiring versions", versions) for _, file := range v.toRemove(versions, time.Now()) { - if fi, err := osutil.Lstat(file); err != nil { + if fi, err := v.versionsFs.Lstat(file); err != nil { l.Warnln("versioner:", err) continue } else if fi.IsDir() { @@ -181,7 +174,7 @@ func (v *Staggered) expire(versions []string) { continue } - if err := os.Remove(file); err != nil { + if err := v.versionsFs.Remove(file); err != nil { l.Warnf("Versioner: can't remove %q: %v", file, err) } } @@ -203,7 +196,7 @@ func (v *Staggered) toRemove(versions []string, now time.Time) []string { // If the file is older than the max age of the last interval, remove it if lastIntv := v.interval[len(v.interval)-1]; lastIntv.end > 0 && age > lastIntv.end { l.Debugln("Versioner: File over maximum age -> delete ", file) - err = os.Remove(file) + err = v.versionsFs.Remove(file) if err != nil { l.Warnf("Versioner: can't remove %q: %v", file, err) } @@ -240,26 +233,26 @@ func (v *Staggered) toRemove(versions []string, now time.Time) []string { // Archive moves the named file away to a version archive. If this function // returns nil, the named file does not exist any more (has been archived). func (v *Staggered) Archive(filePath string) error { - l.Debugln("Waiting for lock on ", v.versionsPath) + l.Debugln("Waiting for lock on ", v.versionsFs) v.mutex.Lock() defer v.mutex.Unlock() - info, err := osutil.Lstat(filePath) - if os.IsNotExist(err) { + info, err := v.folderFs.Lstat(filePath) + if fs.IsNotExist(err) { l.Debugln("not archiving nonexistent file", filePath) return nil } else if err != nil { return err } - if info.Mode()&os.ModeSymlink != 0 { + if info.IsSymlink() { panic("bug: attempting to version a symlink") } - if _, err := os.Stat(v.versionsPath); err != nil { - if os.IsNotExist(err) { - l.Debugln("creating versions dir", v.versionsPath) - osutil.MkdirAll(v.versionsPath, 0755) - osutil.HideFile(v.versionsPath) + if _, err := v.versionsFs.Stat("."); err != nil { + if fs.IsNotExist(err) { + l.Debugln("creating versions dir", v.versionsFs) + v.versionsFs.MkdirAll(".", 0755) + v.versionsFs.Hide(".") } else { return err } @@ -268,36 +261,41 @@ func (v *Staggered) Archive(filePath string) error { l.Debugln("archiving", filePath) file := filepath.Base(filePath) - inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath)) + inFolderPath := filepath.Dir(filePath) if err != nil { return err } - dir := filepath.Join(v.versionsPath, inFolderPath) - err = osutil.MkdirAll(dir, 0755) - if err != nil && !os.IsExist(err) { + err = v.versionsFs.MkdirAll(inFolderPath, 0755) + if err != nil && !fs.IsExist(err) { return err } ver := taggedFilename(file, time.Now().Format(TimeFormat)) - dst := filepath.Join(dir, ver) + dst := filepath.Join(inFolderPath, ver) l.Debugln("moving to", dst) - err = osutil.Rename(filePath, dst) + + /// TODO: Fix this when we have an alternative filesystem implementation + if v.versionsFs.Type() != fs.FilesystemTypeBasic { + panic("bug: staggered versioner used with unsupported filesystem") + } + + err = os.Rename(filepath.Join(v.folderFs.URI(), filePath), filepath.Join(v.versionsFs.URI(), dst)) if err != nil { return err } // Glob according to the new file~timestamp.ext pattern. - pattern := filepath.Join(dir, taggedFilename(file, TimeGlob)) - newVersions, err := osutil.Glob(pattern) + pattern := filepath.Join(inFolderPath, taggedFilename(file, TimeGlob)) + newVersions, err := v.versionsFs.Glob(pattern) if err != nil { l.Warnln("globbing:", err, "for", pattern) return nil } // Also according to the old file.ext~timestamp pattern. - pattern = filepath.Join(dir, file+"~"+TimeGlob) - oldVersions, err := osutil.Glob(pattern) + pattern = filepath.Join(inFolderPath, file+"~"+TimeGlob) + oldVersions, err := v.versionsFs.Glob(pattern) if err != nil { l.Warnln("globbing:", err, "for", pattern) return nil diff --git a/lib/versioner/staggered_test.go b/lib/versioner/staggered_test.go index b91da7b91..cd84c069f 100644 --- a/lib/versioner/staggered_test.go +++ b/lib/versioner/staggered_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/d4l3k/messagediff" + "github.com/syncthing/syncthing/lib/fs" ) func TestStaggeredVersioningVersionCount(t *testing.T) { @@ -62,7 +63,7 @@ func TestStaggeredVersioningVersionCount(t *testing.T) { os.MkdirAll("testdata/.stversions", 0755) defer os.RemoveAll("testdata") - v := NewStaggered("", "testdata", map[string]string{"maxAge": strconv.Itoa(365 * 86400)}).(*Staggered) + v := NewStaggered("", fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), map[string]string{"maxAge": strconv.Itoa(365 * 86400)}).(*Staggered) v.testCleanDone = make(chan struct{}) defer v.Stop() go v.Serve() diff --git a/lib/versioner/trashcan.go b/lib/versioner/trashcan.go index e17b6d0f9..7f9efa685 100644 --- a/lib/versioner/trashcan.go +++ b/lib/versioner/trashcan.go @@ -8,11 +8,11 @@ package versioner import ( "fmt" - "os" "path/filepath" "strconv" "time" + "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/osutil" ) @@ -22,17 +22,17 @@ func init() { } type Trashcan struct { - folderPath string + fs fs.Filesystem cleanoutDays int stop chan struct{} } -func NewTrashcan(folderID, folderPath string, params map[string]string) Versioner { +func NewTrashcan(folderID string, fs fs.Filesystem, params map[string]string) Versioner { cleanoutDays, _ := strconv.Atoi(params["cleanoutDays"]) // On error we default to 0, "do not clean out the trash can" s := &Trashcan{ - folderPath: folderPath, + fs: fs, cleanoutDays: cleanoutDays, stop: make(chan struct{}), } @@ -44,52 +44,47 @@ func NewTrashcan(folderID, folderPath string, params map[string]string) Versione // Archive moves the named file away to a version archive. If this function // returns nil, the named file does not exist any more (has been archived). func (t *Trashcan) Archive(filePath string) error { - info, err := osutil.Lstat(filePath) - if os.IsNotExist(err) { + info, err := t.fs.Lstat(filePath) + if fs.IsNotExist(err) { l.Debugln("not archiving nonexistent file", filePath) return nil } else if err != nil { return err } - if info.Mode()&os.ModeSymlink != 0 { + if info.IsSymlink() { panic("bug: attempting to version a symlink") } - versionsDir := filepath.Join(t.folderPath, ".stversions") - if _, err := os.Stat(versionsDir); err != nil { - if !os.IsNotExist(err) { + versionsDir := ".stversions" + if _, err := t.fs.Stat(versionsDir); err != nil { + if !fs.IsNotExist(err) { return err } l.Debugln("creating versions dir", versionsDir) - if err := osutil.MkdirAll(versionsDir, 0777); err != nil { + if err := t.fs.MkdirAll(versionsDir, 0777); err != nil { return err } - osutil.HideFile(versionsDir) + t.fs.Hide(versionsDir) } l.Debugln("archiving", filePath) - relativePath, err := filepath.Rel(t.folderPath, filePath) - if err != nil { - return err - } - - archivedPath := filepath.Join(versionsDir, relativePath) - if err := osutil.MkdirAll(filepath.Dir(archivedPath), 0777); err != nil && !os.IsExist(err) { + archivedPath := filepath.Join(versionsDir, filePath) + if err := t.fs.MkdirAll(filepath.Dir(archivedPath), 0777); err != nil && !fs.IsExist(err) { return err } l.Debugln("moving to", archivedPath) - if err := osutil.Rename(filePath, archivedPath); err != nil { + if err := osutil.Rename(t.fs, filePath, archivedPath); err != nil { return err } // Set the mtime to the time the file was deleted. This is used by the // cleanout routine. If this fails things won't work optimally but there's // not much we can do about it so we ignore the error. - os.Chtimes(archivedPath, time.Now(), time.Now()) + t.fs.Chtimes(archivedPath, time.Now(), time.Now()) return nil } @@ -129,15 +124,15 @@ func (t *Trashcan) String() string { } func (t *Trashcan) cleanoutArchive() error { - versionsDir := filepath.Join(t.folderPath, ".stversions") - if _, err := osutil.Lstat(versionsDir); os.IsNotExist(err) { + versionsDir := ".stversions" + if _, err := t.fs.Lstat(versionsDir); fs.IsNotExist(err) { return nil } cutoff := time.Now().Add(time.Duration(-24*t.cleanoutDays) * time.Hour) currentDir := "" filesInDir := 0 - walkFn := func(path string, info os.FileInfo, err error) error { + walkFn := func(path string, info fs.FileInfo, err error) error { if err != nil { return err } @@ -147,7 +142,7 @@ func (t *Trashcan) cleanoutArchive() error { // directory was empty and try to remove it. We ignore failure for // the time being. if currentDir != "" && filesInDir == 0 { - os.Remove(currentDir) + t.fs.Remove(currentDir) } currentDir = path filesInDir = 0 @@ -156,7 +151,7 @@ func (t *Trashcan) cleanoutArchive() error { if info.ModTime().Before(cutoff) { // The file is too old; remove it. - os.Remove(path) + t.fs.Remove(path) } else { // Keep this file, and remember it so we don't unnecessarily try // to remove this directory. @@ -165,14 +160,14 @@ func (t *Trashcan) cleanoutArchive() error { return nil } - if err := filepath.Walk(versionsDir, walkFn); err != nil { + if err := t.fs.Walk(versionsDir, walkFn); err != nil { return err } // The last directory seen by the walkFn may not have been removed as it // should be. if currentDir != "" && filesInDir == 0 { - os.Remove(currentDir) + t.fs.Remove(currentDir) } return nil } diff --git a/lib/versioner/trashcan_test.go b/lib/versioner/trashcan_test.go index 251cff65a..1be3c8512 100644 --- a/lib/versioner/trashcan_test.go +++ b/lib/versioner/trashcan_test.go @@ -12,6 +12,8 @@ import ( "path/filepath" "testing" "time" + + "github.com/syncthing/syncthing/lib/fs" ) func TestTrashcanCleanout(t *testing.T) { @@ -49,7 +51,7 @@ func TestTrashcanCleanout(t *testing.T) { } } - versioner := NewTrashcan("default", "testdata", map[string]string{"cleanoutDays": "7"}).(*Trashcan) + versioner := NewTrashcan("default", fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), map[string]string{"cleanoutDays": "7"}).(*Trashcan) if err := versioner.cleanoutArchive(); err != nil { t.Fatal(err) } diff --git a/lib/versioner/versioner.go b/lib/versioner/versioner.go index 0fdda359e..a9334fe4c 100644 --- a/lib/versioner/versioner.go +++ b/lib/versioner/versioner.go @@ -8,11 +8,13 @@ // simple default versioning scheme. package versioner +import "github.com/syncthing/syncthing/lib/fs" + type Versioner interface { Archive(filePath string) error } -var Factories = map[string]func(folderID string, folderDir string, params map[string]string) Versioner{} +var Factories = map[string]func(folderID string, filesystem fs.Filesystem, params map[string]string) Versioner{} const ( TimeFormat = "20060102-150405" diff --git a/lib/weakhash/weakhash.go b/lib/weakhash/weakhash.go index 37f0bab65..fabacfeee 100644 --- a/lib/weakhash/weakhash.go +++ b/lib/weakhash/weakhash.go @@ -9,7 +9,6 @@ package weakhash import ( "bufio" "io" - "os" "github.com/chmduquesne/rollinghash/adler32" ) @@ -72,27 +71,21 @@ func Find(ir io.Reader, hashesToFind []uint32, size int) (map[uint32][]int64, er return offsets, nil } -func NewFinder(path string, size int, hashesToFind []uint32) (*Finder, error) { - file, err := os.Open(path) +func NewFinder(ir io.ReadSeeker, size int, hashesToFind []uint32) (*Finder, error) { + offsets, err := Find(ir, hashesToFind, size) if err != nil { return nil, err } - offsets, err := Find(file, hashesToFind, size) - if err != nil { - file.Close() - return nil, err - } - return &Finder{ - file: file, + reader: ir, size: size, offsets: offsets, }, nil } type Finder struct { - file *os.File + reader io.ReadSeeker size int offsets map[uint32][]int64 } @@ -106,7 +99,11 @@ func (h *Finder) Iterate(hash uint32, buf []byte, iterFunc func(int64) bool) (bo } for _, offset := range h.offsets[hash] { - _, err := h.file.ReadAt(buf, offset) + _, err := h.reader.Seek(offset, io.SeekStart) + if err != nil { + return false, err + } + _, err = h.reader.Read(buf) if err != nil { return false, err } @@ -116,10 +113,3 @@ func (h *Finder) Iterate(hash uint32, buf []byte, iterFunc func(int64) bool) (bo } return false, nil } - -// Close releases any resource associated with the finder -func (h *Finder) Close() { - if h != nil { - h.file.Close() - } -} diff --git a/lib/weakhash/weakhash_test.go b/lib/weakhash/weakhash_test.go index fa9e9e384..0f3927595 100644 --- a/lib/weakhash/weakhash_test.go +++ b/lib/weakhash/weakhash_test.go @@ -11,6 +11,7 @@ package weakhash import ( "bytes" + "io" "io/ioutil" "os" "reflect" @@ -30,13 +31,15 @@ func TestFinder(t *testing.T) { if _, err := f.Write(payload); err != nil { t.Error(err) } + if _, err := f.Seek(0, io.SeekStart); err != nil { + t.Error(err) + } hashes := []uint32{65143183, 65798547} - finder, err := NewFinder(f.Name(), 4, hashes) + finder, err := NewFinder(f, 4, hashes) if err != nil { t.Error(err) } - defer finder.Close() expected := map[uint32][]int64{ 65143183: {1, 27, 53, 79}, diff --git a/vendor/github.com/kballard/go-shellquote/LICENSE b/vendor/github.com/kballard/go-shellquote/LICENSE new file mode 100644 index 000000000..a6d77312e --- /dev/null +++ b/vendor/github.com/kballard/go-shellquote/LICENSE @@ -0,0 +1,19 @@ +Copyright (C) 2014 Kevin Ballard + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/kballard/go-shellquote/doc.go b/vendor/github.com/kballard/go-shellquote/doc.go new file mode 100644 index 000000000..9445fa4ad --- /dev/null +++ b/vendor/github.com/kballard/go-shellquote/doc.go @@ -0,0 +1,3 @@ +// Shellquote provides utilities for joining/splitting strings using sh's +// word-splitting rules. +package shellquote diff --git a/vendor/github.com/kballard/go-shellquote/quote.go b/vendor/github.com/kballard/go-shellquote/quote.go new file mode 100644 index 000000000..72a8cb38b --- /dev/null +++ b/vendor/github.com/kballard/go-shellquote/quote.go @@ -0,0 +1,102 @@ +package shellquote + +import ( + "bytes" + "strings" + "unicode/utf8" +) + +// Join quotes each argument and joins them with a space. +// If passed to /bin/sh, the resulting string will be split back into the +// original arguments. +func Join(args ...string) string { + var buf bytes.Buffer + for i, arg := range args { + if i != 0 { + buf.WriteByte(' ') + } + quote(arg, &buf) + } + return buf.String() +} + +const ( + specialChars = "\\'\"`${[|&;<>()*?!" + extraSpecialChars = " \t\n" + prefixChars = "~" +) + +func quote(word string, buf *bytes.Buffer) { + // We want to try to produce a "nice" output. As such, we will + // backslash-escape most characters, but if we encounter a space, or if we + // encounter an extra-special char (which doesn't work with + // backslash-escaping) we switch over to quoting the whole word. We do this + // with a space because it's typically easier for people to read multi-word + // arguments when quoted with a space rather than with ugly backslashes + // everywhere. + origLen := buf.Len() + + if len(word) == 0 { + // oops, no content + buf.WriteString("''") + return + } + + cur, prev := word, word + atStart := true + for len(cur) > 0 { + c, l := utf8.DecodeRuneInString(cur) + cur = cur[l:] + if strings.ContainsRune(specialChars, c) || (atStart && strings.ContainsRune(prefixChars, c)) { + // copy the non-special chars up to this point + if len(cur) < len(prev) { + buf.WriteString(prev[0 : len(prev)-len(cur)-l]) + } + buf.WriteByte('\\') + buf.WriteRune(c) + prev = cur + } else if strings.ContainsRune(extraSpecialChars, c) { + // start over in quote mode + buf.Truncate(origLen) + goto quote + } + atStart = false + } + if len(prev) > 0 { + buf.WriteString(prev) + } + return + +quote: + // quote mode + // Use single-quotes, but if we find a single-quote in the word, we need + // to terminate the string, emit an escaped quote, and start the string up + // again + inQuote := false + for len(word) > 0 { + i := strings.IndexRune(word, '\'') + if i == -1 { + break + } + if i > 0 { + if !inQuote { + buf.WriteByte('\'') + inQuote = true + } + buf.WriteString(word[0:i]) + } + word = word[i+1:] + if inQuote { + buf.WriteByte('\'') + inQuote = false + } + buf.WriteString("\\'") + } + if len(word) > 0 { + if !inQuote { + buf.WriteByte('\'') + } + buf.WriteString(word) + buf.WriteByte('\'') + } +} diff --git a/vendor/github.com/kballard/go-shellquote/unquote.go b/vendor/github.com/kballard/go-shellquote/unquote.go new file mode 100644 index 000000000..ba3a0f227 --- /dev/null +++ b/vendor/github.com/kballard/go-shellquote/unquote.go @@ -0,0 +1,144 @@ +package shellquote + +import ( + "bytes" + "errors" + "strings" + "unicode/utf8" +) + +var ( + UnterminatedSingleQuoteError = errors.New("Unterminated single-quoted string") + UnterminatedDoubleQuoteError = errors.New("Unterminated double-quoted string") + UnterminatedEscapeError = errors.New("Unterminated backslash-escape") +) + +var ( + splitChars = " \n\t" + singleChar = '\'' + doubleChar = '"' + escapeChar = '\\' + doubleEscapeChars = "$`\"\n\\" +) + +// Split splits a string according to /bin/sh's word-splitting rules. It +// supports backslash-escapes, single-quotes, and double-quotes. Notably it does +// not support the $'' style of quoting. It also doesn't attempt to perform any +// other sort of expansion, including brace expansion, shell expansion, or +// pathname expansion. +// +// If the given input has an unterminated quoted string or ends in a +// backslash-escape, one of UnterminatedSingleQuoteError, +// UnterminatedDoubleQuoteError, or UnterminatedEscapeError is returned. +func Split(input string) (words []string, err error) { + var buf bytes.Buffer + words = make([]string, 0) + + for len(input) > 0 { + // skip any splitChars at the start + c, l := utf8.DecodeRuneInString(input) + if strings.ContainsRune(splitChars, c) { + input = input[l:] + continue + } + + var word string + word, input, err = splitWord(input, &buf) + if err != nil { + return + } + words = append(words, word) + } + return +} + +func splitWord(input string, buf *bytes.Buffer) (word string, remainder string, err error) { + buf.Reset() + +raw: + { + cur := input + for len(cur) > 0 { + c, l := utf8.DecodeRuneInString(cur) + cur = cur[l:] + if c == singleChar { + buf.WriteString(input[0 : len(input)-len(cur)-l]) + input = cur + goto single + } else if c == doubleChar { + buf.WriteString(input[0 : len(input)-len(cur)-l]) + input = cur + goto double + } else if c == escapeChar { + buf.WriteString(input[0 : len(input)-len(cur)-l]) + input = cur + goto escape + } else if strings.ContainsRune(splitChars, c) { + buf.WriteString(input[0 : len(input)-len(cur)-l]) + return buf.String(), cur, nil + } + } + if len(input) > 0 { + buf.WriteString(input) + input = "" + } + goto done + } + +escape: + { + if len(input) == 0 { + return "", "", UnterminatedEscapeError + } + c, l := utf8.DecodeRuneInString(input) + if c == '\n' { + // a backslash-escaped newline is elided from the output entirely + } else { + buf.WriteString(input[:l]) + } + input = input[l:] + } + goto raw + +single: + { + i := strings.IndexRune(input, singleChar) + if i == -1 { + return "", "", UnterminatedSingleQuoteError + } + buf.WriteString(input[0:i]) + input = input[i+1:] + goto raw + } + +double: + { + cur := input + for len(cur) > 0 { + c, l := utf8.DecodeRuneInString(cur) + cur = cur[l:] + if c == doubleChar { + buf.WriteString(input[0 : len(input)-len(cur)-l]) + input = cur + goto raw + } else if c == escapeChar { + // bash only supports certain escapes in double-quoted strings + c2, l2 := utf8.DecodeRuneInString(cur) + cur = cur[l2:] + if strings.ContainsRune(doubleEscapeChars, c2) { + buf.WriteString(input[0 : len(input)-len(cur)-l-l2]) + if c2 == '\n' { + // newline is special, skip the backslash entirely + } else { + buf.WriteRune(c2) + } + input = cur + } + } + } + return "", "", UnterminatedDoubleQuoteError + } + +done: + return buf.String(), input, nil +} diff --git a/vendor/manifest b/vendor/manifest index 0ab9bbd46..d9254b17e 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -249,6 +249,14 @@ "branch": "master", "notests": true }, + { + "importpath": "github.com/kballard/go-shellquote", + "repository": "https://github.com/kballard/go-shellquote", + "vcs": "git", + "revision": "cd60e84ee657ff3dc51de0b4f55dd299a3e136f2", + "branch": "master", + "notests": true + }, { "importpath": "github.com/klauspost/cpuid", "repository": "https://github.com/klauspost/cpuid",