octoleo/syncthing
1
0
Fork 0
mirror of https://github.com/octoleo/syncthing.git synced 2024-09-17 12:19:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

all: Convert folders to use filesystem abstraction

Browse Source 
GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4228
This commit is contained in:
Audrius Butkevicius 2017-08-19 14:36:56 +00:00 committed by Jakob Borg
parent ab8c2fb5c7
commit 3d8b4a42b7
78 changed files with 2588 additions and 1665 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
cmd/stcli/cmd_folders.go
View File

@ -9,6 +9,7 @@ import (
"github.com/AudriusButkevicius/cli"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/fs"
)
func init() {
@ -102,8 +103,10 @@ func foldersList(c *cli.Context) {
if !first {
fmt.Fprintln(writer)
}
fs := folder.Filesystem()
fmt.Fprintln(writer, "ID:\t", folder.ID, "\t")
fmt.Fprintln(writer, "Path:\t", folder.RawPath, "\t(directory)")
fmt.Fprintln(writer, "Path:\t", fs.URI(), "\t(directory)")
fmt.Fprintln(writer, "Path type:\t", fs.Type(), "\t(directory-type)")
fmt.Fprintln(writer, "Folder type:\t", folder.Type, "\t(type)")
fmt.Fprintln(writer, "Ignore permissions:\t", folder.IgnorePerms, "\t(permissions)")
fmt.Fprintln(writer, "Rescan interval in seconds:\t", folder.RescanIntervalS, "\t(rescan)")
@ -124,8 +127,9 @@ func foldersAdd(c *cli.Context) {
abs, err := filepath.Abs(c.Args()[1])
die(err)
folder := config.FolderConfiguration{
ID: c.Args()[0],
RawPath: filepath.Clean(abs),
ID: c.Args()[0],
Path: filepath.Clean(abs),
FilesystemType: fs.FilesystemTypeBasic,
}
cfg.Folders = append(cfg.Folders, folder)
setConfig(c, cfg)
@ -185,7 +189,9 @@ func foldersGet(c *cli.Context) {
}
switch arg {
case "directory":
fmt.Println(folder.RawPath)
fmt.Println(folder.Filesystem().URI())
case "directory-type":
fmt.Println(folder.Filesystem().Type())
case "type":
fmt.Println(folder.Type)
case "permissions":
@ -197,7 +203,7 @@ func foldersGet(c *cli.Context) {
fmt.Println(folder.Versioning.Type)
}
default:
die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, type, permissions, versioning, versioning-<key>")
die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, directory-type, type, permissions, versioning, versioning-<key>")
}
return
}
@ -220,7 +226,11 @@ func foldersSet(c *cli.Context) {
}
switch arg {
case "directory":
cfg.Folders[i].RawPath = val
cfg.Folders[i].Path = val
case "directory-type":
var fsType fs.FilesystemType
fsType.UnmarshalText([]byte(val))
cfg.Folders[i].FilesystemType = fsType
case "type":
var t config.FolderType
if err := t.UnmarshalText([]byte(val)); err != nil {

6
cmd/stindex/util.go
View File

@ -12,7 +12,7 @@ import (
"path/filepath"
"runtime"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/fs"
)
func nulString(bs []byte) string {
@ -33,7 +33,7 @@ func defaultConfigDir() string {
return filepath.Join(os.Getenv("AppData"), "Syncthing")
case "darwin":
dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing")
dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing")
if err != nil {
log.Fatal(err)
}
@ -43,7 +43,7 @@ func defaultConfigDir() string {
if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
return filepath.Join(xdgCfg, "syncthing")
}
dir, err := osutil.ExpandTilde("~/.config/syncthing")
dir, err := fs.ExpandTilde("~/.config/syncthing")
if err != nil {
log.Fatal(err)
}

26
cmd/syncthing/gui.go
View File

@ -28,9 +28,9 @@ import (
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/discover"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/logger"
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/stats"
@ -856,7 +856,7 @@ func (s *apiService) getSystemStatus(w http.ResponseWriter, r *http.Request) {
var m runtime.MemStats
runtime.ReadMemStats(&m)
tilde, _ := osutil.ExpandTilde("~")
tilde, _ := fs.ExpandTilde("~")
res := make(map[string]interface{})
res["myID"] = myID.String()
res["goroutines"] = runtime.NumGoroutine()
@ -1259,23 +1259,35 @@ func (s *apiService) getPeerCompletion(w http.ResponseWriter, r *http.Request) {
func (s *apiService) getSystemBrowse(w http.ResponseWriter, r *http.Request) {
qs := r.URL.Query()
current := qs.Get("current")
// Default value or in case of error unmarshalling ends up being basic fs.
var fsType fs.FilesystemType
fsType.UnmarshalText([]byte(qs.Get("filesystem")))
if current == "" {
if roots, err := osutil.GetFilesystemRoots(); err == nil {
filesystem := fs.NewFilesystem(fsType, "")
if roots, err := filesystem.Roots(); err == nil {
sendJSON(w, roots)
} else {
http.Error(w, err.Error(), 500)
}
return
}
search, _ := osutil.ExpandTilde(current)
pathSeparator := string(os.PathSeparator)
search, _ := fs.ExpandTilde(current)
pathSeparator := string(fs.PathSeparator)
if strings.HasSuffix(current, pathSeparator) && !strings.HasSuffix(search, pathSeparator) {
search = search + pathSeparator
}
subdirectories, _ := osutil.Glob(search + "*")
searchDir := filepath.Dir(search)
searchFile := filepath.Base(search)
fs := fs.NewFilesystem(fsType, searchDir)
subdirectories, _ := fs.Glob(searchFile + "*")
ret := make([]string, 0, len(subdirectories))
for _, subdirectory := range subdirectories {
info, err := os.Stat(subdirectory)
info, err := fs.Stat(subdirectory)
if err == nil && info.IsDir() {
ret = append(ret, subdirectory+pathSeparator)
}

10
cmd/syncthing/locations.go
View File

@ -13,7 +13,7 @@ import (
"strings"
"time"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/fs"
)
type locationEnum string
@ -65,7 +65,7 @@ func expandLocations() error {
dir = strings.Replace(dir, "${"+varName+"}", value, -1)
}
var err error
dir, err = osutil.ExpandTilde(dir)
dir, err = fs.ExpandTilde(dir)
if err != nil {
return err
}
@ -86,7 +86,7 @@ func defaultConfigDir() string {
return filepath.Join(os.Getenv("AppData"), "Syncthing")
case "darwin":
dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing")
dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing")
if err != nil {
l.Fatalln(err)
}
@ -96,7 +96,7 @@ func defaultConfigDir() string {
if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
return filepath.Join(xdgCfg, "syncthing")
}
dir, err := osutil.ExpandTilde("~/.config/syncthing")
dir, err := fs.ExpandTilde("~/.config/syncthing")
if err != nil {
l.Fatalln(err)
}
@ -106,7 +106,7 @@ func defaultConfigDir() string {
// homeDir returns the user's home directory, or dies trying.
func homeDir() string {
home, err := osutil.ExpandTilde("~")
home, err := fs.ExpandTilde("~")
if err != nil {
l.Fatalln(err)
}

22
cmd/syncthing/main.go
View File

@ -37,6 +37,7 @@ import (
"github.com/syncthing/syncthing/lib/dialer"
"github.com/syncthing/syncthing/lib/discover"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/logger"
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/osutil"
@ -444,7 +445,7 @@ func openGUI() {
}
func generate(generateDir string) {
dir, err := osutil.ExpandTilde(generateDir)
dir, err := fs.ExpandTilde(generateDir)
if err != nil {
l.Fatalln("generate:", err)
}
@ -1085,7 +1086,7 @@ func defaultConfig(myName string) config.Configuration {
if !noDefaultFolder {
l.Infoln("Default folder created and/or linked to new config")
defaultFolder = config.NewFolderConfiguration("default", locations[locDefFolder])
defaultFolder = config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, locations[locDefFolder])
defaultFolder.Label = "Default Folder"
defaultFolder.RescanIntervalS = 60
defaultFolder.MinDiskFree = config.Size{Value: 1, Unit: "%"}
@ -1141,19 +1142,20 @@ func shutdown() {
stop <- exitSuccess
}
func ensureDir(dir string, mode os.FileMode) {
err := osutil.MkdirAll(dir, mode)
func ensureDir(dir string, mode fs.FileMode) {
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
err := fs.MkdirAll(".", mode)
if err != nil {
l.Fatalln(err)
}
if fi, err := os.Stat(dir); err == nil {
if fi, err := fs.Stat("."); err == nil {
// Apprently the stat may fail even though the mkdirall passed. If it
// does, we'll just assume things are in order and let other things
// fail (like loading or creating the config...).
currentMode := fi.Mode() & 0777
if currentMode != mode {
err := os.Chmod(dir, mode)
err := fs.Chmod(".", mode)
// This can fail on crappy filesystems, nothing we can do about it.
if err != nil {
l.Warnln(err)
@ -1276,22 +1278,22 @@ func cleanConfigDirectory() {
}
for pat, dur := range patterns {
pat = filepath.Join(baseDirs["config"], pat)
files, err := osutil.Glob(pat)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, baseDirs["config"])
files, err := fs.Glob(pat)
if err != nil {
l.Infoln("Cleaning:", err)
continue
}
for _, file := range files {
info, err := osutil.Lstat(file)
info, err := fs.Lstat(file)
if err != nil {
l.Infoln("Cleaning:", err)
continue
}
if time.Since(info.ModTime()) > dur {
if err = os.RemoveAll(file); err != nil {
if err = fs.RemoveAll(file); err != nil {
l.Infoln("Cleaning:", err)
} else {
l.Infoln("Cleaned away old file", filepath.Base(file))

2
gui/default/syncthing/core/notifications.html
View File

@ -40,4 +40,4 @@
<div class="clearfix"></div>
</div>
</div>
</notification>
</notification>

2
gui/default/syncthing/folder/editFolderModalView.html
View File

@ -180,7 +180,7 @@
<label translate for="externalCommand">Command</label>
<input name="externalCommand" id="externalCommand" class="form-control" type="text" ng-model="currentFolder.externalCommand" required="" aria-required="true" />
<p class="help-block">
<span translate ng-if="folderEditor.externalCommand.$valid || folderEditor.externalCommand.$pristine">The first command line parameter is the folder path and the second parameter is the relative path in the folder.</span>
<span translate ng-if="folderEditor.externalCommand.$valid || folderEditor.externalCommand.$pristine">See external versioner help for supported templated command line parameters.</span>
<span translate ng-if="folderEditor.externalCommand.$error.required && folderEditor.externalCommand.$dirty">The path cannot be blank.</span>
</p>
</div>

43
lib/config/config.go
View File

@ -23,6 +23,7 @@ import (
"strconv"
"strings"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/upgrade"
@ -31,7 +32,7 @@ import (
const (
OldestHandledVersion = 10
CurrentVersion = 21
CurrentVersion = 22
MaxRescanIntervalS = 365 * 24 * 60 * 60
)
@ -319,6 +320,9 @@ func (cfg *Configuration) clean() error {
if cfg.Version == 20 {
convertV20V21(cfg)
}
if cfg.Version == 21 {
convertV21V22(cfg)
}
// Build a list of available devices
existingDevices := make(map[protocol.DeviceID]bool)
@ -368,23 +372,38 @@ func (cfg *Configuration) clean() error {
return nil
}
func convertV21V22(cfg *Configuration) {
for i := range cfg.Folders {
cfg.Folders[i].FilesystemType = fs.FilesystemTypeBasic
// Migrate to templated external versioner commands
if cfg.Folders[i].Versioning.Type == "external" {
cfg.Folders[i].Versioning.Params["command"] += " %FOLDER_PATH% %FILE_PATH%"
}
}
cfg.Version = 22
}
func convertV20V21(cfg *Configuration) {
for _, folder := range cfg.Folders {
if folder.FilesystemType != fs.FilesystemTypeBasic {
continue
}
switch folder.Versioning.Type {
case "simple", "trashcan":
// Clean out symlinks in the known place
cleanSymlinks(filepath.Join(folder.Path(), ".stversions"))
cleanSymlinks(folder.Filesystem(), ".stversions")
case "staggered":
versionDir := folder.Versioning.Params["versionsPath"]
if versionDir == "" {
// default place
cleanSymlinks(filepath.Join(folder.Path(), ".stversions"))
cleanSymlinks(folder.Filesystem(), ".stversions")
} else if filepath.IsAbs(versionDir) {
// absolute
cleanSymlinks(versionDir)
cleanSymlinks(fs.NewFilesystem(fs.FilesystemTypeBasic, versionDir), ".")
} else {
// relative to folder
cleanSymlinks(filepath.Join(folder.Path(), versionDir))
cleanSymlinks(folder.Filesystem(), versionDir)
}
}
}
@ -428,9 +447,7 @@ func convertV17V18(cfg *Configuration) {
}
func convertV16V17(cfg *Configuration) {
for i := range cfg.Folders {
cfg.Folders[i].Fsync = true
}
// Fsync = true removed
cfg.Version = 17
}
@ -670,21 +687,21 @@ loop:
return devices[0:count]
}
func cleanSymlinks(dir string) {
func cleanSymlinks(filesystem fs.Filesystem, dir string) {
if runtime.GOOS == "windows" {
// We don't do symlinks on Windows. Additionally, there may
// be things that look like symlinks that are not, which we
// should leave alone. Deduplicated files, for example.
return
}
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
filesystem.Walk(dir, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 {
if info.IsSymlink() {
l.Infoln("Removing incorrectly versioned symlink", path)
os.Remove(path)
return filepath.SkipDir
filesystem.Remove(path)
return fs.SkipDir
}
return nil
})

63
lib/config/config_test.go
View File

@ -19,6 +19,7 @@ import (
"testing"
"github.com/d4l3k/messagediff"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
@ -103,7 +104,8 @@ func TestDeviceConfig(t *testing.T) {
expectedFolders := []FolderConfiguration{
{
ID: "test",
RawPath: "testdata",
FilesystemType: fs.FilesystemTypeBasic,
Path: "testdata",
Devices: []FolderDeviceConfiguration{{DeviceID: device1}, {DeviceID: device4}},
Type: FolderTypeSendOnly,
RescanIntervalS: 600,
@ -113,7 +115,6 @@ func TestDeviceConfig(t *testing.T) {
AutoNormalize: true,
MinDiskFree: Size{1, "%"},
MaxConflicts: -1,
Fsync: true,
Versioning: VersioningConfiguration{
Params: map[string]string{},
},
@ -121,15 +122,11 @@ func TestDeviceConfig(t *testing.T) {
},
}
// The cachedPath will have been resolved to an absolute path,
// The cachedFilesystem will have been resolved to an absolute path,
// depending on where the tests are running. Zero it out so we don't
// fail based on that.
for i := range cfg.Folders {
cfg.Folders[i].cachedPath = ""
}
if runtime.GOOS != "windows" {
expectedFolders[0].RawPath += string(filepath.Separator)
cfg.Folders[i].cachedFilesystem = nil
}
expectedDevices := []DeviceConfiguration{
@ -377,16 +374,17 @@ func TestVersioningConfig(t *testing.T) {
}
func TestIssue1262(t *testing.T) {
if runtime.GOOS != "windows" {
t.Skipf("path gets converted to absolute as part of the filesystem initialization on linux")
}
cfg, err := Load("testdata/issue-1262.xml", device4)
if err != nil {
t.Fatal(err)
}
actual := cfg.Folders()["test"].RawPath
expected := "e:/"
if runtime.GOOS == "windows" {
expected = `e:\`
}
actual := cfg.Folders()["test"].Filesystem().URI()
expected := `e:\`
if actual != expected {
t.Errorf("%q != %q", actual, expected)
@ -416,43 +414,12 @@ func TestIssue1750(t *testing.T) {
}
}
func TestWindowsPaths(t *testing.T) {
if runtime.GOOS != "windows" {
t.Skip("Not useful on non-Windows")
return
}
folder := FolderConfiguration{
RawPath: `e:\`,
}
expected := `\\?\e:\`
actual := folder.Path()
if actual != expected {
t.Errorf("%q != %q", actual, expected)
}
folder.RawPath = `\\192.0.2.22\network\share`
expected = folder.RawPath
actual = folder.Path()
if actual != expected {
t.Errorf("%q != %q", actual, expected)
}
folder.RawPath = `relative\path`
expected = folder.RawPath
actual = folder.Path()
if actual == expected || !strings.HasPrefix(actual, "\\\\?\\") {
t.Errorf("%q == %q, expected absolutification", actual, expected)
}
}
func TestFolderPath(t *testing.T) {
folder := FolderConfiguration{
RawPath: "~/tmp",
Path: "~/tmp",
}
realPath := folder.Path()
realPath := folder.Filesystem().URI()
if !filepath.IsAbs(realPath) {
t.Error(realPath, "should be absolute")
}
@ -677,8 +644,8 @@ func TestEmptyFolderPaths(t *testing.T) {
t.Fatal(err)
}
folder := wrapper.Folders()["f1"]
if folder.Path() != "" {
t.Errorf("Expected %q to be empty", folder.Path())
if folder.cachedFilesystem != nil {
t.Errorf("Expected %q to be empty", folder.cachedFilesystem)
}
}

113
lib/config/folderconfiguration.go
View File

@ -8,19 +8,17 @@ package config
import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
type FolderConfiguration struct {
ID string `xml:"id,attr" json:"id"`
Label string `xml:"label,attr" json:"label"`
RawPath string `xml:"path,attr" json:"path"`
FilesystemType fs.FilesystemType `xml:"filesystemType" json:"filesystemType"`
Path string `xml:"path,attr" json:"path"`
Type FolderType `xml:"type,attr" json:"type"`
Devices []FolderDeviceConfiguration `xml:"device" json:"devices"`
RescanIntervalS int `xml:"rescanIntervalS,attr" json:"rescanIntervalS"`
@ -39,11 +37,10 @@ type FolderConfiguration struct {
MaxConflicts int `xml:"maxConflicts" json:"maxConflicts"`
DisableSparseFiles bool `xml:"disableSparseFiles" json:"disableSparseFiles"`
DisableTempIndexes bool `xml:"disableTempIndexes" json:"disableTempIndexes"`
Fsync bool `xml:"fsync" json:"fsync"`
Paused bool `xml:"paused" json:"paused"`
WeakHashThresholdPct int `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash.
cachedPath string
cachedFilesystem fs.Filesystem
DeprecatedReadOnly bool `xml:"ro,attr,omitempty" json:"-"`
DeprecatedMinDiskFreePct float64 `xml:"minDiskFreePct,omitempty" json:"-"`
@ -54,10 +51,11 @@ type FolderDeviceConfiguration struct {
IntroducedBy protocol.DeviceID `xml:"introducedBy,attr" json:"introducedBy"`
}
func NewFolderConfiguration(id, path string) FolderConfiguration {
func NewFolderConfiguration(id string, fsType fs.FilesystemType, path string) FolderConfiguration {
f := FolderConfiguration{
ID: id,
RawPath: path,
ID: id,
FilesystemType: fsType,
Path: path,
}
f.prepare()
return f
@ -71,53 +69,57 @@ func (f FolderConfiguration) Copy() FolderConfiguration {
return c
}
func (f FolderConfiguration) Path() string {
func (f FolderConfiguration) Filesystem() fs.Filesystem {
// This is intentionally not a pointer method, because things like
// cfg.Folders["default"].Path() should be valid.
if f.cachedPath == "" && f.RawPath != "" {
l.Infoln("bug: uncached path call (should only happen in tests)")
return f.cleanedPath()
// cfg.Folders["default"].Filesystem() should be valid.
if f.cachedFilesystem == nil && f.Path != "" {
l.Infoln("bug: uncached filesystem call (should only happen in tests)")
return fs.NewFilesystem(f.FilesystemType, f.Path)
}
return f.cachedPath
return f.cachedFilesystem
}
func (f *FolderConfiguration) CreateMarker() error {
if !f.HasMarker() {
marker := filepath.Join(f.Path(), ".stfolder")
fd, err := os.Create(marker)
fs := f.Filesystem()
fd, err := fs.Create(".stfolder")
if err != nil {
return err
}
fd.Close()
if err := osutil.SyncDir(filepath.Dir(marker)); err != nil {
l.Infof("fsync %q failed: %v", filepath.Dir(marker), err)
if dir, err := fs.Open("."); err == nil {
if serr := dir.Sync(); err != nil {
l.Infof("fsync %q failed: %v", ".", serr)
}
} else {
l.Infof("fsync %q failed: %v", ".", err)
}
osutil.HideFile(marker)
fs.Hide(".stfolder")
}
return nil
}
func (f *FolderConfiguration) HasMarker() bool {
_, err := os.Stat(filepath.Join(f.Path(), ".stfolder"))
_, err := f.Filesystem().Stat(".stfolder")
return err == nil
}
func (f *FolderConfiguration) CreateRoot() (err error) {
// Directory permission bits. Will be filtered down to something
// sane by umask on Unixes.
permBits := os.FileMode(0777)
permBits := fs.FileMode(0777)
if runtime.GOOS == "windows" {
// Windows has no umask so we must chose a safer set of bits to
// begin with.
permBits = 0700
}
if _, err = os.Stat(f.Path()); os.IsNotExist(err) {
if err = osutil.MkdirAll(f.Path(), permBits); err != nil {
l.Warnf("Creating directory for %v: %v",
f.Description(), err)
filesystem := f.Filesystem()
if _, err = filesystem.Stat("."); fs.IsNotExist(err) {
if err = filesystem.MkdirAll(".", permBits); err != nil {
l.Warnf("Creating directory for %v: %v", f.Description(), err)
}
}
@ -140,24 +142,10 @@ func (f *FolderConfiguration) DeviceIDs() []protocol.DeviceID {
}
func (f *FolderConfiguration) prepare() {
if f.RawPath != "" {
// The reason it's done like this:
// C: -> C:\ -> C:\ (issue that this is trying to fix)
// C:\somedir -> C:\somedir\ -> C:\somedir
// C:\somedir\ -> C:\somedir\\ -> C:\somedir
// This way in the tests, we get away without OS specific separators
// in the test configs.
f.RawPath = filepath.Dir(f.RawPath + string(filepath.Separator))
// If we're not on Windows, we want the path to end with a slash to
// penetrate symlinks. On Windows, paths must not end with a slash.
if runtime.GOOS != "windows" && f.RawPath[len(f.RawPath)-1] != filepath.Separator {
f.RawPath = f.RawPath + string(filepath.Separator)
}
if f.Path != "" {
f.cachedFilesystem = fs.NewFilesystem(f.FilesystemType, f.Path)
}
f.cachedPath = f.cleanedPath()
if f.RescanIntervalS > MaxRescanIntervalS {
f.RescanIntervalS = MaxRescanIntervalS
} else if f.RescanIntervalS < 0 {
@ -173,43 +161,6 @@ func (f *FolderConfiguration) prepare() {
}
}
func (f *FolderConfiguration) cleanedPath() string {
if f.RawPath == "" {
return ""
}
cleaned := f.RawPath
// Attempt tilde expansion; leave unchanged in case of error
if path, err := osutil.ExpandTilde(cleaned); err == nil {
cleaned = path
}
// Attempt absolutification; leave unchanged in case of error
if !filepath.IsAbs(cleaned) {
// Abs() looks like a fairly expensive syscall on Windows, while
// IsAbs() is a whole bunch of string mangling. I think IsAbs() may be
// somewhat faster in the general case, hence the outer if...
if path, err := filepath.Abs(cleaned); err == nil {
cleaned = path
}
}
// Attempt to enable long filename support on Windows. We may still not
// have an absolute path here if the previous steps failed.
if runtime.GOOS == "windows" && filepath.IsAbs(cleaned) && !strings.HasPrefix(f.RawPath, `\\`) {
return `\\?\` + cleaned
}
// If we're not on Windows, we want the path to end with a slash to
// penetrate symlinks. On Windows, paths must not end with a slash.
if runtime.GOOS != "windows" && cleaned[len(cleaned)-1] != filepath.Separator {
cleaned = cleaned + string(filepath.Separator)
}
return cleaned
}
type FolderDeviceConfigurationList []FolderDeviceConfiguration
func (l FolderDeviceConfigurationList) Less(a, b int) bool {

16
lib/config/testdata/v22.xml vendored Normal file
View File

@ -0,0 +1,16 @@
<configuration version="22">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

4
lib/config/testdata/versioningconfig.xml vendored
View File

@ -1,5 +1,5 @@
<configuration version="10">
<folder id="test" directory="testdata/" ro="true">
<configuration version="22">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<versioning type="simple">
<param key="foo" val="bar"/>
<param key="baz" val="quux"/>

6
lib/db/set.go
View File

@ -25,6 +25,7 @@ import (
type FileSet struct {
sequence int64 // Our local sequence number
folder string
fs fs.Filesystem
db *Instance
blockmap *BlockMap
localSize sizeTracker
@ -113,10 +114,11 @@ func (s *sizeTracker) Size() Counts {
return s.Counts
}
func NewFileSet(folder string, db *Instance) *FileSet {
func NewFileSet(folder string, fs fs.Filesystem, db *Instance) *FileSet {
var s = FileSet{
remoteSequence: make(map[protocol.DeviceID]int64),
folder: folder,
fs: fs,
db: db,
blockmap: NewBlockMap(db, db.folderIdx.ID([]byte(folder))),
updateMutex: sync.NewMutex(),
@ -303,7 +305,7 @@ func (s *FileSet) SetIndexID(device protocol.DeviceID, id protocol.IndexID) {
func (s *FileSet) MtimeFS() *fs.MtimeFS {
prefix := s.db.mtimesKey([]byte(s.folder))
kv := NewNamespacedKV(s.db, string(prefix))
return fs.NewMtimeFS(fs.DefaultFilesystem, kv)
return fs.NewMtimeFS(s.fs, kv)
}
func (s *FileSet) ListDevices() []protocol.DeviceID {

29
lib/db/set_test.go
View File

@ -15,6 +15,7 @@ import (
"github.com/d4l3k/messagediff"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
@ -97,7 +98,7 @@ func (l fileList) String() string {
func TestGlobalSet(t *testing.T) {
ldb := db.OpenMemory()
m := db.NewFileSet("test", ldb)
m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local0 := fileList{
protocol.FileInfo{Name: "a", Sequence: 1, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
@ -312,7 +313,7 @@ func TestGlobalSet(t *testing.T) {
func TestNeedWithInvalid(t *testing.T) {
ldb := db.OpenMemory()
s := db.NewFileSet("test", ldb)
s := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
localHave := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
@ -349,7 +350,7 @@ func TestNeedWithInvalid(t *testing.T) {
func TestUpdateToInvalid(t *testing.T) {
ldb := db.OpenMemory()
s := db.NewFileSet("test", ldb)
s := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
localHave := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
@ -381,7 +382,7 @@ func TestUpdateToInvalid(t *testing.T) {
func TestInvalidAvailability(t *testing.T) {
ldb := db.OpenMemory()
s := db.NewFileSet("test", ldb)
s := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
remote0Have := fileList{
protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
@ -419,7 +420,7 @@ func TestInvalidAvailability(t *testing.T) {
func TestGlobalReset(t *testing.T) {
ldb := db.OpenMemory()
m := db.NewFileSet("test", ldb)
m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@ -457,7 +458,7 @@ func TestGlobalReset(t *testing.T) {
func TestNeed(t *testing.T) {
ldb := db.OpenMemory()
m := db.NewFileSet("test", ldb)
m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local := []protocol.FileInfo{
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@ -495,7 +496,7 @@ func TestNeed(t *testing.T) {
func TestSequence(t *testing.T) {
ldb := db.OpenMemory()
m := db.NewFileSet("test", ldb)
m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local1 := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@ -525,7 +526,7 @@ func TestSequence(t *testing.T) {
func TestListDropFolder(t *testing.T) {
ldb := db.OpenMemory()
s0 := db.NewFileSet("test0", ldb)
s0 := db.NewFileSet("test0", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local1 := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@ -533,7 +534,7 @@ func TestListDropFolder(t *testing.T) {
}
s0.Replace(protocol.LocalDeviceID, local1)
s1 := db.NewFileSet("test1", ldb)
s1 := db.NewFileSet("test1", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local2 := []protocol.FileInfo{
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
@ -575,7 +576,7 @@ func TestListDropFolder(t *testing.T) {
func TestGlobalNeedWithInvalid(t *testing.T) {
ldb := db.OpenMemory()
s := db.NewFileSet("test1", ldb)
s := db.NewFileSet("test1", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
rem0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
@ -612,7 +613,7 @@ func TestGlobalNeedWithInvalid(t *testing.T) {
func TestLongPath(t *testing.T) {
ldb := db.OpenMemory()
s := db.NewFileSet("test", ldb)
s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
var b bytes.Buffer
for i := 0; i < 100; i++ {
@ -642,7 +643,7 @@ func TestCommitted(t *testing.T) {
ldb := db.OpenMemory()
s := db.NewFileSet("test", ldb)
s := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local := []protocol.FileInfo{
{Name: string("file"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
@ -688,7 +689,7 @@ func BenchmarkUpdateOneFile(b *testing.B) {
os.RemoveAll("testdata/benchmarkupdate.db")
}()
m := db.NewFileSet("test", ldb)
m := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
m.Replace(protocol.LocalDeviceID, local0)
l := local0[4:5]
@ -703,7 +704,7 @@ func BenchmarkUpdateOneFile(b *testing.B) {
func TestIndexID(t *testing.T) {
ldb := db.OpenMemory()
s := db.NewFileSet("test", ldb)
s := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
// The Index ID for some random device is zero by default.
id := s.IndexID(remoteDevice0)

237
lib/fs/basicfs.go
View File

@ -9,30 +9,156 @@ package fs
import (
"errors"
"os"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/calmh/du"
)
var (
ErrInvalidFilename = errors.New("filename is invalid")
ErrNotRelative = errors.New("not a relative path")
)
// The BasicFilesystem implements all aspects by delegating to package os.
// All paths are relative to the root and cannot (should not) escape the root directory.
type BasicFilesystem struct {
root string
}
func NewBasicFilesystem() *BasicFilesystem {
return new(BasicFilesystem)
func newBasicFilesystem(root string) *BasicFilesystem {
// The reason it's done like this:
// C: -> C:\ -> C:\ (issue that this is trying to fix)
// C:\somedir -> C:\somedir\ -> C:\somedir
// C:\somedir\ -> C:\somedir\\ -> C:\somedir
// This way in the tests, we get away without OS specific separators
// in the test configs.
root = filepath.Dir(root + string(filepath.Separator))
// Attempt tilde expansion; leave unchanged in case of error
if path, err := ExpandTilde(root); err == nil {
root = path
}
// Attempt absolutification; leave unchanged in case of error
if !filepath.IsAbs(root) {
// Abs() looks like a fairly expensive syscall on Windows, while
// IsAbs() is a whole bunch of string mangling. I think IsAbs() may be
// somewhat faster in the general case, hence the outer if...
if path, err := filepath.Abs(root); err == nil {
root = path
}
}
// Attempt to enable long filename support on Windows. We may still not
// have an absolute path here if the previous steps failed.
if runtime.GOOS == "windows" {
if filepath.IsAbs(root) && !strings.HasPrefix(root, `\\`) {
root = `\\?\` + root
}
// If we're not on Windows, we want the path to end with a slash to
// penetrate symlinks. On Windows, paths must not end with a slash.
} else if root[len(root)-1] != filepath.Separator {
root = root + string(filepath.Separator)
}
return &BasicFilesystem{
root: root,
}
}
// rooted expands the relative path to the full path that is then used with os
// package. If the relative path somehow causes the final path to escape the root
// directoy, this returns an error, to prevent accessing files that are not in the
// shared directory.
func (f *BasicFilesystem) rooted(rel string) (string, error) {
// The root must not be empty.
if f.root == "" {
return "", ErrInvalidFilename
}
pathSep := string(PathSeparator)
// The expected prefix for the resulting path is the root, with a path
// separator at the end.
expectedPrefix := filepath.FromSlash(f.root)
if !strings.HasSuffix(expectedPrefix, pathSep) {
expectedPrefix += pathSep
}
// The relative path should be clean from internal dotdots and similar
// funkyness.
rel = filepath.FromSlash(rel)
if filepath.Clean(rel) != rel {
return "", ErrInvalidFilename
}
// It is not acceptable to attempt to traverse upwards.
switch rel {
case "..", pathSep:
return "", ErrNotRelative
}
if strings.HasPrefix(rel, ".."+pathSep) {
return "", ErrNotRelative
}
if strings.HasPrefix(rel, pathSep+pathSep) {
// The relative path may pretend to be an absolute path within the
// root, but the double path separator on Windows implies something
// else. It would get cleaned by the Join below, but it's out of
// spec anyway.
return "", ErrNotRelative
}
// The supposedly correct path is the one filepath.Join will return, as
// it does cleaning and so on. Check that one first to make sure no
// obvious escape attempts have been made.
joined := filepath.Join(f.root, rel)
if rel == "." && !strings.HasSuffix(joined, pathSep) {
joined += pathSep
}
if !strings.HasPrefix(joined, expectedPrefix) {
return "", ErrNotRelative
}
return joined, nil
}
func (f *BasicFilesystem) unrooted(path string) string {
return strings.TrimPrefix(strings.TrimPrefix(path, f.root), string(PathSeparator))
}
func (f *BasicFilesystem) Chmod(name string, mode FileMode) error {
name, err := f.rooted(name)
if err != nil {
return err
}
return os.Chmod(name, os.FileMode(mode))
}
func (f *BasicFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error {
name, err := f.rooted(name)
if err != nil {
return err
}
return os.Chtimes(name, atime, mtime)
}
func (f *BasicFilesystem) Mkdir(name string, perm FileMode) error {
name, err := f.rooted(name)
if err != nil {
return err
}
return os.Mkdir(name, os.FileMode(perm))
}
func (f *BasicFilesystem) Lstat(name string) (FileInfo, error) {
name, err := f.rooted(name)
if err != nil {
return nil, err
}
fi, err := underlyingLstat(name)
if err != nil {
return nil, err
@ -41,14 +167,38 @@ func (f *BasicFilesystem) Lstat(name string) (FileInfo, error) {
}
func (f *BasicFilesystem) Remove(name string) error {
name, err := f.rooted(name)
if err != nil {
return err
}
return os.Remove(name)
}
func (f *BasicFilesystem) RemoveAll(name string) error {
name, err := f.rooted(name)
if err != nil {
return err
}
return os.RemoveAll(name)
}
func (f *BasicFilesystem) Rename(oldpath, newpath string) error {
oldpath, err := f.rooted(oldpath)
if err != nil {
return err
}
newpath, err = f.rooted(newpath)
if err != nil {
return err
}
return os.Rename(oldpath, newpath)
}
func (f *BasicFilesystem) Stat(name string) (FileInfo, error) {
name, err := f.rooted(name)
if err != nil {
return nil, err
}
fi, err := os.Stat(name)
if err != nil {
return nil, err
@ -57,7 +207,11 @@ func (f *BasicFilesystem) Stat(name string) (FileInfo, error) {
}
func (f *BasicFilesystem) DirNames(name string) ([]string, error) {
fd, err := os.OpenFile(name, os.O_RDONLY, 0777)
name, err := f.rooted(name)
if err != nil {
return nil, err
}
fd, err := os.OpenFile(name, OptReadOnly, 0777)
if err != nil {
return nil, err
}
@ -72,19 +226,39 @@ func (f *BasicFilesystem) DirNames(name string) ([]string, error) {
}
func (f *BasicFilesystem) Open(name string) (File, error) {
fd, err := os.Open(name)
rootedName, err := f.rooted(name)
if err != nil {
return nil, err
}
return fsFile{fd}, err
fd, err := os.Open(rootedName)
if err != nil {
return nil, err
}
return fsFile{fd, name}, err
}
func (f *BasicFilesystem) OpenFile(name string, flags int, mode FileMode) (File, error) {
rootedName, err := f.rooted(name)
if err != nil {
return nil, err
}
fd, err := os.OpenFile(rootedName, flags, os.FileMode(mode))
if err != nil {
return nil, err
}
return fsFile{fd, name}, err
}
func (f *BasicFilesystem) Create(name string) (File, error) {
fd, err := os.Create(name)
rootedName, err := f.rooted(name)
if err != nil {
return nil, err
}
return fsFile{fd}, err
fd, err := os.Create(rootedName)
if err != nil {
return nil, err
}
return fsFile{fd, name}, err
}
func (f *BasicFilesystem) Walk(root string, walkFn WalkFunc) error {
@ -92,9 +266,47 @@ func (f *BasicFilesystem) Walk(root string, walkFn WalkFunc) error {
return errors.New("not implemented")
}
func (f *BasicFilesystem) Glob(pattern string) ([]string, error) {
pattern, err := f.rooted(pattern)
if err != nil {
return nil, err
}
files, err := filepath.Glob(pattern)
unrooted := make([]string, len(files))
for i := range files {
unrooted[i] = f.unrooted(files[i])
}
return unrooted, err
}
func (f *BasicFilesystem) Usage(name string) (Usage, error) {
name, err := f.rooted(name)
if err != nil {
return Usage{}, err
}
u, err := du.Get(name)
return Usage{
Free: u.FreeBytes,
Total: u.TotalBytes,
}, err
}
func (f *BasicFilesystem) Type() FilesystemType {
return FilesystemTypeBasic
}
func (f *BasicFilesystem) URI() string {
return strings.TrimPrefix(f.root, `\\?\`)
}
// fsFile implements the fs.File interface on top of an os.File
type fsFile struct {
*os.File
name string
}
func (f fsFile) Name() string {
return f.name
}
func (f fsFile) Stat() (FileInfo, error) {
@ -105,6 +317,17 @@ func (f fsFile) Stat() (FileInfo, error) {
return fsFileInfo{info}, nil
}
func (f fsFile) Sync() error {
err := f.File.Sync()
// On Windows, fsyncing a directory returns a "handle is invalid"
// So we swallow that and let things go through in order not to have to add
// a separate way of syncing directories versus files.
if err != nil && (runtime.GOOS != "windows" || !strings.Contains(err.Error(), "handle is invalid")) {
return err
}
return nil
}
// fsFileInfo implements the fs.FileInfo interface on top of an os.FileInfo.
type fsFileInfo struct {
os.FileInfo

29
lib/fs/basicfs_symlink_unix.go
View File

@ -1,29 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build !windows
package fs
import "os"
var symlinksSupported = true
func DisableSymlinks() {
symlinksSupported = false
}
func (BasicFilesystem) SymlinksSupported() bool {
return symlinksSupported
}
func (BasicFilesystem) CreateSymlink(name, target string) error {
return os.Symlink(target, name)
}
func (BasicFilesystem) ReadSymlink(path string) (string, error) {
return os.Readlink(path)
}

27
lib/fs/basicfs_symlink_windows.go
View File

@ -1,27 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build windows
package fs
import "errors"
var errNotSupported = errors.New("symlinks not supported")
func DisableSymlinks() {}
func (BasicFilesystem) SymlinksSupported() bool {
return false
}
func (BasicFilesystem) ReadSymlink(path string) (string, error) {
return "", errNotSupported
}
func (BasicFilesystem) CreateSymlink(path, target string) error {
return errNotSupported
}

486
lib/fs/basicfs_test.go Normal file
View File

@ -0,0 +1,486 @@
// Copyright (C) 2017 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package fs
import (
"io/ioutil"
"os"
"path/filepath"
"runtime"
"sort"
"strings"
"testing"
"time"
)
func setup(t *testing.T) (Filesystem, string) {
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
return newBasicFilesystem(dir), dir
}
func TestChmodFile(t *testing.T) {
fs, dir := setup(t)
path := filepath.Join(dir, "file")
defer os.RemoveAll(dir)
defer os.Chmod(path, 0666)
fd, err := os.Create(path)
if err != nil {
t.Error(err)
}
fd.Close()
if err := os.Chmod(path, 0666); err != nil {
t.Error(err)
}
if stat, err := os.Stat(path); err != nil || stat.Mode()&os.ModePerm != 0666 {
t.Errorf("wrong perm: %t %#o", err == nil, stat.Mode()&os.ModePerm)
}
if err := fs.Chmod("file", 0444); err != nil {
t.Error(err)
}
if stat, err := os.Stat(path); err != nil || stat.Mode()&os.ModePerm != 0444 {
t.Errorf("wrong perm: %t %#o", err == nil, stat.Mode()&os.ModePerm)
}
}
func TestChmodDir(t *testing.T) {
fs, dir := setup(t)
path := filepath.Join(dir, "dir")
defer os.RemoveAll(dir)
mode := os.FileMode(0755)
if runtime.GOOS == "windows" {
mode = os.FileMode(0777)
}
defer os.Chmod(path, mode)
if err := os.Mkdir(path, mode); err != nil {
t.Error(err)
}
if stat, err := os.Stat(path); err != nil || stat.Mode()&os.ModePerm != mode {
t.Errorf("wrong perm: %t %#o", err == nil, stat.Mode()&os.ModePerm)
}
if err := fs.Chmod("dir", 0555); err != nil {
t.Error(err)
}
if stat, err := os.Stat(path); err != nil || stat.Mode()&os.ModePerm != 0555 {
t.Errorf("wrong perm: %t %#o", err == nil, stat.Mode()&os.ModePerm)
}
}
func TestChtimes(t *testing.T) {
fs, dir := setup(t)
path := filepath.Join(dir, "file")
defer os.RemoveAll(dir)
fd, err := os.Create(path)
if err != nil {
t.Error(err)
}
fd.Close()
mtime := time.Now().Add(-time.Hour)
fs.Chtimes("file", mtime, mtime)
stat, err := os.Stat(path)
if err != nil {
t.Error(err)
}
diff := stat.ModTime().Sub(mtime)
if diff > 3*time.Second || diff < -3*time.Second {
t.Errorf("%s != %s", stat.Mode(), mtime)
}
}
func TestCreate(t *testing.T) {
fs, dir := setup(t)
path := filepath.Join(dir, "file")
defer os.RemoveAll(dir)
if _, err := os.Stat(path); err == nil {
t.Errorf("exists?")
}
fd, err := fs.Create("file")
if err != nil {
t.Error(err)
}
fd.Close()
if _, err := os.Stat(path); err != nil {
t.Error(err)
}
}
func TestCreateSymlink(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("windows not supported")
}
fs, dir := setup(t)
path := filepath.Join(dir, "file")
defer os.RemoveAll(dir)
if err := fs.CreateSymlink("blah", "file"); err != nil {
t.Error(err)
}
if target, err := os.Readlink(path); err != nil || target != "blah" {
t.Error("target", target, "err", err)
}
if err := os.Remove(path); err != nil {
t.Error(err)
}
if err := fs.CreateSymlink(filepath.Join("..", "blah"), "file"); err != nil {
t.Error(err)
}
if target, err := os.Readlink(path); err != nil || target != filepath.Join("..", "blah") {
t.Error("target", target, "err", err)
}
}
func TestDirNames(t *testing.T) {
fs, dir := setup(t)
defer os.RemoveAll(dir)
// Case differences
testCases := []string{
"a",
"bC",
}
sort.Strings(testCases)
for _, sub := range testCases {
if err := os.Mkdir(filepath.Join(dir, sub), 0777); err != nil {
t.Error(err)
}
}
if dirs, err := fs.DirNames("."); err != nil || len(dirs) != len(testCases) {
t.Errorf("%s %s %s", err, dirs, testCases)
} else {
sort.Strings(dirs)
for i := range dirs {
if dirs[i] != testCases[i] {
t.Errorf("%s != %s", dirs[i], testCases[i])
}
}
}
}
func TestNames(t *testing.T) {
// Tests that all names are without the root directory.
fs, dir := setup(t)
defer os.RemoveAll(dir)
expected := "file"
fd, err := fs.Create(expected)
if err != nil {
t.Error(err)
}
defer fd.Close()
if fd.Name() != expected {
t.Errorf("incorrect %s != %s", fd.Name(), expected)
}
if stat, err := fd.Stat(); err != nil || stat.Name() != expected {
t.Errorf("incorrect %s != %s (%v)", stat.Name(), expected, err)
}
if err := fs.Mkdir("dir", 0777); err != nil {
t.Error(err)
}
expected = filepath.Join("dir", "file")
fd, err = fs.Create(expected)
if err != nil {
t.Error(err)
}
defer fd.Close()
if fd.Name() != expected {
t.Errorf("incorrect %s != %s", fd.Name(), expected)
}
// os.fd.Stat() returns just base, so do we.
if stat, err := fd.Stat(); err != nil || stat.Name() != filepath.Base(expected) {
t.Errorf("incorrect %s != %s (%v)", stat.Name(), filepath.Base(expected), err)
}
}
func TestGlob(t *testing.T) {
// Tests that all names are without the root directory.
fs, dir := setup(t)
defer os.RemoveAll(dir)
for _, dirToCreate := range []string{
filepath.Join("a", "test", "b"),
filepath.Join("a", "best", "b"),
filepath.Join("a", "best", "c"),
} {
if err := fs.MkdirAll(dirToCreate, 0777); err != nil {
t.Error(err)
}
}
testCases := []struct {
pattern string
matches []string
}{
{
filepath.Join("a", "?est", "?"),
[]string{
filepath.Join("a", "test", "b"),
filepath.Join("a", "best", "b"),
filepath.Join("a", "best", "c"),
},
},
{
filepath.Join("a", "?est", "b"),
[]string{
filepath.Join("a", "test", "b"),
filepath.Join("a", "best", "b"),
},
},
{
filepath.Join("a", "best", "?"),
[]string{
filepath.Join("a", "best", "b"),
filepath.Join("a", "best", "c"),
},
},
}
for _, testCase := range testCases {
results, err := fs.Glob(testCase.pattern)
sort.Strings(results)
sort.Strings(testCase.matches)
if err != nil {
t.Error(err)
}
if len(results) != len(testCase.matches) {
t.Errorf("result count mismatch")
}
for i := range testCase.matches {
if results[i] != testCase.matches[i] {
t.Errorf("%s != %s", results[i], testCase.matches[i])
}
}
}
}
func TestUsage(t *testing.T) {
fs, dir := setup(t)
defer os.RemoveAll(dir)
usage, err := fs.Usage(".")
if err != nil {
if runtime.GOOS == "netbsd" || runtime.GOOS == "openbsd" || runtime.GOOS == "solaris" {
t.Skip()
}
t.Errorf("Unexpected error: %s", err)
}
if usage.Free < 1 {
t.Error("Disk is full?", usage.Free)
}
}
func TestWindowsPaths(t *testing.T) {
if runtime.GOOS != "windows" {
t.Skip("Not useful on non-Windows")
return
}
testCases := []struct {
input string
expectedRoot string
expectedURI string
}{
{`e:\`, `\\?\e:\`, `e:\`},
{`\\?\e:\`, `\\?\e:\`, `e:\`},
{`\\192.0.2.22\network\share`, `\\192.0.2.22\network\share`, `\\192.0.2.22\network\share`},
}
for _, testCase := range testCases {
fs := newBasicFilesystem(testCase.input)
if fs.root != testCase.expectedRoot {
t.Errorf("root %q != %q", fs.root, testCase.expectedRoot)
}
if fs.URI() != testCase.expectedURI {
t.Errorf("uri %q != %q", fs.URI(), testCase.expectedURI)
}
}
fs := newBasicFilesystem(`relative\path`)
if fs.root == `relative\path` || !strings.HasPrefix(fs.root, "\\\\?\\") {
t.Errorf("%q == %q, expected absolutification", fs.root, `relative\path`)
}
}
func TestRooted(t *testing.T) {
type testcase struct {
root string
rel string
joined string
ok bool
}
cases := []testcase{
// Valid cases
{"foo", "bar", "foo/bar", true},
{"foo", "/bar", "foo/bar", true},
{"foo/", "bar", "foo/bar", true},
{"foo/", "/bar", "foo/bar", true},
{"baz/foo", "bar", "baz/foo/bar", true},
{"baz/foo", "/bar", "baz/foo/bar", true},
{"baz/foo/", "bar", "baz/foo/bar", true},
{"baz/foo/", "/bar", "baz/foo/bar", true},
{"foo", "bar/baz", "foo/bar/baz", true},
{"foo", "/bar/baz", "foo/bar/baz", true},
{"foo/", "bar/baz", "foo/bar/baz", true},
{"foo/", "/bar/baz", "foo/bar/baz", true},
{"baz/foo", "bar/baz", "baz/foo/bar/baz", true},
{"baz/foo", "/bar/baz", "baz/foo/bar/baz", true},
{"baz/foo/", "bar/baz", "baz/foo/bar/baz", true},
{"baz/foo/", "/bar/baz", "baz/foo/bar/baz", true},
// Not escape attempts, but oddly formatted relative paths. Disallowed.
{"foo", "./bar", "", false},
{"baz/foo", "./bar", "", false},
{"foo", "./bar/baz", "", false},
{"baz/foo", "./bar/baz", "", false},
{"baz/foo", "bar/../baz", "", false},
{"baz/foo", "/bar/../baz", "", false},
{"baz/foo", "./bar/../baz", "", false},
{"baz/foo", "bar/../baz", "", false},
{"baz/foo", "/bar/../baz", "", false},
{"baz/foo", "./bar/../baz", "", false},
// Results in an allowed path, but does it by probing. Disallowed.
{"foo", "../foo", "", false},
{"foo", "../foo/bar", "", false},
{"baz/foo", "../foo/bar", "", false},
{"baz/foo", "../../baz/foo/bar", "", false},
{"baz/foo", "bar/../../foo/bar", "", false},
{"baz/foo", "bar/../../../baz/foo/bar", "", false},
// Escape attempts.
{"foo", "", "", false},
{"foo", "/", "", false},
{"foo", "..", "", false},
{"foo", "/..", "", false},
{"foo", "../", "", false},
{"foo", "../bar", "", false},
{"foo", "../foobar", "", false},
{"foo/", "../bar", "", false},
{"foo/", "../foobar", "", false},
{"baz/foo", "../bar", "", false},
{"baz/foo", "../foobar", "", false},
{"baz/foo/", "../bar", "", false},
{"baz/foo/", "../foobar", "", false},
{"baz/foo/", "bar/../../quux/baz", "", false},
// Empty root is a misconfiguration.
{"", "/foo", "", false},
{"", "foo", "", false},
{"", ".", "", false},
{"", "..", "", false},
{"", "/", "", false},
{"", "", "", false},
// Root=/ is valid, and things should be verified as usual.
{"/", "foo", "/foo", true},
{"/", "/foo", "/foo", true},
{"/", "../foo", "", false},
{"/", "..", "", false},
{"/", "/", "", false},
{"/", "", "", false},
// special case for filesystems to be able to MkdirAll('.') for example
{"/", ".", "/", true},
}
if runtime.GOOS == "windows" {
extraCases := []testcase{
{`c:\`, `foo`, `c:\foo`, true},
{`\\?\c:\`, `foo`, `\\?\c:\foo`, true},
{`c:\`, `\foo`, `c:\foo`, true},
{`\\?\c:\`, `\foo`, `\\?\c:\foo`, true},
{`c:\`, `\\foo`, ``, false},
{`c:\`, ``, ``, false},
{`c:\`, `\`, ``, false},
{`\\?\c:\`, `\\foo`, ``, false},
{`\\?\c:\`, ``, ``, false},
{`\\?\c:\`, `\`, ``, false},
// makes no sense, but will be treated simply as a bad filename
{`c:\foo`, `d:\bar`, `c:\foo\d:\bar`, true},
// special case for filesystems to be able to MkdirAll('.') for example
{`c:\`, `.`, `c:\`, true},
{`\\?\c:\`, `.`, `\\?\c:\`, true},
}
for _, tc := range cases {
// Add case where root is backslashed, rel is forward slashed
extraCases = append(extraCases, testcase{
root: filepath.FromSlash(tc.root),
rel: tc.rel,
joined: tc.joined,
ok: tc.ok,
})
// and the opposite
extraCases = append(extraCases, testcase{
root: tc.root,
rel: filepath.FromSlash(tc.rel),
joined: tc.joined,
ok: tc.ok,
})
// and both backslashed
extraCases = append(extraCases, testcase{
root: filepath.FromSlash(tc.root),
rel: filepath.FromSlash(tc.rel),
joined: tc.joined,
ok: tc.ok,
})
}
cases = append(cases, extraCases...)
}
for _, tc := range cases {
fs := BasicFilesystem{root: tc.root}
res, err := fs.rooted(tc.rel)
if tc.ok {
if err != nil {
t.Errorf("Unexpected error for rooted(%q, %q): %v", tc.root, tc.rel, err)
continue
}
exp := filepath.FromSlash(tc.joined)
if res != exp {
t.Errorf("Unexpected result for rooted(%q, %q): %q != expected %q", tc.root, tc.rel, res, exp)
}
} else if err == nil {
t.Errorf("Unexpected pass for rooted(%q, %q) => %q", tc.root, tc.rel, res)
continue
}
}
}

57
lib/fs/basicfs_unix.go Normal file
View File

@ -0,0 +1,57 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build !windows
package fs
import "os"
func (BasicFilesystem) SymlinksSupported() bool {
return true
}
func (f *BasicFilesystem) CreateSymlink(target, name string) error {
name, err := f.rooted(name)
if err != nil {
return err
}
return os.Symlink(target, name)
}
func (f *BasicFilesystem) ReadSymlink(name string) (string, error) {
name, err := f.rooted(name)
if err != nil {
return "", err
}
return os.Readlink(name)
}
func (f *BasicFilesystem) MkdirAll(name string, perm FileMode) error {
name, err := f.rooted(name)
if err != nil {
return err
}
return os.MkdirAll(name, os.FileMode(perm))
}
// Unhide is a noop on unix, as unhiding files requires renaming them.
// We still check that the relative path does not try to escape the root
func (f *BasicFilesystem) Unhide(name string) error {
_, err := f.rooted(name)
return err
}
// Hide is a noop on unix, as hiding files requires renaming them.
// We still check that the relative path does not try to escape the root
func (f *BasicFilesystem) Hide(name string) error {
_, err := f.rooted(name)
return err
}
func (f *BasicFilesystem) Roots() ([]string, error) {
return []string{"/"}, nil
}

165
lib/fs/basicfs_windows.go Normal file
View File

@ -0,0 +1,165 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build windows
package fs
import (
"bytes"
"errors"
"fmt"
"os"
"path/filepath"
"syscall"
"unsafe"
)
var errNotSupported = errors.New("symlinks not supported")
func (BasicFilesystem) SymlinksSupported() bool {
return false
}
func (BasicFilesystem) ReadSymlink(path string) (string, error) {
return "", errNotSupported
}
func (BasicFilesystem) CreateSymlink(path, target string) error {
return errNotSupported
}
// MkdirAll creates a directory named path, along with any necessary parents,
// and returns nil, or else returns an error.
// The permission bits perm are used for all directories that MkdirAll creates.
// If path is already a directory, MkdirAll does nothing and returns nil.
func (f *BasicFilesystem) MkdirAll(path string, perm FileMode) error {
path, err := f.rooted(path)
if err != nil {
return err
}
return f.mkdirAll(path, os.FileMode(perm))
}
// Required due to https://github.com/golang/go/issues/10900
func (f *BasicFilesystem) mkdirAll(path string, perm os.FileMode) error {
// Fast path: if we can tell whether path is a directory or file, stop with success or error.
dir, err := os.Stat(path)
if err == nil {
if dir.IsDir() {
return nil
}
return &os.PathError{
Op: "mkdir",
Path: path,
Err: syscall.ENOTDIR,
}
}
// Slow path: make sure parent exists and then call Mkdir for path.
i := len(path)
for i > 0 && IsPathSeparator(path[i-1]) { // Skip trailing path separator.
i--
}
j := i
for j > 0 && !IsPathSeparator(path[j-1]) { // Scan backward over element.
j--
}
if j > 1 {
// Create parent
parent := path[0 : j-1]
if parent != filepath.VolumeName(parent) {
err = os.MkdirAll(parent, perm)
if err != nil {
return err
}
}
}
// Parent now exists; invoke Mkdir and use its result.
err = os.Mkdir(path, perm)
if err != nil {
// Handle arguments like "foo/." by
// double-checking that directory doesn't exist.
dir, err1 := os.Lstat(path)
if err1 == nil && dir.IsDir() {
return nil
}
return err
}
return nil
}
func (f *BasicFilesystem) Unhide(name string) error {
name, err := f.rooted(name)
if err != nil {
return err
}
p, err := syscall.UTF16PtrFromString(name)
if err != nil {
return err
}
attrs, err := syscall.GetFileAttributes(p)
if err != nil {
return err
}
attrs &^= syscall.FILE_ATTRIBUTE_HIDDEN
return syscall.SetFileAttributes(p, attrs)
}
func (f *BasicFilesystem) Hide(name string) error {
name, err := f.rooted(name)
if err != nil {
return err
}
p, err := syscall.UTF16PtrFromString(name)
if err != nil {
return err
}
attrs, err := syscall.GetFileAttributes(p)
if err != nil {
return err
}
attrs |= syscall.FILE_ATTRIBUTE_HIDDEN
return syscall.SetFileAttributes(p, attrs)
}
func (f *BasicFilesystem) Roots() ([]string, error) {
kernel32, err := syscall.LoadDLL("kernel32.dll")
if err != nil {
return nil, err
}
getLogicalDriveStringsHandle, err := kernel32.FindProc("GetLogicalDriveStringsA")
if err != nil {
return nil, err
}
buffer := [1024]byte{}
bufferSize := uint32(len(buffer))
hr, _, _ := getLogicalDriveStringsHandle.Call(uintptr(unsafe.Pointer(&bufferSize)), uintptr(unsafe.Pointer(&buffer)))
if hr == 0 {
return nil, fmt.Errorf("Syscall failed")
}
var drives []string
parts := bytes.Split(buffer[:], []byte{0})
for _, part := range parts {
if len(part) == 0 {
break
}
drives = append(drives, string(part))
}
return drives, nil
}

22
lib/fs/debug.go Normal file
View File

@ -0,0 +1,22 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package fs
import (
"os"
"strings"
"github.com/syncthing/syncthing/lib/logger"
)
var (
l = logger.DefaultLogger.NewFacility("filesystem", "Filesystem access")
)
func init() {
l.SetDebug("filesystem", strings.Contains(os.Getenv("STTRACE"), "filesystem") || os.Getenv("STTRACE") == "all")
}

41
lib/fs/errorfs.go Normal file
View File

@ -0,0 +1,41 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package fs
import "time"
type errorFilesystem struct {
err error
fsType FilesystemType
uri string
}
func (fs *errorFilesystem) Chmod(name string, mode FileMode) error { return fs.err }
func (fs *errorFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error { return fs.err }
func (fs *errorFilesystem) Create(name string) (File, error) { return nil, fs.err }
func (fs *errorFilesystem) CreateSymlink(name, target string) error { return fs.err }
func (fs *errorFilesystem) DirNames(name string) ([]string, error) { return nil, fs.err }
func (fs *errorFilesystem) Lstat(name string) (FileInfo, error) { return nil, fs.err }
func (fs *errorFilesystem) Mkdir(name string, perm FileMode) error { return fs.err }
func (fs *errorFilesystem) MkdirAll(name string, perm FileMode) error { return fs.err }
func (fs *errorFilesystem) Open(name string) (File, error) { return nil, fs.err }
func (fs *errorFilesystem) OpenFile(string, int, FileMode) (File, error) { return nil, fs.err }
func (fs *errorFilesystem) ReadSymlink(name string) (string, error) { return "", fs.err }
func (fs *errorFilesystem) Remove(name string) error { return fs.err }
func (fs *errorFilesystem) RemoveAll(name string) error { return fs.err }
func (fs *errorFilesystem) Rename(oldname, newname string) error { return fs.err }
func (fs *errorFilesystem) Stat(name string) (FileInfo, error) { return nil, fs.err }
func (fs *errorFilesystem) SymlinksSupported() bool { return false }
func (fs *errorFilesystem) Walk(root string, walkFn WalkFunc) error { return fs.err }
func (fs *errorFilesystem) Unhide(name string) error { return fs.err }
func (fs *errorFilesystem) Hide(name string) error { return fs.err }
func (fs *errorFilesystem) Glob(pattern string) ([]string, error) { return nil, fs.err }
func (fs *errorFilesystem) SyncDir(name string) error { return fs.err }
func (fs *errorFilesystem) Roots() ([]string, error) { return nil, fs.err }
func (fs *errorFilesystem) Usage(name string) (Usage, error) { return Usage{}, fs.err }
func (fs *errorFilesystem) Type() FilesystemType { return fs.fsType }
func (fs *errorFilesystem) URI() string { return fs.uri }

71
lib/fs/filesystem.go
View File

@ -7,6 +7,7 @@
package fs
import (
"errors"
"io"
"os"
"path/filepath"
@ -22,23 +23,38 @@ type Filesystem interface {
DirNames(name string) ([]string, error)
Lstat(name string) (FileInfo, error)
Mkdir(name string, perm FileMode) error
MkdirAll(name string, perm FileMode) error
Open(name string) (File, error)
OpenFile(name string, flags int, mode FileMode) (File, error)
ReadSymlink(name string) (string, error)
Remove(name string) error
RemoveAll(name string) error
Rename(oldname, newname string) error
Stat(name string) (FileInfo, error)
SymlinksSupported() bool
Walk(root string, walkFn WalkFunc) error
Hide(name string) error
Unhide(name string) error
Glob(pattern string) ([]string, error)
Roots() ([]string, error)
Usage(name string) (Usage, error)
Type() FilesystemType
URI() string
}
// The File interface abstracts access to a regular file, being a somewhat
// smaller interface than os.File
type File interface {
io.Reader
io.WriterAt
io.Closer
io.Reader
io.ReaderAt
io.Seeker
io.Writer
io.WriterAt
Name() string
Truncate(size int64) error
Stat() (FileInfo, error)
Sync() error
}
// The FileInfo interface is almost the same as os.FileInfo, but with the
@ -59,12 +75,27 @@ type FileInfo interface {
// FileMode is similar to os.FileMode
type FileMode uint32
// ModePerm is the equivalent of os.ModePerm
const ModePerm = FileMode(os.ModePerm)
// Usage represents filesystem space usage
type Usage struct {
Free int64
Total int64
}
// DefaultFilesystem is the fallback to use when nothing explicitly has
// been passed.
var DefaultFilesystem Filesystem = NewWalkFilesystem(NewBasicFilesystem())
// Equivalents from os package.
const ModePerm = FileMode(os.ModePerm)
const ModeSetgid = FileMode(os.ModeSetgid)
const ModeSetuid = FileMode(os.ModeSetuid)
const ModeSticky = FileMode(os.ModeSticky)
const PathSeparator = os.PathSeparator
const OptAppend = os.O_APPEND
const OptCreate = os.O_CREATE
const OptExclusive = os.O_EXCL
const OptReadOnly = os.O_RDONLY
const OptReadWrite = os.O_RDWR
const OptSync = os.O_SYNC
const OptTruncate = os.O_TRUNC
const OptWriteOnly = os.O_WRONLY
// SkipDir is used as a return value from WalkFuncs to indicate that
// the directory named in the call is to be skipped. It is not returned
@ -76,3 +107,29 @@ var IsExist = os.IsExist
// IsNotExist is the equivalent of os.IsNotExist
var IsNotExist = os.IsNotExist
// IsPermission is the equivalent of os.IsPermission
var IsPermission = os.IsPermission
// IsPathSeparator is the equivalent of os.IsPathSeparator
var IsPathSeparator = os.IsPathSeparator
func NewFilesystem(fsType FilesystemType, uri string) Filesystem {
var fs Filesystem
switch fsType {
case FilesystemTypeBasic:
fs = NewWalkFilesystem(newBasicFilesystem(uri))
default:
l.Debugln("Unknown filesystem", fsType, uri)
fs = &errorFilesystem{
fsType: fsType,
uri: uri,
err: errors.New("filesystem with type " + fsType.String() + " does not exist."),
}
}
if l.ShouldDebug("filesystem") {
fs = &logFilesystem{fs}
}
return fs
}

158
lib/fs/logfs.go Normal file
View File

@ -0,0 +1,158 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package fs
import (
"fmt"
"path/filepath"
"runtime"
"time"
)
type logFilesystem struct {
Filesystem
}
func getCaller() string {
_, file, line, ok := runtime.Caller(2)
if !ok {
return "unknown"
}
return fmt.Sprintf("%s:%d", filepath.Base(file), line)
}
func (fs *logFilesystem) Chmod(name string, mode FileMode) error {
err := fs.Filesystem.Chmod(name, mode)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Chmod", name, mode, err)
return err
}
func (fs *logFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error {
err := fs.Filesystem.Chtimes(name, atime, mtime)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Chtimes", name, atime, mtime, err)
return err
}
func (fs *logFilesystem) Create(name string) (File, error) {
file, err := fs.Filesystem.Create(name)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Create", name, file, err)
return file, err
}
func (fs *logFilesystem) CreateSymlink(name, target string) error {
err := fs.Filesystem.CreateSymlink(name, target)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "CreateSymlink", name, target, err)
return err
}
func (fs *logFilesystem) DirNames(name string) ([]string, error) {
names, err := fs.Filesystem.DirNames(name)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "DirNames", name, names, err)
return names, err
}
func (fs *logFilesystem) Lstat(name string) (FileInfo, error) {
info, err := fs.Filesystem.Lstat(name)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Lstat", name, info, err)
return info, err
}
func (fs *logFilesystem) Mkdir(name string, perm FileMode) error {
err := fs.Filesystem.Mkdir(name, perm)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Mkdir", name, perm, err)
return err
}
func (fs *logFilesystem) MkdirAll(name string, perm FileMode) error {
err := fs.Filesystem.MkdirAll(name, perm)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "MkdirAll", name, perm, err)
return err
}
func (fs *logFilesystem) Open(name string) (File, error) {
file, err := fs.Filesystem.Open(name)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Open", name, file, err)
return file, err
}
func (fs *logFilesystem) OpenFile(name string, flags int, mode FileMode) (File, error) {
file, err := fs.Filesystem.OpenFile(name, flags, mode)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "OpenFile", name, flags, mode, file, err)
return file, err
}
func (fs *logFilesystem) ReadSymlink(name string) (string, error) {
target, err := fs.Filesystem.ReadSymlink(name)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "ReadSymlink", name, target, err)
return target, err
}
func (fs *logFilesystem) Remove(name string) error {
err := fs.Filesystem.Remove(name)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Remove", name, err)
return err
}
func (fs *logFilesystem) RemoveAll(name string) error {
err := fs.Filesystem.RemoveAll(name)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "RemoveAll", name, err)
return err
}
func (fs *logFilesystem) Rename(oldname, newname string) error {
err := fs.Filesystem.Rename(oldname, newname)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Rename", oldname, newname, err)
return err
}
func (fs *logFilesystem) Stat(name string) (FileInfo, error) {
info, err := fs.Filesystem.Stat(name)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Stat", name, info, err)
return info, err
}
func (fs *logFilesystem) SymlinksSupported() bool {
supported := fs.Filesystem.SymlinksSupported()
l.Debugln(getCaller(), fs.Type(), fs.URI(), "SymlinksSupported", supported)
return supported
}
func (fs *logFilesystem) Walk(root string, walkFn WalkFunc) error {
err := fs.Filesystem.Walk(root, walkFn)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Walk", root, walkFn, err)
return err
}
func (fs *logFilesystem) Unhide(name string) error {
err := fs.Filesystem.Unhide(name)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Unhide", name, err)
return err
}
func (fs *logFilesystem) Hide(name string) error {
err := fs.Filesystem.Hide(name)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Hide", name, err)
return err
}
func (fs *logFilesystem) Glob(name string) ([]string, error) {
names, err := fs.Filesystem.Glob(name)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Glob", name, names, err)
return names, err
}
func (fs *logFilesystem) Roots() ([]string, error) {
roots, err := fs.Filesystem.Roots()
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Roots", roots, err)
return roots, err
}
func (fs *logFilesystem) Usage(name string) (Usage, error) {
usage, err := fs.Filesystem.Usage(name)
l.Debugln(getCaller(), fs.Type(), fs.URI(), "Usage", name, usage, err)
return usage, err
}

23
lib/fs/mtimefs.go
View File

@ -6,12 +6,7 @@
package fs
import (
"os"
"time"
"github.com/syncthing/syncthing/lib/osutil"
)
import "time"
// The database is where we store the virtual mtimes
type database interface {
@ -20,36 +15,34 @@ type database interface {
Delete(key string)
}
// variable so that we can mock it for testing
var osChtimes = os.Chtimes
// The MtimeFS is a filesystem with nanosecond mtime precision, regardless
// of what shenanigans the underlying filesystem gets up to. A nil MtimeFS
// just does the underlying operations with no additions.
type MtimeFS struct {
Filesystem
db database
chtimes func(string, time.Time, time.Time) error
db database
}
func NewMtimeFS(underlying Filesystem, db database) *MtimeFS {
return &MtimeFS{
Filesystem: underlying,
chtimes: underlying.Chtimes, // for mocking it out in the tests
db: db,
}
}
func (f *MtimeFS) Chtimes(name string, atime, mtime time.Time) error {
if f == nil {
return osChtimes(name, atime, mtime)
return f.chtimes(name, atime, mtime)
}
// Do a normal Chtimes call, don't care if it succeeds or not.
osChtimes(name, atime, mtime)
f.chtimes(name, atime, mtime)
// Stat the file to see what happened. Here we *do* return an error,
// because it might be "does not exist" or similar. osutil.Lstat is the
// souped up version to account for Android breakage.
info, err := osutil.Lstat(name)
// because it might be "does not exist" or similar.
info, err := f.Filesystem.Lstat(name)
if err != nil {
return err
}

8
lib/fs/mtimefs_test.go
View File

@ -25,22 +25,22 @@ func TestMtimeFS(t *testing.T) {
// a random time with nanosecond precision
testTime := time.Unix(1234567890, 123456789)
mtimefs := NewMtimeFS(DefaultFilesystem, make(mapStore))
mtimefs := NewMtimeFS(newBasicFilesystem("."), make(mapStore))
// Do one Chtimes call that will go through to the normal filesystem
osChtimes = os.Chtimes
mtimefs.chtimes = os.Chtimes
if err := mtimefs.Chtimes("testdata/exists0", testTime, testTime); err != nil {
t.Error("Should not have failed:", err)
}
// Do one call that gets an error back from the underlying Chtimes
osChtimes = failChtimes
mtimefs.chtimes = failChtimes
if err := mtimefs.Chtimes("testdata/exists1", testTime, testTime); err != nil {
t.Error("Should not have failed:", err)
}
// Do one call that gets struck by an exceptionally evil Chtimes
osChtimes = evilChtimes
mtimefs.chtimes = evilChtimes
if err := mtimefs.Chtimes("testdata/exists2", testTime, testTime); err != nil {
t.Error("Should not have failed:", err)
}

36
lib/fs/types.go Normal file
View File

@ -0,0 +1,36 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package fs
type FilesystemType int
const (
FilesystemTypeBasic FilesystemType = iota // default is basic
)
func (t FilesystemType) String() string {
switch t {
case FilesystemTypeBasic:
return "basic"
default:
return "unknown"
}
}
func (t FilesystemType) MarshalText() ([]byte, error) {
return []byte(t.String()), nil
}
func (t *FilesystemType) UnmarshalText(bs []byte) error {
switch string(bs) {
case "basic":
*t = FilesystemTypeBasic
default:
*t = FilesystemTypeBasic
}
return nil
}

55
lib/fs/util.go Normal file
View File

@ -0,0 +1,55 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package fs
import (
"errors"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
)
var errNoHome = errors.New("no home directory found - set $HOME (or the platform equivalent)")
func ExpandTilde(path string) (string, error) {
if path == "~" {
return getHomeDir()
}
path = filepath.FromSlash(path)
if !strings.HasPrefix(path, fmt.Sprintf("~%c", PathSeparator)) {
return path, nil
}
home, err := getHomeDir()
if err != nil {
return "", err
}
return filepath.Join(home, path[2:]), nil
}
func getHomeDir() (string, error) {
var home string
switch runtime.GOOS {
case "windows":
home = filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath"))
if home == "" {
home = os.Getenv("UserProfile")
}
default:
home = os.Getenv("HOME")
}
if home == "" {
return "", errNoHome
}
return home, nil
}

10
lib/fs/walkfs.go
View File

@ -28,16 +28,16 @@ import "path/filepath"
// Walk skips the remaining files in the containing directory.
type WalkFunc func(path string, info FileInfo, err error) error
type WalkFilesystem struct {
type walkFilesystem struct {
Filesystem
}
func NewWalkFilesystem(next Filesystem) *WalkFilesystem {
return &WalkFilesystem{next}
func NewWalkFilesystem(next Filesystem) Filesystem {
return &walkFilesystem{next}
}
// walk recursively descends path, calling walkFn.
func (f *WalkFilesystem) walk(path string, info FileInfo, walkFn WalkFunc) error {
func (f *walkFilesystem) walk(path string, info FileInfo, walkFn WalkFunc) error {
err := walkFn(path, info, nil)
if err != nil {
if info.IsDir() && err == SkipDir {
@ -80,7 +80,7 @@ func (f *WalkFilesystem) walk(path string, info FileInfo, walkFn WalkFunc) error
// order, which makes the output deterministic but means that for very
// large directories Walk can be inefficient.
// Walk does not follow symbolic links.
func (f *WalkFilesystem) Walk(root string, walkFn WalkFunc) error {
func (f *walkFilesystem) Walk(root string, walkFn WalkFunc) error {
info, err := f.Lstat(root)
if err != nil {
return walkFn(root, nil, err)

36
lib/ignore/ignore.go
View File

@ -12,13 +12,13 @@ import (
"crypto/md5"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/gobwas/glob"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/sync"
)
@ -77,6 +77,7 @@ type ChangeDetector interface {
}
type Matcher struct {
fs fs.Filesystem
lines []string // exact lines read from .stignore
patterns []Pattern // patterns including those from included files
withCache bool
@ -105,8 +106,9 @@ func WithChangeDetector(cd ChangeDetector) Option {
}
}
func New(opts ...Option) *Matcher {
func New(fs fs.Filesystem, opts ...Option) *Matcher {
m := &Matcher{
fs: fs,
stop: make(chan struct{}),
mut: sync.NewMutex(),
}
@ -114,7 +116,7 @@ func New(opts ...Option) *Matcher {
opt(m)
}
if m.changeDetector == nil {
m.changeDetector = newModtimeChecker()
m.changeDetector = newModtimeChecker(fs)
}
if m.withCache {
go m.clean(2 * time.Hour)
@ -130,7 +132,7 @@ func (m *Matcher) Load(file string) error {
return nil
}
fd, err := os.Open(file)
fd, err := m.fs.Open(file)
if err != nil {
m.parseLocked(&bytes.Buffer{}, file)
return err
@ -156,7 +158,7 @@ func (m *Matcher) Parse(r io.Reader, file string) error {
}
func (m *Matcher) parseLocked(r io.Reader, file string) error {
lines, patterns, err := parseIgnoreFile(r, file, m.changeDetector)
lines, patterns, err := parseIgnoreFile(m.fs, r, file, m.changeDetector)
// Error is saved and returned at the end. We process the patterns
// (possibly blank) anyway.
@ -298,12 +300,12 @@ func hashPatterns(patterns []Pattern) string {
return fmt.Sprintf("%x", h.Sum(nil))
}
func loadIgnoreFile(file string, cd ChangeDetector) ([]string, []Pattern, error) {
func loadIgnoreFile(fs fs.Filesystem, file string, cd ChangeDetector) ([]string, []Pattern, error) {
if cd.Seen(file) {
return nil, nil, fmt.Errorf("multiple include of ignore file %q", file)
}
fd, err := os.Open(file)
fd, err := fs.Open(file)
if err != nil {
return nil, nil, err
}
@ -316,10 +318,10 @@ func loadIgnoreFile(file string, cd ChangeDetector) ([]string, []Pattern, error)
cd.Remember(file, info.ModTime())
return parseIgnoreFile(fd, file, cd)
return parseIgnoreFile(fs, fd, file, cd)
}
func parseIgnoreFile(fd io.Reader, currentFile string, cd ChangeDetector) ([]string, []Pattern, error) {
func parseIgnoreFile(fs fs.Filesystem, fd io.Reader, currentFile string, cd ChangeDetector) ([]string, []Pattern, error) {
var lines []string
var patterns []Pattern
@ -386,7 +388,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, cd ChangeDetector) ([]str
} else if strings.HasPrefix(line, "#include ") {
includeRel := line[len("#include "):]
includeFile := filepath.Join(filepath.Dir(currentFile), includeRel)
_, includePatterns, err := loadIgnoreFile(includeFile, cd)
_, includePatterns, err := loadIgnoreFile(fs, includeFile, cd)
if err != nil {
return fmt.Errorf("include of %q: %v", includeRel, err)
}
@ -450,7 +452,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, cd ChangeDetector) ([]str
// path must be clean (i.e., in canonical shortest form).
func IsInternal(file string) bool {
internals := []string{".stfolder", ".stignore", ".stversions"}
pathSep := string(os.PathSeparator)
pathSep := string(fs.PathSeparator)
for _, internal := range internals {
if file == internal {
return true
@ -463,8 +465,8 @@ func IsInternal(file string) bool {
}
// WriteIgnores is a convenience function to avoid code duplication
func WriteIgnores(path string, content []string) error {
fd, err := osutil.CreateAtomic(path)
func WriteIgnores(filesystem fs.Filesystem, path string, content []string) error {
fd, err := osutil.CreateAtomicFilesystem(filesystem, path)
if err != nil {
return err
}
@ -476,18 +478,20 @@ func WriteIgnores(path string, content []string) error {
if err := fd.Close(); err != nil {
return err
}
osutil.HideFile(path)
filesystem.Hide(path)
return nil
}
// modtimeChecker is the default implementation of ChangeDetector
type modtimeChecker struct {
fs fs.Filesystem
modtimes map[string]time.Time
}
func newModtimeChecker() *modtimeChecker {
func newModtimeChecker(fs fs.Filesystem) *modtimeChecker {
return &modtimeChecker{
fs: fs,
modtimes: map[string]time.Time{},
}
}
@ -507,7 +511,7 @@ func (c *modtimeChecker) Reset() {
func (c *modtimeChecker) Changed() bool {
for name, modtime := range c.modtimes {
info, err := os.Stat(name)
info, err := c.fs.Stat(name)
if err != nil {
return true
}

114
lib/ignore/ignore_test.go
View File

@ -15,11 +15,14 @@ import (
"runtime"
"testing"
"time"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
)
func TestIgnore(t *testing.T) {
pats := New(WithCache(true))
err := pats.Load("testdata/.stignore")
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), WithCache(true))
err := pats.Load(".stignore")
if err != nil {
t.Fatal(err)
}
@ -68,7 +71,7 @@ func TestExcludes(t *testing.T) {
i*2
!ign2
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -113,7 +116,7 @@ func TestFlagOrder(t *testing.T) {
(?i)(?d)(?d)!ign9
(?d)(?d)!ign10
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -148,7 +151,7 @@ func TestDeletables(t *testing.T) {
ign7
(?i)ign8
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -187,7 +190,7 @@ func TestBadPatterns(t *testing.T) {
}
for _, pat := range badPatterns {
err := New(WithCache(true)).Parse(bytes.NewBufferString(pat), ".stignore")
err := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)).Parse(bytes.NewBufferString(pat), ".stignore")
if err == nil {
t.Errorf("No error for pattern %q", pat)
}
@ -195,7 +198,7 @@ func TestBadPatterns(t *testing.T) {
}
func TestCaseSensitivity(t *testing.T) {
ign := New(WithCache(true))
ign := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := ign.Parse(bytes.NewBufferString("test"), ".stignore")
if err != nil {
t.Error(err)
@ -225,29 +228,36 @@ func TestCaseSensitivity(t *testing.T) {
}
func TestCaching(t *testing.T) {
fd1, err := ioutil.TempFile("", "")
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
fd2, err := ioutil.TempFile("", "")
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
fd1, err := osutil.TempFile(fs, "", "")
if err != nil {
t.Fatal(err)
}
fd2, err := osutil.TempFile(fs, "", "")
if err != nil {
t.Fatal(err)
}
defer fd1.Close()
defer fd2.Close()
defer os.Remove(fd1.Name())
defer os.Remove(fd2.Name())
defer fs.Remove(fd1.Name())
defer fs.Remove(fd2.Name())
_, err = fd1.WriteString("/x/\n#include " + filepath.Base(fd2.Name()) + "\n")
_, err = fd1.Write([]byte("/x/\n#include " + filepath.Base(fd2.Name()) + "\n"))
if err != nil {
t.Fatal(err)
}
fd2.WriteString("/y/\n")
fd2.Write([]byte("/y/\n"))
pats := New(WithCache(true))
pats := New(fs, WithCache(true))
err = pats.Load(fd1.Name())
if err != nil {
t.Fatal(err)
@ -280,10 +290,10 @@ func TestCaching(t *testing.T) {
// Modify the include file, expect empty cache. Ensure the timestamp on
// the file changes.
fd2.WriteString("/z/\n")
fd2.Write([]byte("/z/\n"))
fd2.Sync()
fakeTime := time.Now().Add(5 * time.Second)
os.Chtimes(fd2.Name(), fakeTime, fakeTime)
fs.Chtimes(fd2.Name(), fakeTime, fakeTime)
err = pats.Load(fd1.Name())
if err != nil {
@ -312,10 +322,10 @@ func TestCaching(t *testing.T) {
// Modify the root file, expect cache to be invalidated
fd1.WriteString("/a/\n")
fd1.Write([]byte("/a/\n"))
fd1.Sync()
fakeTime = time.Now().Add(5 * time.Second)
os.Chtimes(fd1.Name(), fakeTime, fakeTime)
fs.Chtimes(fd1.Name(), fakeTime, fakeTime)
err = pats.Load(fd1.Name())
if err != nil {
@ -354,7 +364,7 @@ func TestCommentsAndBlankLines(t *testing.T) {
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Error(err)
@ -382,7 +392,7 @@ flamingo
*.crow
*.crow
`
pats := New()
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
b.Error(err)
@ -411,20 +421,27 @@ flamingo
*.crow
`
// Caches per file, hence write the patterns to a file.
fd, err := ioutil.TempFile("", "")
dir, err := ioutil.TempDir("", "")
if err != nil {
b.Fatal(err)
}
_, err = fd.WriteString(stignore)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
fd, err := osutil.TempFile(fs, "", "")
if err != nil {
b.Fatal(err)
}
_, err = fd.Write([]byte(stignore))
defer fd.Close()
defer os.Remove(fd.Name())
defer fs.Remove(fd.Name())
if err != nil {
b.Fatal(err)
}
// Load the patterns
pats := New(WithCache(true))
pats := New(fs, WithCache(true))
err = pats.Load(fd.Name())
if err != nil {
b.Fatal(err)
@ -445,22 +462,29 @@ flamingo
}
func TestCacheReload(t *testing.T) {
fd, err := ioutil.TempFile("", "")
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
fd, err := osutil.TempFile(fs, "", "")
if err != nil {
t.Fatal(err)
}
defer fd.Close()
defer os.Remove(fd.Name())
defer fs.Remove(fd.Name())
// Ignore file matches f1 and f2
_, err = fd.WriteString("f1\nf2\n")
_, err = fd.Write([]byte("f1\nf2\n"))
if err != nil {
t.Fatal(err)
}
pats := New(WithCache(true))
pats := New(fs, WithCache(true))
err = pats.Load(fd.Name())
if err != nil {
t.Fatal(err)
@ -488,13 +512,13 @@ func TestCacheReload(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = fd.WriteString("f1\nf3\n")
_, err = fd.Write([]byte("f1\nf3\n"))
if err != nil {
t.Fatal(err)
}
fd.Sync()
fakeTime := time.Now().Add(5 * time.Second)
os.Chtimes(fd.Name(), fakeTime, fakeTime)
fs.Chtimes(fd.Name(), fakeTime, fakeTime)
err = pats.Load(fd.Name())
if err != nil {
@ -515,7 +539,7 @@ func TestCacheReload(t *testing.T) {
}
func TestHash(t *testing.T) {
p1 := New(WithCache(true))
p1 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := p1.Load("testdata/.stignore")
if err != nil {
t.Fatal(err)
@ -531,7 +555,7 @@ func TestHash(t *testing.T) {
/ffile
lost+found
`
p2 := New(WithCache(true))
p2 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err = p2.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -546,7 +570,7 @@ func TestHash(t *testing.T) {
/ffile
lost+found
`
p3 := New(WithCache(true))
p3 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err = p3.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -570,7 +594,7 @@ func TestHash(t *testing.T) {
}
func TestHashOfEmpty(t *testing.T) {
p1 := New(WithCache(true))
p1 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := p1.Load("testdata/.stignore")
if err != nil {
t.Fatal(err)
@ -608,7 +632,7 @@ func TestWindowsPatterns(t *testing.T) {
a/b
c\d
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -633,7 +657,7 @@ func TestAutomaticCaseInsensitivity(t *testing.T) {
A/B
c/d
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -652,7 +676,7 @@ func TestCommas(t *testing.T) {
foo,bar.txt
{baz,quux}.txt
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -683,7 +707,7 @@ func TestIssue3164(t *testing.T) {
(?d)(?i)/foo
(?d)(?i)**/bar
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -719,7 +743,7 @@ func TestIssue3174(t *testing.T) {
stignore := `
*ä*
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -734,7 +758,7 @@ func TestIssue3639(t *testing.T) {
stignore := `
foo/
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -767,7 +791,7 @@ func TestIssue3674(t *testing.T) {
{"as/dc", true},
}
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -799,7 +823,7 @@ func TestGobwasGlobIssue18(t *testing.T) {
{"bbaa", false},
}
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -859,7 +883,7 @@ func TestRoot(t *testing.T) {
{"b", true},
}
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@ -876,12 +900,12 @@ func TestRoot(t *testing.T) {
func TestLines(t *testing.T) {
stignore := `
#include testdata/excludes
!/a
/*
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)

178
lib/model/model.go
View File

@ -14,7 +14,6 @@ import (
"fmt"
"io"
"net"
"os"
"path/filepath"
"reflect"
"runtime"
@ -81,6 +80,7 @@ type Model struct {
clientVersion string
folderCfgs map[string]config.FolderConfiguration // folder -> cfg
folderFs map[string]fs.Filesystem // folder -> fs
folderFiles map[string]*db.FileSet // folder -> files
folderDevices folderDeviceSet // folder -> deviceIDs
deviceFolders map[protocol.DeviceID][]string // deviceID -> folders
@ -99,21 +99,18 @@ type Model struct {
pmut sync.RWMutex // protects the above
}
type folderFactory func(*Model, config.FolderConfiguration, versioner.Versioner, *fs.MtimeFS) service
type folderFactory func(*Model, config.FolderConfiguration, versioner.Versioner, fs.Filesystem) service
var (
folderFactories = make(map[config.FolderType]folderFactory, 0)
)
var (
errFolderPathEmpty = errors.New("folder path empty")
errFolderPathMissing = errors.New("folder path missing")
errFolderMarkerMissing = errors.New("folder marker missing")
errInvalidFilename = errors.New("filename is invalid")
errDeviceUnknown = errors.New("unknown device")
errDevicePaused = errors.New("device is paused")
errDeviceIgnored = errors.New("device is ignored")
errNotRelative = errors.New("not a relative path")
errFolderPaused = errors.New("folder is paused")
errFolderMissing = errors.New("no such folder")
errNetworkNotAllowed = errors.New("network not allowed")
@ -140,6 +137,7 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, clientName, clientVersi
clientName: clientName,
clientVersion: clientVersion,
folderCfgs: make(map[string]config.FolderConfiguration),
folderFs: make(map[string]fs.Filesystem),
folderFiles: make(map[string]*db.FileSet),
folderDevices: make(folderDeviceSet),
deviceFolders: make(map[protocol.DeviceID][]string),
@ -245,7 +243,7 @@ func (m *Model) startFolderLocked(folder string) config.FolderType {
l.Fatalf("Requested versioning type %q that does not exist", cfg.Versioning.Type)
}
ver = versionerFactory(folder, cfg.Path(), cfg.Versioning.Params)
ver = versionerFactory(folder, cfg.Filesystem(), cfg.Versioning.Params)
if service, ok := ver.(suture.Service); ok {
// The versioner implements the suture.Service interface, so
// expects to be run in the background in addition to being called
@ -271,7 +269,12 @@ func (m *Model) warnAboutOverwritingProtectedFiles(folder string) {
return
}
folderLocation := m.folderCfgs[folder].Path()
// This is a bit of a hack.
ffs := m.folderCfgs[folder].Filesystem()
if ffs.Type() != fs.FilesystemTypeBasic {
return
}
folderLocation := ffs.URI()
ignores := m.folderIgnores[folder]
var filesAtRisk []string
@ -300,6 +303,10 @@ func (m *Model) AddFolder(cfg config.FolderConfiguration) {
panic("cannot add empty folder id")
}
if len(cfg.Path) == 0 {
panic("cannot add empty folder path")
}
m.fmut.Lock()
m.addFolderLocked(cfg)
m.fmut.Unlock()
@ -307,15 +314,16 @@ func (m *Model) AddFolder(cfg config.FolderConfiguration) {
func (m *Model) addFolderLocked(cfg config.FolderConfiguration) {
m.folderCfgs[cfg.ID] = cfg
m.folderFiles[cfg.ID] = db.NewFileSet(cfg.ID, m.db)
folderFs := cfg.Filesystem()
m.folderFiles[cfg.ID] = db.NewFileSet(cfg.ID, folderFs, m.db)
for _, device := range cfg.Devices {
m.folderDevices.set(device.DeviceID, cfg.ID)
m.deviceFolders[device.DeviceID] = append(m.deviceFolders[device.DeviceID], cfg.ID)
}
ignores := ignore.New(ignore.WithCache(m.cacheIgnoredFiles))
if err := ignores.Load(filepath.Join(cfg.Path(), ".stignore")); err != nil && !os.IsNotExist(err) {
ignores := ignore.New(folderFs, ignore.WithCache(m.cacheIgnoredFiles))
if err := ignores.Load(".stignore"); err != nil && !fs.IsNotExist(err) {
l.Warnln("Loading ignores:", err)
}
m.folderIgnores[cfg.ID] = ignores
@ -327,8 +335,8 @@ func (m *Model) RemoveFolder(folder string) {
// Delete syncthing specific files
folderCfg := m.folderCfgs[folder]
folderPath := folderCfg.Path()
os.Remove(filepath.Join(folderPath, ".stfolder"))
fs := folderCfg.Filesystem()
fs.Remove(".stfolder")
m.tearDownFolderLocked(folder)
// Remove it from the database
@ -1139,16 +1147,10 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
}
m.fmut.RLock()
folderCfg := m.folderCfgs[folder]
folderPath := folderCfg.Path()
folderIgnores := m.folderIgnores[folder]
m.fmut.RUnlock()
fn, err := rootedJoinedPath(folderPath, name)
if err != nil {
// Request tries to escape!
l.Debugf("%v Invalid REQ(in) tries to escape: %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, len(buf))
return protocol.ErrInvalid
}
folderFs := folderCfg.Filesystem()
// Having passed the rootedJoinedPath check above, we know "name" is
// acceptable relative to "folderPath" and in canonical form, so we can
@ -1164,7 +1166,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
return protocol.ErrNoSuchFile
}
if err := osutil.TraversesSymlink(folderPath, filepath.Dir(name)); err != nil {
if err := osutil.TraversesSymlink(folderFs, filepath.Dir(name)); err != nil {
l.Debugf("%v REQ(in) traversal check: %s - %s: %q / %q o=%d s=%d", m, err, deviceID, folder, name, offset, len(buf))
return protocol.ErrNoSuchFile
}
@ -1172,29 +1174,29 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
// Only check temp files if the flag is set, and if we are set to advertise
// the temp indexes.
if fromTemporary && !folderCfg.DisableTempIndexes {
tempFn := filepath.Join(folderPath, ignore.TempName(name))
tempFn := ignore.TempName(name)
if info, err := osutil.Lstat(tempFn); err != nil || !info.Mode().IsRegular() {
if info, err := folderFs.Lstat(tempFn); err != nil || !info.IsRegular() {
// Reject reads for anything that doesn't exist or is something
// other than a regular file.
return protocol.ErrNoSuchFile
}
if err := readOffsetIntoBuf(tempFn, offset, buf); err == nil {
if err := readOffsetIntoBuf(folderFs, tempFn, offset, buf); err == nil {
return nil
}
// Fall through to reading from a non-temp file, just incase the temp
// file has finished downloading.
}
if info, err := osutil.Lstat(fn); err != nil || !info.Mode().IsRegular() {
if info, err := folderFs.Lstat(name); err != nil || !info.IsRegular() {
// Reject reads for anything that doesn't exist or is something
// other than a regular file.
return protocol.ErrNoSuchFile
}
err = readOffsetIntoBuf(fn, offset, buf)
if os.IsNotExist(err) {
err := readOffsetIntoBuf(folderFs, name, offset, buf)
if fs.IsNotExist(err) {
return protocol.ErrNoSuchFile
} else if err != nil {
return protocol.ErrGeneric
@ -1259,9 +1261,8 @@ func (m *Model) GetIgnores(folder string) ([]string, []string, error) {
}
if cfg, ok := m.cfg.Folders()[folder]; ok {
matcher := ignore.New()
path := filepath.Join(cfg.Path(), ".stignore")
if err := matcher.Load(path); err != nil {
matcher := ignore.New(cfg.Filesystem())
if err := matcher.Load(".stignore"); err != nil {
return nil, nil, err
}
return matcher.Lines(), matcher.Patterns(), nil
@ -1276,7 +1277,7 @@ func (m *Model) SetIgnores(folder string, content []string) error {
return fmt.Errorf("Folder %s does not exist", folder)
}
if err := ignore.WriteIgnores(filepath.Join(cfg.Path(), ".stignore"), content); err != nil {
if err := ignore.WriteIgnores(cfg.Filesystem(), ".stignore", content); err != nil {
l.Warnln("Saving .stignore:", err)
return err
}
@ -1610,8 +1611,6 @@ func (m *Model) updateLocals(folder string, fs []protocol.FileInfo) {
}
func (m *Model) diskChangeDetected(folderCfg config.FolderConfiguration, files []protocol.FileInfo, typeOfEvent events.EventType) {
path := strings.Replace(folderCfg.Path(), `\\?\`, "", 1)
for _, file := range files {
objType := "file"
action := "modified"
@ -1634,10 +1633,6 @@ func (m *Model) diskChangeDetected(folderCfg config.FolderConfiguration, files [
action = "deleted"
}
// The full file path, adjusted to the local path separator character. Also
// for windows paths, strip unwanted chars from the front.
path := filepath.Join(path, filepath.FromSlash(file.Name))
// Two different events can be fired here based on what EventType is passed into function
events.Default.Log(typeOfEvent, map[string]string{
"folder": folderCfg.ID,
@ -1645,7 +1640,7 @@ func (m *Model) diskChangeDetected(folderCfg config.FolderConfiguration, files [
"label": folderCfg.Label,
"action": action,
"type": objType,
"path": path,
"path": filepath.FromSlash(file.Name),
"modifiedBy": file.ModifiedBy.String(),
})
}
@ -1738,20 +1733,17 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
// not relevant, we just want the dotdot escape detection here. For
// historical reasons we may get paths that end in a slash. We
// remove that first to allow the rootedJoinedPath to pass.
sub = strings.TrimRight(sub, string(os.PathSeparator))
if _, err := rootedJoinedPath("root", sub); err != nil {
return errors.New("invalid subpath")
}
sub = strings.TrimRight(sub, string(fs.PathSeparator))
subDirs[i] = sub
}
m.fmut.Lock()
fs := m.folderFiles[folder]
fset := m.folderFiles[folder]
folderCfg := m.folderCfgs[folder]
ignores := m.folderIgnores[folder]
runner, ok := m.folderRunners[folder]
m.fmut.Unlock()
mtimefs := fs.MtimeFS()
mtimefs := fset.MtimeFS()
// Check if the ignore patterns changed as part of scanning this folder.
// If they did we should schedule a pull of the folder so that we
@ -1778,7 +1770,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
return err
}
if err := ignores.Load(filepath.Join(folderCfg.Path(), ".stignore")); err != nil && !os.IsNotExist(err) {
if err := ignores.Load(".stignore"); err != nil && !fs.IsNotExist(err) {
err = fmt.Errorf("loading ignores: %v", err)
runner.setError(err)
l.Infof("Stopping folder %s due to error: %s", folderCfg.Description(), err)
@ -1789,7 +1781,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
// directory, and don't scan subdirectories of things we've already
// scanned.
subDirs = unifySubs(subDirs, func(f string) bool {
_, ok := fs.Get(protocol.LocalDeviceID, f)
_, ok := fset.Get(protocol.LocalDeviceID, f)
return ok
})
@ -1797,7 +1789,6 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
fchan, err := scanner.Walk(ctx, scanner.Config{
Folder: folderCfg.ID,
Dir: folderCfg.Path(),
Subs: subDirs,
Matcher: ignores,
BlockSize: protocol.BlockSize,
@ -1860,7 +1851,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
for _, sub := range subDirs {
var iterError error
fs.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool {
fset.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool {
f := fi.(db.FileInfoTruncated)
if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
if err := m.CheckFolderHealth(folder); err != nil {
@ -1895,9 +1886,9 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
// The file is valid and not deleted. Lets check if it's
// still here.
if _, err := mtimefs.Lstat(filepath.Join(folderCfg.Path(), f.Name)); err != nil {
if _, err := mtimefs.Lstat(f.Name); err != nil {
// We don't specifically verify that the error is
// os.IsNotExist because there is a corner case when a
// fs.IsNotExist because there is a corner case when a
// directory is suddenly transformed into a file. When that
// happens, files that were in the directory (that is now a
// file) are deleted but will return a confusing error ("not a
@ -2275,11 +2266,9 @@ func (m *Model) CheckFolderHealth(id string) error {
// checkFolderPath returns nil if the folder path exists and has the marker file.
func (m *Model) checkFolderPath(folder config.FolderConfiguration) error {
if folder.Path() == "" {
return errFolderPathEmpty
}
fs := folder.Filesystem()
if fi, err := os.Stat(folder.Path()); err != nil || !fi.IsDir() {
if fi, err := fs.Stat("."); err != nil || !fi.IsDir() {
return errFolderPathMissing
}
@ -2293,30 +2282,35 @@ func (m *Model) checkFolderPath(folder config.FolderConfiguration) error {
// checkFolderFreeSpace returns nil if the folder has the required amount of
// free space, or if folder free space checking is disabled.
func (m *Model) checkFolderFreeSpace(folder config.FolderConfiguration) error {
return m.checkFreeSpace(folder.MinDiskFree, folder.Path())
return m.checkFreeSpace(folder.MinDiskFree, folder.Filesystem())
}
// checkHomeDiskFree returns nil if the home disk has the required amount of
// free space, or if home disk free space checking is disabled.
func (m *Model) checkHomeDiskFree() error {
return m.checkFreeSpace(m.cfg.Options().MinHomeDiskFree, m.cfg.ConfigPath())
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, filepath.Dir(m.cfg.ConfigPath()))
return m.checkFreeSpace(m.cfg.Options().MinHomeDiskFree, fs)
}
func (m *Model) checkFreeSpace(req config.Size, path string) error {
func (m *Model) checkFreeSpace(req config.Size, fs fs.Filesystem) error {
val := req.BaseValue()
if val <= 0 {
return nil
}
usage, err := fs.Usage(".")
if err != nil {
return fmt.Errorf("failed to check available storage space")
}
if req.Percentage() {
free, err := osutil.DiskFreePercentage(path)
if err == nil && free < val {
return fmt.Errorf("insufficient space in %v: %f %% < %v", path, free, req)
freePct := (1 - float64(usage.Free)/float64(usage.Total)) * 100
if err == nil && freePct < val {
return fmt.Errorf("insufficient space in %v %v: %f %% < %v", fs.Type(), fs.URI(), freePct, req)
}
} else {
free, err := osutil.DiskFreeBytes(path)
if err == nil && float64(free) < val {
return fmt.Errorf("insufficient space in %v: %v < %v", path, free, req)
if err == nil && float64(usage.Free) < val {
return fmt.Errorf("insufficient space in %v %v: %v < %v", fs.Type(), fs.URI(), usage.Free, req)
}
}
@ -2533,8 +2527,8 @@ func stringSliceWithout(ss []string, s string) []string {
return ss
}
func readOffsetIntoBuf(file string, offset int64, buf []byte) error {
fd, err := os.Open(file)
func readOffsetIntoBuf(fs fs.Filesystem, file string, offset int64, buf []byte) error {
fd, err := fs.Open(file)
if err != nil {
l.Debugln("readOffsetIntoBuf.Open", file, err)
return err
@ -2585,7 +2579,7 @@ func simplifySortedPaths(subs []string) []string {
next:
for _, sub := range subs {
for _, existing := range cleaned {
if sub == existing || strings.HasPrefix(sub, existing+string(os.PathSeparator)) {
if sub == existing || strings.HasPrefix(sub, existing+string(fs.PathSeparator)) {
continue next
}
}
@ -2666,57 +2660,3 @@ func (s folderDeviceSet) sortedDevices(folder string) []protocol.DeviceID {
sort.Sort(protocol.DeviceIDs(devs))
return devs
}
// rootedJoinedPath takes a root and a supposedly relative path inside that
// root and returns the joined path. An error is returned if the joined path
// is not in fact inside the root.
func rootedJoinedPath(root, rel string) (string, error) {
// The root must not be empty.
if root == "" {
return "", errInvalidFilename
}
pathSep := string(os.PathSeparator)
// The expected prefix for the resulting path is the root, with a path
// separator at the end.
expectedPrefix := filepath.FromSlash(root)
if !strings.HasSuffix(expectedPrefix, pathSep) {
expectedPrefix += pathSep
}
// The relative path should be clean from internal dotdots and similar
// funkyness.
rel = filepath.FromSlash(rel)
if filepath.Clean(rel) != rel {
return "", errInvalidFilename
}
// It is not acceptable to attempt to traverse upwards or refer to the
// root itself.
switch rel {
case ".", "..", pathSep:
return "", errNotRelative
}
if strings.HasPrefix(rel, ".."+pathSep) {
return "", errNotRelative
}
if strings.HasPrefix(rel, pathSep+pathSep) {
// The relative path may pretend to be an absolute path within the
// root, but the double path separator on Windows implies something
// else. It would get cleaned by the Join below, but it's out of
// spec anyway.
return "", errNotRelative
}
// The supposedly correct path is the one filepath.Join will return, as
// it does cleaning and so on. Check that one first to make sure no
// obvious escape attempts have been made.
joined := filepath.Join(root, rel)
if !strings.HasPrefix(joined, expectedPrefix) {
return "", errNotRelative
}
return joined, nil
}

247
lib/model/model_test.go
View File

@ -25,8 +25,8 @@ import (
"github.com/d4l3k/messagediff"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
srand "github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/scanner"
@ -35,12 +35,14 @@ import (
var device1, device2 protocol.DeviceID
var defaultConfig *config.Wrapper
var defaultFolderConfig config.FolderConfiguration
var defaultFs fs.Filesystem
func init() {
device1, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
device2, _ = protocol.DeviceIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
defaultFs = fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata")
defaultFolderConfig = config.NewFolderConfiguration("default", "testdata")
defaultFolderConfig = config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "testdata")
defaultFolderConfig.Devices = []config.FolderDeviceConfiguration{{DeviceID: device1}}
_defaultConfig := config.Configuration{
Folders: []config.FolderConfiguration{defaultFolderConfig},
@ -513,14 +515,16 @@ func TestClusterConfig(t *testing.T) {
}
cfg.Folders = []config.FolderConfiguration{
{
ID: "folder1",
ID: "folder1",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2},
},
},
{
ID: "folder2",
ID: "folder2",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2},
@ -622,13 +626,15 @@ func TestIntroducer(t *testing.T) {
},
Folders: []config.FolderConfiguration{
{
ID: "folder1",
ID: "folder1",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
},
},
{
ID: "folder2",
ID: "folder2",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
},
@ -671,14 +677,16 @@ func TestIntroducer(t *testing.T) {
},
Folders: []config.FolderConfiguration{
{
ID: "folder1",
ID: "folder1",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2, IntroducedBy: device1},
},
},
{
ID: "folder2",
ID: "folder2",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
},
@ -726,14 +734,16 @@ func TestIntroducer(t *testing.T) {
},
Folders: []config.FolderConfiguration{
{
ID: "folder1",
ID: "folder1",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2, IntroducedBy: device1},
},
},
{
ID: "folder2",
ID: "folder2",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2, IntroducedBy: device1},
@ -771,14 +781,16 @@ func TestIntroducer(t *testing.T) {
},
Folders: []config.FolderConfiguration{
{
ID: "folder1",
ID: "folder1",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2, IntroducedBy: device1},
},
},
{
ID: "folder2",
ID: "folder2",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2, IntroducedBy: device1},
@ -816,14 +828,16 @@ func TestIntroducer(t *testing.T) {
},
Folders: []config.FolderConfiguration{
{
ID: "folder1",
ID: "folder1",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2, IntroducedBy: device1},
},
},
{
ID: "folder2",
ID: "folder2",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
},
@ -872,14 +886,16 @@ func TestIntroducer(t *testing.T) {
},
Folders: []config.FolderConfiguration{
{
ID: "folder1",
ID: "folder1",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2, IntroducedBy: device1},
},
},
{
ID: "folder2",
ID: "folder2",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2},
@ -916,14 +932,16 @@ func TestIntroducer(t *testing.T) {
},
Folders: []config.FolderConfiguration{
{
ID: "folder1",
ID: "folder1",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2, IntroducedBy: device1},
},
},
{
ID: "folder2",
ID: "folder2",
Path: "testdata",
Devices: []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2, IntroducedBy: protocol.LocalDeviceID},
@ -1026,7 +1044,7 @@ func TestIgnores(t *testing.T) {
// because we will be changing the files on disk often enough that the
// mtimes will be unreliable to determine change status.
m.fmut.Lock()
m.folderIgnores["default"] = ignore.New(ignore.WithCache(true), ignore.WithChangeDetector(newAlwaysChanged()))
m.folderIgnores["default"] = ignore.New(defaultFs, ignore.WithCache(true), ignore.WithChangeDetector(newAlwaysChanged()))
m.fmut.Unlock()
// Make sure the initial scan has finished (ScanFolders is blocking)
@ -1050,7 +1068,7 @@ func TestIgnores(t *testing.T) {
}
// Invalid path, marker should be missing, hence returns an error.
m.AddFolder(config.FolderConfiguration{ID: "fresh", RawPath: "XXX"})
m.AddFolder(config.FolderConfiguration{ID: "fresh", Path: "XXX"})
_, _, err = m.GetIgnores("fresh")
if err == nil {
t.Error("No error")
@ -1069,14 +1087,14 @@ func TestIgnores(t *testing.T) {
func TestROScanRecovery(t *testing.T) {
ldb := db.OpenMemory()
set := db.NewFileSet("default", ldb)
set := db.NewFileSet("default", defaultFs, ldb)
set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
{Name: "dummyfile"},
})
fcfg := config.FolderConfiguration{
ID: "default",
RawPath: "testdata/rotestfolder",
Path: "testdata/rotestfolder",
Type: config.FolderTypeSendOnly,
RescanIntervalS: 1,
}
@ -1089,7 +1107,7 @@ func TestROScanRecovery(t *testing.T) {
},
})
os.RemoveAll(fcfg.RawPath)
os.RemoveAll(fcfg.Path)
m := NewModel(cfg, protocol.LocalDeviceID, "syncthing", "dev", ldb, nil)
m.AddFolder(fcfg)
@ -1120,14 +1138,14 @@ func TestROScanRecovery(t *testing.T) {
return
}
os.Mkdir(fcfg.RawPath, 0700)
os.Mkdir(fcfg.Path, 0700)
if err := waitFor("folder marker missing"); err != nil {
t.Error(err)
return
}
fd, err := os.Create(filepath.Join(fcfg.RawPath, ".stfolder"))
fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder"))
if err != nil {
t.Error(err)
return
@ -1139,14 +1157,14 @@ func TestROScanRecovery(t *testing.T) {
return
}
os.Remove(filepath.Join(fcfg.RawPath, ".stfolder"))
os.Remove(filepath.Join(fcfg.Path, ".stfolder"))
if err := waitFor("folder marker missing"); err != nil {
t.Error(err)
return
}
os.Remove(fcfg.RawPath)
os.Remove(fcfg.Path)
if err := waitFor("folder path missing"); err != nil {
t.Error(err)
@ -1156,14 +1174,14 @@ func TestROScanRecovery(t *testing.T) {
func TestRWScanRecovery(t *testing.T) {
ldb := db.OpenMemory()
set := db.NewFileSet("default", ldb)
set := db.NewFileSet("default", defaultFs, ldb)
set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
{Name: "dummyfile"},
})
fcfg := config.FolderConfiguration{
ID: "default",
RawPath: "testdata/rwtestfolder",
Path: "testdata/rwtestfolder",
Type: config.FolderTypeSendReceive,
RescanIntervalS: 1,
}
@ -1176,7 +1194,7 @@ func TestRWScanRecovery(t *testing.T) {
},
})
os.RemoveAll(fcfg.RawPath)
os.RemoveAll(fcfg.Path)
m := NewModel(cfg, protocol.LocalDeviceID, "syncthing", "dev", ldb, nil)
m.AddFolder(fcfg)
@ -1207,14 +1225,14 @@ func TestRWScanRecovery(t *testing.T) {
return
}
os.Mkdir(fcfg.RawPath, 0700)
os.Mkdir(fcfg.Path, 0700)
if err := waitFor("folder marker missing"); err != nil {
t.Error(err)
return
}
fd, err := os.Create(filepath.Join(fcfg.RawPath, ".stfolder"))
fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder"))
if err != nil {
t.Error(err)
return
@ -1226,14 +1244,14 @@ func TestRWScanRecovery(t *testing.T) {
return
}
os.Remove(filepath.Join(fcfg.RawPath, ".stfolder"))
os.Remove(filepath.Join(fcfg.Path, ".stfolder"))
if err := waitFor("folder marker missing"); err != nil {
t.Error(err)
return
}
os.Remove(fcfg.RawPath)
os.Remove(fcfg.Path)
if err := waitFor("folder path missing"); err != nil {
t.Error(err)
@ -1861,14 +1879,14 @@ func TestIssue3164(t *testing.T) {
f := protocol.FileInfo{
Name: "issue3164",
}
m := ignore.New()
m := ignore.New(defaultFs)
if err := m.Parse(bytes.NewBufferString("(?d)oktodelete"), ""); err != nil {
t.Fatal(err)
}
fl := sendReceiveFolder{
dbUpdates: make(chan dbUpdateJob, 1),
dir: "testdata",
fs: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
}
fl.deleteDir(f, m)
@ -1955,7 +1973,7 @@ func TestIssue2782(t *testing.T) {
if err := os.RemoveAll(testDir); err != nil {
t.Skip(err)
}
if err := osutil.MkdirAll(testDir+"/syncdir", 0755); err != nil {
if err := os.MkdirAll(testDir+"/syncdir", 0755); err != nil {
t.Skip(err)
}
if err := ioutil.WriteFile(testDir+"/syncdir/file", []byte("hello, world\n"), 0644); err != nil {
@ -1968,7 +1986,7 @@ func TestIssue2782(t *testing.T) {
db := db.OpenMemory()
m := NewModel(defaultConfig, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
m.AddFolder(config.NewFolderConfiguration("default", "~/"+testName+"/synclink/"))
m.AddFolder(config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "~/"+testName+"/synclink/"))
m.StartFolder("default")
m.ServeBackground()
defer m.Stop()
@ -1985,7 +2003,7 @@ func TestIssue2782(t *testing.T) {
func TestIndexesForUnknownDevicesDropped(t *testing.T) {
dbi := db.OpenMemory()
files := db.NewFileSet("default", dbi)
files := db.NewFileSet("default", defaultFs, dbi)
files.Replace(device1, genFiles(1))
files.Replace(device2, genFiles(1))
@ -1998,7 +2016,7 @@ func TestIndexesForUnknownDevicesDropped(t *testing.T) {
m.StartFolder("default")
// Remote sequence is cached, hence need to recreated.
files = db.NewFileSet("default", dbi)
files = db.NewFileSet("default", defaultFs, dbi)
if len(files.ListDevices()) != 1 {
t.Error("Expected one device")
@ -2008,7 +2026,7 @@ func TestIndexesForUnknownDevicesDropped(t *testing.T) {
func TestSharedWithClearedOnDisconnect(t *testing.T) {
dbi := db.OpenMemory()
fcfg := config.NewFolderConfiguration("default", "testdata")
fcfg := config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "testdata")
fcfg.Devices = []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2},
@ -2247,7 +2265,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
dbi := db.OpenMemory()
fcfg := config.NewFolderConfiguration("default", "testdata")
fcfg := config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "testdata")
fcfg.Devices = []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2},
@ -2335,151 +2353,6 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
}
}
func TestRootedJoinedPath(t *testing.T) {
type testcase struct {
root string
rel string
joined string
ok bool
}
cases := []testcase{
// Valid cases
{"foo", "bar", "foo/bar", true},
{"foo", "/bar", "foo/bar", true},
{"foo/", "bar", "foo/bar", true},
{"foo/", "/bar", "foo/bar", true},
{"baz/foo", "bar", "baz/foo/bar", true},
{"baz/foo", "/bar", "baz/foo/bar", true},
{"baz/foo/", "bar", "baz/foo/bar", true},
{"baz/foo/", "/bar", "baz/foo/bar", true},
{"foo", "bar/baz", "foo/bar/baz", true},
{"foo", "/bar/baz", "foo/bar/baz", true},
{"foo/", "bar/baz", "foo/bar/baz", true},
{"foo/", "/bar/baz", "foo/bar/baz", true},
{"baz/foo", "bar/baz", "baz/foo/bar/baz", true},
{"baz/foo", "/bar/baz", "baz/foo/bar/baz", true},
{"baz/foo/", "bar/baz", "baz/foo/bar/baz", true},
{"baz/foo/", "/bar/baz", "baz/foo/bar/baz", true},
// Not escape attempts, but oddly formatted relative paths. Disallowed.
{"foo", "./bar", "", false},
{"baz/foo", "./bar", "", false},
{"foo", "./bar/baz", "", false},
{"baz/foo", "./bar/baz", "", false},
{"baz/foo", "bar/../baz", "", false},
{"baz/foo", "/bar/../baz", "", false},
{"baz/foo", "./bar/../baz", "", false},
{"baz/foo", "bar/../baz", "", false},
{"baz/foo", "/bar/../baz", "", false},
{"baz/foo", "./bar/../baz", "", false},
// Results in an allowed path, but does it by probing. Disallowed.
{"foo", "../foo", "", false},
{"foo", "../foo/bar", "", false},
{"baz/foo", "../foo/bar", "", false},
{"baz/foo", "../../baz/foo/bar", "", false},
{"baz/foo", "bar/../../foo/bar", "", false},
{"baz/foo", "bar/../../../baz/foo/bar", "", false},
// Escape attempts.
{"foo", "", "", false},
{"foo", "/", "", false},
{"foo", "..", "", false},
{"foo", "/..", "", false},
{"foo", "../", "", false},
{"foo", "../bar", "", false},
{"foo", "../foobar", "", false},
{"foo/", "../bar", "", false},
{"foo/", "../foobar", "", false},
{"baz/foo", "../bar", "", false},
{"baz/foo", "../foobar", "", false},
{"baz/foo/", "../bar", "", false},
{"baz/foo/", "../foobar", "", false},
{"baz/foo/", "bar/../../quux/baz", "", false},
// Empty root is a misconfiguration.
{"", "/foo", "", false},
{"", "foo", "", false},
{"", ".", "", false},
{"", "..", "", false},
{"", "/", "", false},
{"", "", "", false},
// Root=/ is valid, and things should be verified as usual.
{"/", "foo", "/foo", true},
{"/", "/foo", "/foo", true},
{"/", "../foo", "", false},
{"/", ".", "", false},
{"/", "..", "", false},
{"/", "/", "", false},
{"/", "", "", false},
}
if runtime.GOOS == "windows" {
extraCases := []testcase{
{`c:\`, `foo`, `c:\foo`, true},
{`\\?\c:\`, `foo`, `\\?\c:\foo`, true},
{`c:\`, `\foo`, `c:\foo`, true},
{`\\?\c:\`, `\foo`, `\\?\c:\foo`, true},
{`c:\`, `\\foo`, ``, false},
{`c:\`, ``, ``, false},
{`c:\`, `.`, ``, false},
{`c:\`, `\`, ``, false},
{`\\?\c:\`, `\\foo`, ``, false},
{`\\?\c:\`, ``, ``, false},
{`\\?\c:\`, `.`, ``, false},
{`\\?\c:\`, `\`, ``, false},
// makes no sense, but will be treated simply as a bad filename
{`c:\foo`, `d:\bar`, `c:\foo\d:\bar`, true},
}
for _, tc := range cases {
// Add case where root is backslashed, rel is forward slashed
extraCases = append(extraCases, testcase{
root: filepath.FromSlash(tc.root),
rel: tc.rel,
joined: tc.joined,
ok: tc.ok,
})
// and the opposite
extraCases = append(extraCases, testcase{
root: tc.root,
rel: filepath.FromSlash(tc.rel),
joined: tc.joined,
ok: tc.ok,
})
// and both backslashed
extraCases = append(extraCases, testcase{
root: filepath.FromSlash(tc.root),
rel: filepath.FromSlash(tc.rel),
joined: tc.joined,
ok: tc.ok,
})
}
cases = append(cases, extraCases...)
}
for _, tc := range cases {
res, err := rootedJoinedPath(tc.root, tc.rel)
if tc.ok {
if err != nil {
t.Errorf("Unexpected error for rootedJoinedPath(%q, %q): %v", tc.root, tc.rel, err)
continue
}
exp := filepath.FromSlash(tc.joined)
if res != exp {
t.Errorf("Unexpected result for rootedJoinedPath(%q, %q): %q != expected %q", tc.root, tc.rel, res, exp)
}
} else if err == nil {
t.Errorf("Unexpected pass for rootedJoinedPath(%q, %q) => %q", tc.root, tc.rel, res)
continue
}
}
}
func addFakeConn(m *Model, dev protocol.DeviceID) *fakeConnection {
fc := &fakeConnection{id: dev, model: m}
m.AddConnection(fc, protocol.HelloResult{})

5
lib/model/requests_test.go
View File

@ -18,6 +18,7 @@ import (
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
@ -214,7 +215,7 @@ func TestRequestVersioningSymlinkAttack(t *testing.T) {
// deleted symlink to escape
cfg := defaultConfig.RawCopy()
cfg.Folders[0] = config.NewFolderConfiguration("default", "_tmpfolder")
cfg.Folders[0] = config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "_tmpfolder")
cfg.Folders[0].PullerSleepS = 1
cfg.Folders[0].Devices = []config.FolderDeviceConfiguration{
{DeviceID: device1},
@ -287,7 +288,7 @@ func TestRequestVersioningSymlinkAttack(t *testing.T) {
func setupModelWithConnection() (*Model, *fakeConnection) {
cfg := defaultConfig.RawCopy()
cfg.Folders[0] = config.NewFolderConfiguration("default", "_tmpfolder")
cfg.Folders[0] = config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, "_tmpfolder")
cfg.Folders[0].PullerSleepS = 1
cfg.Folders[0].Devices = []config.FolderDeviceConfiguration{
{DeviceID: device1},

2
lib/model/rofolder.go
View File

@ -24,7 +24,7 @@ type sendOnlyFolder struct {
config.FolderConfiguration
}
func newSendOnlyFolder(model *Model, cfg config.FolderConfiguration, _ versioner.Versioner, _ *fs.MtimeFS) service {
func newSendOnlyFolder(model *Model, cfg config.FolderConfiguration, _ versioner.Versioner, _ fs.Filesystem) service {
ctx, cancel := context.WithCancel(context.Background())
return &sendOnlyFolder{

278
lib/model/rwfolder.go
View File

@ -11,7 +11,6 @@ import (
"errors"
"fmt"
"math/rand"
"os"
"path/filepath"
"runtime"
"sort"
@ -51,7 +50,7 @@ type copyBlocksState struct {
}
// Which filemode bits to preserve
const retainBits = os.ModeSetgid | os.ModeSetuid | os.ModeSticky
const retainBits = fs.ModeSetgid | fs.ModeSetuid | fs.ModeSticky
var (
activity = newDeviceActivity()
@ -84,8 +83,7 @@ type sendReceiveFolder struct {
folder
config.FolderConfiguration
mtimeFS *fs.MtimeFS
dir string
fs fs.Filesystem
versioner versioner.Versioner
sleep time.Duration
pause time.Duration
@ -99,7 +97,7 @@ type sendReceiveFolder struct {
errorsMut sync.Mutex
}
func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, mtimeFS *fs.MtimeFS) service {
func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service {
ctx, cancel := context.WithCancel(context.Background())
f := &sendReceiveFolder{
@ -113,8 +111,7 @@ func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver vers
},
FolderConfiguration: cfg,
mtimeFS: mtimeFS,
dir: cfg.Path(),
fs: fs,
versioner: ver,
queue: newJobQueue(),
@ -434,7 +431,7 @@ func (f *sendReceiveFolder) pullerIteration(ignores *ignore.Matcher) int {
for _, fi := range processDirectly {
// Verify that the thing we are handling lives inside a directory,
// and not a symlink or empty space.
if err := osutil.TraversesSymlink(f.dir, filepath.Dir(fi.Name)); err != nil {
if err := osutil.TraversesSymlink(f.fs, filepath.Dir(fi.Name)); err != nil {
f.newError(fi.Name, err)
continue
}
@ -523,7 +520,7 @@ nextFile:
// Verify that the thing we are handling lives inside a directory,
// and not a symlink or empty space.
if err := osutil.TraversesSymlink(f.dir, filepath.Dir(fi.Name)); err != nil {
if err := osutil.TraversesSymlink(f.fs, filepath.Dir(fi.Name)); err != nil {
f.newError(fi.Name, err)
continue
}
@ -610,12 +607,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo) {
})
}()
realName, err := rootedJoinedPath(f.dir, file.Name)
if err != nil {
f.newError(file.Name, err)
return
}
mode := os.FileMode(file.Permissions & 0777)
mode := fs.FileMode(file.Permissions & 0777)
if f.ignorePermissions(file) {
mode = 0777
}
@ -625,13 +617,13 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo) {
l.Debugf("need dir\n\t%v\n\t%v", file, curFile)
}
info, err := f.mtimeFS.Lstat(realName)
info, err := f.fs.Lstat(file.Name)
switch {
// There is already something under that name, but it's a file/link.
// Most likely a file/link is getting replaced with a directory.
// Remove the file/link and fall through to directory creation.
case err == nil && (!info.IsDir() || info.IsSymlink()):
err = osutil.InWritableDir(os.Remove, realName)
err = osutil.InWritableDir(f.fs.Remove, f.fs, file.Name)
if err != nil {
l.Infof("Puller (folder %q, dir %q): %v", f.folderID, file.Name, err)
f.newError(file.Name, err)
@ -640,28 +632,28 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo) {
fallthrough
// The directory doesn't exist, so we create it with the right
// mode bits from the start.
case err != nil && os.IsNotExist(err):
case err != nil && fs.IsNotExist(err):
// We declare a function that acts on only the path name, so
// we can pass it to InWritableDir. We use a regular Mkdir and
// not MkdirAll because the parent should already exist.
mkdir := func(path string) error {
err = os.Mkdir(path, mode)
err = f.fs.Mkdir(path, mode)
if err != nil || f.ignorePermissions(file) {
return err
}
// Stat the directory so we can check its permissions.
info, err := f.mtimeFS.Lstat(path)
info, err := f.fs.Lstat(path)
if err != nil {
return err
}
// Mask for the bits we want to preserve and add them in to the
// directories permissions.
return os.Chmod(path, mode|(os.FileMode(info.Mode())&retainBits))
return f.fs.Chmod(path, mode|(info.Mode()&retainBits))
}
if err = osutil.InWritableDir(mkdir, realName); err == nil {
if err = osutil.InWritableDir(mkdir, f.fs, file.Name); err == nil {
f.dbUpdates <- dbUpdateJob{file, dbUpdateHandleDir}
} else {
l.Infof("Puller (folder %q, dir %q): %v", f.folderID, file.Name, err)
@ -681,7 +673,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo) {
// It's OK to change mode bits on stuff within non-writable directories.
if f.ignorePermissions(file) {
f.dbUpdates <- dbUpdateJob{file, dbUpdateHandleDir}
} else if err := os.Chmod(realName, mode|(os.FileMode(info.Mode())&retainBits)); err == nil {
} else if err := f.fs.Chmod(file.Name, mode|(fs.FileMode(info.Mode())&retainBits)); err == nil {
f.dbUpdates <- dbUpdateJob{file, dbUpdateHandleDir}
} else {
l.Infof("Puller (folder %q, dir %q): %v", f.folderID, file.Name, err)
@ -712,12 +704,6 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo) {
})
}()
realName, err := rootedJoinedPath(f.dir, file.Name)
if err != nil {
f.newError(file.Name, err)
return
}
if shouldDebug() {
curFile, _ := f.model.CurrentFolderFile(f.folderID, file.Name)
l.Debugf("need symlink\n\t%v\n\t%v", file, curFile)
@ -732,11 +718,11 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo) {
return
}
if _, err = f.mtimeFS.Lstat(realName); err == nil {
if _, err = f.fs.Lstat(file.Name); err == nil {
// There is already something under that name. Remove it to replace
// with the symlink. This also handles the "change symlink type"
// path.
err = osutil.InWritableDir(os.Remove, realName)
err = osutil.InWritableDir(f.fs.Remove, f.fs, file.Name)
if err != nil {
l.Infof("Puller (folder %q, dir %q): %v", f.folderID, file.Name, err)
f.newError(file.Name, err)
@ -747,10 +733,10 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo) {
// We declare a function that acts on only the path name, so
// we can pass it to InWritableDir.
createLink := func(path string) error {
return os.Symlink(file.SymlinkTarget, path)
return f.fs.CreateSymlink(file.SymlinkTarget, path)
}
if err = osutil.InWritableDir(createLink, realName); err == nil {
if err = osutil.InWritableDir(createLink, f.fs, file.Name); err == nil {
f.dbUpdates <- dbUpdateJob{file, dbUpdateHandleSymlink}
} else {
l.Infof("Puller (folder %q, dir %q): %v", f.folderID, file.Name, err)
@ -781,31 +767,21 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, matcher *ignore.Ma
})
}()
realName, err := rootedJoinedPath(f.dir, file.Name)
if err != nil {
f.newError(file.Name, err)
return
}
// Delete any temporary files lying around in the directory
dir, _ := os.Open(realName)
if dir != nil {
files, _ := dir.Readdirnames(-1)
for _, dirFile := range files {
fullDirFile := filepath.Join(file.Name, dirFile)
if ignore.IsTemporary(dirFile) || (matcher != nil &&
matcher.Match(fullDirFile).IsDeletable()) {
os.RemoveAll(filepath.Join(f.dir, fullDirFile))
}
files, _ := f.fs.DirNames(file.Name)
for _, dirFile := range files {
fullDirFile := filepath.Join(file.Name, dirFile)
if ignore.IsTemporary(dirFile) || (matcher != nil && matcher.Match(fullDirFile).IsDeletable()) {
f.fs.RemoveAll(fullDirFile)
}
dir.Close()
}
err = osutil.InWritableDir(os.Remove, realName)
if err == nil || os.IsNotExist(err) {
err = osutil.InWritableDir(f.fs.Remove, f.fs, file.Name)
if err == nil || fs.IsNotExist(err) {
// It was removed or it doesn't exist to start with
f.dbUpdates <- dbUpdateJob{file, dbUpdateDeleteDir}
} else if _, serr := f.mtimeFS.Lstat(realName); serr != nil && !os.IsPermission(serr) {
} else if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) {
// We get an error just looking at the directory, and it's not a
// permission problem. Lets assume the error is in fact some variant
// of "file does not exist" (possibly expressed as some parent being a
@ -840,12 +816,6 @@ func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo) {
})
}()
realName, err := rootedJoinedPath(f.dir, file.Name)
if err != nil {
f.newError(file.Name, err)
return
}
cur, ok := f.model.CurrentFolderFile(f.folderID, file.Name)
if ok && f.inConflict(cur.Version, file.Version) {
// There is a conflict here. Move the file to a conflict copy instead
@ -854,17 +824,17 @@ func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo) {
file.Version = file.Version.Merge(cur.Version)
err = osutil.InWritableDir(func(name string) error {
return f.moveForConflict(name, file.ModifiedBy.String())
}, realName)
}, f.fs, file.Name)
} else if f.versioner != nil && !cur.IsSymlink() {
err = osutil.InWritableDir(f.versioner.Archive, realName)
err = osutil.InWritableDir(f.versioner.Archive, f.fs, file.Name)
} else {
err = osutil.InWritableDir(os.Remove, realName)
err = osutil.InWritableDir(f.fs.Remove, f.fs, file.Name)
}
if err == nil || os.IsNotExist(err) {
if err == nil || fs.IsNotExist(err) {
// It was removed or it doesn't exist to start with
f.dbUpdates <- dbUpdateJob{file, dbUpdateDeleteFile}
} else if _, serr := f.mtimeFS.Lstat(realName); serr != nil && !os.IsPermission(serr) {
} else if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) {
// We get an error just looking at the file, and it's not a permission
// problem. Lets assume the error is in fact some variant of "file
// does not exist" (possibly expressed as some parent being a file and
@ -915,24 +885,13 @@ func (f *sendReceiveFolder) renameFile(source, target protocol.FileInfo) {
l.Debugln(f, "taking rename shortcut", source.Name, "->", target.Name)
from, err := rootedJoinedPath(f.dir, source.Name)
if err != nil {
f.newError(source.Name, err)
return
}
to, err := rootedJoinedPath(f.dir, target.Name)
if err != nil {
f.newError(target.Name, err)
return
}
if f.versioner != nil {
err = osutil.Copy(from, to)
err = osutil.Copy(f.fs, source.Name, target.Name)
if err == nil {
err = osutil.InWritableDir(f.versioner.Archive, from)
err = osutil.InWritableDir(f.versioner.Archive, f.fs, source.Name)
}
} else {
err = osutil.TryRename(from, to)
err = osutil.TryRename(f.fs, source.Name, target.Name)
}
if err == nil {
@ -955,7 +914,7 @@ func (f *sendReceiveFolder) renameFile(source, target protocol.FileInfo) {
// get rid of. Attempt to delete it instead so that we make *some*
// progress. The target is unhandled.
err = osutil.InWritableDir(os.Remove, from)
err = osutil.InWritableDir(f.fs.Remove, f.fs, source.Name)
if err != nil {
l.Infof("Puller (folder %q, file %q): delete %q after failed rename: %v", f.folderID, target.Name, source.Name, err)
f.newError(target.Name, err)
@ -1041,26 +1000,16 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
return
}
// Figure out the absolute filenames we need once and for all
tempName, err := rootedJoinedPath(f.dir, ignore.TempName(file.Name))
if err != nil {
f.newError(file.Name, err)
return
}
realName, err := rootedJoinedPath(f.dir, file.Name)
if err != nil {
f.newError(file.Name, err)
return
}
tempName := ignore.TempName(file.Name)
if hasCurFile && !curFile.IsDirectory() && !curFile.IsSymlink() {
// Check that the file on disk is what we expect it to be according to
// the database. If there's a mismatch here, there might be local
// changes that we don't know about yet and we should scan before
// touching the file. If we can't stat the file we'll just pull it.
if info, err := f.mtimeFS.Lstat(realName); err == nil {
if info, err := f.fs.Lstat(file.Name); err == nil {
if !info.ModTime().Equal(curFile.ModTime()) || info.Size() != curFile.Size {
l.Debugln("file modified but not rescanned; not pulling:", realName)
l.Debugln("file modified but not rescanned; not pulling:", file.Name)
// Scan() is synchronous (i.e. blocks until the scan is
// completed and returns an error), but a scan can't happen
// while we're in the puller routine. Request the scan in the
@ -1082,7 +1031,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
// Check for an old temporary file which might have some blocks we could
// reuse.
tempBlocks, err := scanner.HashFile(f.ctx, fs.DefaultFilesystem, tempName, protocol.BlockSize, nil, false)
tempBlocks, err := scanner.HashFile(f.ctx, f.fs, tempName, protocol.BlockSize, nil, false)
if err == nil {
// Check for any reusable blocks in the temp file
tempCopyBlocks, _ := scanner.BlockDiff(tempBlocks, file.Blocks)
@ -1110,7 +1059,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
// Otherwise, discard the file ourselves in order for the
// sharedpuller not to panic when it fails to exclusively create a
// file which already exists
osutil.InWritableDir(os.Remove, tempName)
osutil.InWritableDir(f.fs.Remove, f.fs, tempName)
}
} else {
// Copy the blocks, as we don't want to shuffle them on the FileInfo
@ -1119,8 +1068,8 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
}
if f.MinDiskFree.BaseValue() > 0 {
if free, err := osutil.DiskFreeBytes(f.dir); err == nil && free < blocksSize {
l.Warnf(`Folder "%s": insufficient disk space in %s for %s: have %.2f MiB, need %.2f MiB`, f.folderID, f.dir, file.Name, float64(free)/1024/1024, float64(blocksSize)/1024/1024)
if usage, err := f.fs.Usage("."); err == nil && usage.Free < blocksSize {
l.Warnf(`Folder "%s": insufficient disk space in %s for %s: have %.2f MiB, need %.2f MiB`, f.folderID, f.fs.URI(), file.Name, float64(usage.Free)/1024/1024, float64(blocksSize)/1024/1024)
f.newError(file.Name, errors.New("insufficient space"))
return
}
@ -1141,9 +1090,10 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
s := sharedPullerState{
file: file,
fs: f.fs,
folder: f.folderID,
tempName: tempName,
realName: realName,
realName: file.Name,
copyTotal: len(blocks),
copyNeeded: len(blocks),
reused: len(reused),
@ -1170,20 +1120,15 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
// shortcutFile sets file mode and modification time, when that's the only
// thing that has changed.
func (f *sendReceiveFolder) shortcutFile(file protocol.FileInfo) error {
realName, err := rootedJoinedPath(f.dir, file.Name)
if err != nil {
f.newError(file.Name, err)
return err
}
if !f.ignorePermissions(file) {
if err := os.Chmod(realName, os.FileMode(file.Permissions&0777)); err != nil {
if err := f.fs.Chmod(file.Name, fs.FileMode(file.Permissions&0777)); err != nil {
l.Infof("Puller (folder %q, file %q): shortcut: chmod: %v", f.folderID, file.Name, err)
f.newError(file.Name, err)
return err
}
}
f.mtimeFS.Chtimes(realName, file.ModTime(), file.ModTime()) // never fails
f.fs.Chtimes(file.Name, file.ModTime(), file.ModTime()) // never fails
// This may have been a conflict. We should merge the version vectors so
// that our clock doesn't move backwards.
@ -1211,15 +1156,16 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
f.model.progressEmitter.Register(state.sharedPullerState)
}
folderRoots := make(map[string]string)
folderFilesystems := make(map[string]fs.Filesystem)
var folders []string
f.model.fmut.RLock()
for folder, cfg := range f.model.folderCfgs {
folderRoots[folder] = cfg.Path()
folderFilesystems[folder] = cfg.Filesystem()
folders = append(folders, folder)
}
f.model.fmut.RUnlock()
var file fs.File
var weakHashFinder *weakhash.Finder
if weakhash.Enabled {
@ -1237,9 +1183,12 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
}
if len(hashesToFind) > 0 {
weakHashFinder, err = weakhash.NewFinder(state.realName, protocol.BlockSize, hashesToFind)
if err != nil {
l.Debugln("weak hasher", err)
file, err = f.fs.Open(state.file.Name)
if err == nil {
weakHashFinder, err = weakhash.NewFinder(file, protocol.BlockSize, hashesToFind)
if err != nil {
l.Debugln("weak hasher", err)
}
}
} else {
l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name)
@ -1289,12 +1238,9 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
}
if !found {
found = f.model.finder.Iterate(folders, block.Hash, func(folder, file string, index int32) bool {
inFile, err := rootedJoinedPath(folderRoots[folder], file)
if err != nil {
return false
}
fd, err := os.Open(inFile)
found = f.model.finder.Iterate(folders, block.Hash, func(folder, path string, index int32) bool {
fs := folderFilesystems[folder]
fd, err := fs.Open(path)
if err != nil {
return false
}
@ -1308,8 +1254,8 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
hash, err := scanner.VerifyBuffer(buf, block)
if err != nil {
if hash != nil {
l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, file, index, block.Hash, hash)
err = f.model.finder.Fix(folder, file, index, block.Hash, hash)
l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, path, index, block.Hash, hash)
err = f.model.finder.Fix(folder, path, index, block.Hash, hash)
if err != nil {
l.Warnln("finder fix:", err)
}
@ -1323,7 +1269,7 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
if err != nil {
state.fail("dst write", err)
}
if file == state.file.Name {
if path == state.file.Name {
state.copiedFromOrigin()
}
return true
@ -1345,7 +1291,12 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
state.copyDone(block)
}
}
weakHashFinder.Close()
if file != nil {
// os.File used to return invalid argument if nil.
// fs.File panics as it's an interface.
file.Close()
}
out <- state.sharedPullerState
}
}
@ -1426,12 +1377,12 @@ func (f *sendReceiveFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *
func (f *sendReceiveFolder) performFinish(state *sharedPullerState) error {
// Set the correct permission bits on the new file
if !f.ignorePermissions(state.file) {
if err := os.Chmod(state.tempName, os.FileMode(state.file.Permissions&0777)); err != nil {
if err := f.fs.Chmod(state.tempName, fs.FileMode(state.file.Permissions&0777)); err != nil {
return err
}
}
if stat, err := f.mtimeFS.Lstat(state.realName); err == nil {
if stat, err := f.fs.Lstat(state.file.Name); err == nil {
// There is an old file or directory already in place. We need to
// handle that.
@ -1445,7 +1396,7 @@ func (f *sendReceiveFolder) performFinish(state *sharedPullerState) error {
// and future hard ignores before attempting a directory delete.
// Should share code with f.deletDir().
if err = osutil.InWritableDir(os.Remove, state.realName); err != nil {
if err = osutil.InWritableDir(f.fs.Remove, f.fs, state.file.Name); err != nil {
return err
}
@ -1458,7 +1409,7 @@ func (f *sendReceiveFolder) performFinish(state *sharedPullerState) error {
state.file.Version = state.file.Version.Merge(state.version)
err = osutil.InWritableDir(func(name string) error {
return f.moveForConflict(name, state.file.ModifiedBy.String())
}, state.realName)
}, f.fs, state.file.Name)
if err != nil {
return err
}
@ -1468,7 +1419,7 @@ func (f *sendReceiveFolder) performFinish(state *sharedPullerState) error {
// file before we replace it. Archiving a non-existent file is not
// an error.
if err = f.versioner.Archive(state.realName); err != nil {
if err = f.versioner.Archive(state.file.Name); err != nil {
return err
}
}
@ -1476,12 +1427,12 @@ func (f *sendReceiveFolder) performFinish(state *sharedPullerState) error {
// Replace the original content with the new one. If it didn't work,
// leave the temp file in place for reuse.
if err := osutil.TryRename(state.tempName, state.realName); err != nil {
if err := osutil.TryRename(f.fs, state.tempName, state.file.Name); err != nil {
return err
}
// Set the correct timestamp on the new file
f.mtimeFS.Chtimes(state.realName, state.file.ModTime(), state.file.ModTime()) // never fails
f.fs.Chtimes(state.file.Name, state.file.ModTime(), state.file.ModTime()) // never fails
// Record the updated file in the index
f.dbUpdates <- dbUpdateJob{state.file, dbUpdateHandleFile}
@ -1540,26 +1491,7 @@ func (f *sendReceiveFolder) dbUpdaterRoutine() {
tick := time.NewTicker(maxBatchTime)
defer tick.Stop()
var changedFiles []string
var changedDirs []string
if f.Fsync {
changedFiles = make([]string, 0, maxBatchSize)
changedDirs = make([]string, 0, maxBatchSize)
}
syncFilesOnce := func(files []string, syncFn func(string) error) {
sort.Strings(files)
var lastFile string
for _, file := range files {
if lastFile == file {
continue
}
lastFile = file
if err := syncFn(file); err != nil {
l.Infof("fsync %q failed: %v", file, err)
}
}
}
changedDirs := make(map[string]struct{})
handleBatch := func() {
found := false
@ -1567,20 +1499,16 @@ func (f *sendReceiveFolder) dbUpdaterRoutine() {
for _, job := range batch {
files = append(files, job.file)
if f.Fsync {
// collect changed files and dirs
switch job.jobType {
case dbUpdateHandleFile, dbUpdateShortcutFile:
changedFiles = append(changedFiles, filepath.Join(f.dir, job.file.Name))
case dbUpdateHandleDir:
changedDirs = append(changedDirs, filepath.Join(f.dir, job.file.Name))
case dbUpdateHandleSymlink:
// fsyncing symlinks is only supported by MacOS, ignore
}
if job.jobType != dbUpdateShortcutFile {
changedDirs = append(changedDirs, filepath.Dir(filepath.Join(f.dir, job.file.Name)))
}
switch job.jobType {
case dbUpdateHandleFile, dbUpdateShortcutFile:
changedDirs[filepath.Dir(job.file.Name)] = struct{}{}
case dbUpdateHandleDir:
changedDirs[job.file.Name] = struct{}{}
case dbUpdateHandleSymlink:
// fsyncing symlinks is only supported by MacOS, ignore
}
if job.file.IsInvalid() || (job.file.IsDirectory() && !job.file.IsSymlink()) {
continue
}
@ -1593,12 +1521,18 @@ func (f *sendReceiveFolder) dbUpdaterRoutine() {
lastFile = job.file
}
if f.Fsync {
// sync files and dirs to disk
syncFilesOnce(changedFiles, osutil.SyncFile)
changedFiles = changedFiles[:0]
syncFilesOnce(changedDirs, osutil.SyncDir)
changedDirs = changedDirs[:0]
// sync directories
for dir := range changedDirs {
delete(changedDirs, dir)
fd, err := f.fs.Open(dir)
if err != nil {
l.Infof("fsync %q failed: %v", dir, err)
continue
}
if err := fd.Sync(); err != nil {
l.Infof("fsync %q failed: %v", dir, err)
}
fd.Close()
}
// All updates to file/folder objects that originated remotely
@ -1669,14 +1603,14 @@ func removeAvailability(availabilities []Availability, availability Availability
func (f *sendReceiveFolder) moveForConflict(name string, lastModBy string) error {
if strings.Contains(filepath.Base(name), ".sync-conflict-") {
l.Infoln("Conflict for", name, "which is already a conflict copy; not copying again.")
if err := os.Remove(name); err != nil && !os.IsNotExist(err) {
if err := f.fs.Remove(name); err != nil && !fs.IsNotExist(err) {
return err
}
return nil
}
if f.MaxConflicts == 0 {
if err := os.Remove(name); err != nil && !os.IsNotExist(err) {
if err := f.fs.Remove(name); err != nil && !fs.IsNotExist(err) {
return err
}
return nil
@ -1685,8 +1619,8 @@ func (f *sendReceiveFolder) moveForConflict(name string, lastModBy string) error
ext := filepath.Ext(name)
withoutExt := name[:len(name)-len(ext)]
newName := withoutExt + time.Now().Format(".sync-conflict-20060102-150405-") + lastModBy + ext
err := os.Rename(name, newName)
if os.IsNotExist(err) {
err := f.fs.Rename(name, newName)
if fs.IsNotExist(err) {
// We were supposed to move a file away but it does not exist. Either
// the user has already moved it away, or the conflict was between a
// remote modification and a local delete. In either way it does not
@ -1694,11 +1628,11 @@ func (f *sendReceiveFolder) moveForConflict(name string, lastModBy string) error
err = nil
}
if f.MaxConflicts > -1 {
matches, gerr := osutil.Glob(withoutExt + ".sync-conflict-????????-??????*" + ext)
matches, gerr := f.fs.Glob(withoutExt + ".sync-conflict-????????-??????*" + ext)
if gerr == nil && len(matches) > f.MaxConflicts {
sort.Sort(sort.Reverse(sort.StringSlice(matches)))
for _, match := range matches[f.MaxConflicts:] {
gerr = os.Remove(match)
gerr = f.fs.Remove(match)
if gerr != nil {
l.Debugln(f, "removing extra conflict", gerr)
}
@ -1772,7 +1706,7 @@ func fileValid(file db.FileIntf) error {
return errSymlinksUnsupported
case runtime.GOOS == "windows" && windowsInvalidFilename(file.FileName()):
return errInvalidFilename
return fs.ErrInvalidFilename
}
return nil
@ -1821,7 +1755,7 @@ func (l byComponentCount) Swap(a, b int) {
func componentCount(name string) int {
count := 0
for _, codepoint := range name {
if codepoint == os.PathSeparator {
if codepoint == fs.PathSeparator {
count++
}
}

5
lib/model/rwfolder_test.go
View File

@ -87,8 +87,7 @@ func setUpSendReceiveFolder(model *Model) *sendReceiveFolder {
ctx: context.TODO(),
},
mtimeFS: fs.NewMtimeFS(fs.DefaultFilesystem, db.NewNamespacedKV(model.db, "mtime")),
dir: "testdata",
fs: fs.NewMtimeFS(fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), db.NewNamespacedKV(model.db, "mtime")),
queue: newJobQueue(),
errors: make(map[string]string),
errorsMut: sync.NewMutex(),
@ -246,7 +245,7 @@ func TestCopierFinder(t *testing.T) {
}
// Verify that the fetched blocks have actually been written to the temp file
blks, err := scanner.HashFile(context.TODO(), fs.DefaultFilesystem, tempFile, protocol.BlockSize, nil, false)
blks, err := scanner.HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, "."), tempFile, protocol.BlockSize, nil, false)
if err != nil {
t.Log(err)
}

36
lib/model/sharedpullerstate.go
View File

@ -8,10 +8,10 @@ package model
import (
"io"
"os"
"path/filepath"
"time"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
)
@ -21,6 +21,7 @@ import (
type sharedPullerState struct {
// Immutable, does not require locking
file protocol.FileInfo // The new file (desired end state)
fs fs.Filesystem
folder string
tempName string
realName string
@ -32,7 +33,7 @@ type sharedPullerState struct {
// Mutable, must be locked for access
err error // The first error we hit
fd *os.File // The fd of the temp file
fd fs.File // The fd of the temp file
copyTotal int // Total number of copy actions for the whole job
pullTotal int // Total number of pull actions for the whole job
copyOrigin int // Number of blocks copied from the original file
@ -92,8 +93,8 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
// osutil.InWritableDir except we need to do more stuff so we duplicate it
// here.
dir := filepath.Dir(s.tempName)
if info, err := os.Stat(dir); err != nil {
if os.IsNotExist(err) {
if info, err := s.fs.Stat(dir); err != nil {
if fs.IsNotExist(err) {
// XXX: This works around a bug elsewhere, a race condition when
// things are deleted while being synced. However that happens, we
// end up with a directory for "foo" with the delete bit, but a
@ -103,7 +104,7 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
// next scan it'll be found and the delete bit on it is removed.
// The user can then clean up as they like...
l.Infoln("Resurrecting directory", dir)
if err := os.MkdirAll(dir, 0755); err != nil {
if err := s.fs.MkdirAll(dir, 0755); err != nil {
s.failLocked("resurrect dir", err)
return nil, err
}
@ -112,10 +113,10 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
return nil, err
}
} else if info.Mode()&0200 == 0 {
err := os.Chmod(dir, 0755)
err := s.fs.Chmod(dir, 0755)
if !s.ignorePerms && err == nil {
defer func() {
err := os.Chmod(dir, info.Mode().Perm())
err := s.fs.Chmod(dir, info.Mode()&fs.ModePerm)
if err != nil {
panic(err)
}
@ -128,7 +129,7 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
// permissions will be set to the final value later, but in the meantime
// we don't want to have a temporary file with looser permissions than
// the final outcome.
mode := os.FileMode(s.file.Permissions) | 0600
mode := fs.FileMode(s.file.Permissions) | 0600
if s.ignorePerms {
// When ignorePerms is set we use a very permissive mode and let the
// system umask filter it.
@ -137,9 +138,9 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
// Attempt to create the temp file
// RDWR because of issue #2994.
flags := os.O_RDWR
flags := fs.OptReadWrite
if s.reused == 0 {
flags |= os.O_CREATE | os.O_EXCL
flags |= fs.OptCreate | fs.OptExclusive
} else if !s.ignorePerms {
// With sufficiently bad luck when exiting or crashing, we may have
// had time to chmod the temp file to read only state but not yet
@ -151,12 +152,12 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
// already and make no modification, as we would otherwise override
// what the umask dictates.
if err := os.Chmod(s.tempName, mode); err != nil {
if err := s.fs.Chmod(s.tempName, mode); err != nil {
s.failLocked("dst create chmod", err)
return nil, err
}
}
fd, err := os.OpenFile(s.tempName, flags, mode)
fd, err := s.fs.OpenFile(s.tempName, flags, mode)
if err != nil {
s.failLocked("dst create", err)
return nil, err
@ -180,7 +181,7 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
}
// sourceFile opens the existing source file for reading
func (s *sharedPullerState) sourceFile() (*os.File, error) {
func (s *sharedPullerState) sourceFile() (fs.File, error) {
s.mut.Lock()
defer s.mut.Unlock()
@ -190,7 +191,7 @@ func (s *sharedPullerState) sourceFile() (*os.File, error) {
}
// Attempt to open the existing file
fd, err := os.Open(s.realName)
fd, err := s.fs.Open(s.realName)
if err != nil {
s.failLocked("src open", err)
return nil, err
@ -292,9 +293,12 @@ func (s *sharedPullerState) finalClose() (bool, error) {
}
if s.fd != nil {
// This is our error if we weren't errored before. Otherwise we
// keep the earlier error.
if fsyncErr := s.fd.Sync(); fsyncErr != nil && s.err == nil {
s.err = fsyncErr
}
if closeErr := s.fd.Close(); closeErr != nil && s.err == nil {
// This is our error if we weren't errored before. Otherwise we
// keep the earlier error.
s.err = closeErr
}
s.fd = nil

8
lib/model/sharedpullerstate_test.go
View File

@ -10,12 +10,14 @@ import (
"os"
"testing"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/sync"
)
func TestSourceFileOK(t *testing.T) {
s := sharedPullerState{
realName: "testdata/foo",
fs: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
realName: "foo",
mut: sync.NewRWMutex(),
}
@ -47,6 +49,7 @@ func TestSourceFileOK(t *testing.T) {
func TestSourceFileBad(t *testing.T) {
s := sharedPullerState{
fs: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
realName: "nonexistent",
mut: sync.NewRWMutex(),
}
@ -73,7 +76,8 @@ func TestReadOnlyDir(t *testing.T) {
}()
s := sharedPullerState{
tempName: "testdata/read_only_dir/.temp_name",
fs: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
tempName: "read_only_dir/.temp_name",
mut: sync.NewRWMutex(),
}

30
lib/osutil/atomic.go
View File

@ -8,10 +8,10 @@ package osutil
import (
"errors"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"github.com/syncthing/syncthing/lib/fs"
)
var (
@ -25,7 +25,8 @@ var (
// returned on Close, so a lazy user can ignore errors until Close.
type AtomicWriter struct {
path string
next *os.File
next fs.File
fs fs.Filesystem
err error
}
@ -33,11 +34,19 @@ type AtomicWriter struct {
// instead of the given name. The file is created with secure (0600)
// permissions.
func CreateAtomic(path string) (*AtomicWriter, error) {
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, filepath.Dir(path))
return CreateAtomicFilesystem(fs, filepath.Base(path))
}
// CreateAtomicFilesystem is like os.Create, except a temporary file name is used
// instead of the given name. The file is created with secure (0600)
// permissions.
func CreateAtomicFilesystem(filesystem fs.Filesystem, path string) (*AtomicWriter, error) {
// The security of this depends on the tempfile having secure
// permissions, 0600, from the beginning. This is what ioutil.TempFile
// does. We have a test that verifies that that is the case, should this
// ever change in the standard library in the future.
fd, err := ioutil.TempFile(filepath.Dir(path), TempPrefix)
fd, err := TempFile(filesystem, filepath.Dir(path), TempPrefix)
if err != nil {
return nil, err
}
@ -45,6 +54,7 @@ func CreateAtomic(path string) (*AtomicWriter, error) {
w := &AtomicWriter{
path: path,
next: fd,
fs: filesystem,
}
return w, nil
@ -71,7 +81,7 @@ func (w *AtomicWriter) Close() error {
}
// Try to not leave temp file around, but ignore error.
defer os.Remove(w.next.Name())
defer w.fs.Remove(w.next.Name())
if err := w.next.Sync(); err != nil {
w.err = err
@ -88,17 +98,21 @@ func (w *AtomicWriter) Close() error {
// either. Return this error because it may be more informative. On non-
// Windows we want the atomic rename behavior so we don't attempt remove.
if runtime.GOOS == "windows" {
if err := os.Remove(w.path); err != nil && !os.IsNotExist(err) {
if err := w.fs.Remove(w.path); err != nil && !fs.IsNotExist(err) {
return err
}
}
if err := os.Rename(w.next.Name(), w.path); err != nil {
if err := w.fs.Rename(w.next.Name(), w.path); err != nil {
w.err = err
return err
}
SyncDir(filepath.Dir(w.next.Name()))
// fsync the directory too
if fd, err := w.fs.Open(filepath.Dir(w.next.Name())); err == nil {
fd.Sync()
fd.Close()
}
// Set w.err to return appropriately for any future operations.
w.err = ErrClosed

13
lib/osutil/fsroots_unix.go
View File

@ -1,13 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build !windows
package osutil
func GetFilesystemRoots() ([]string, error) {
return []string{"/"}, nil
}

46
lib/osutil/fsroots_windows.go
View File

@ -1,46 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build windows
package osutil
import (
"bytes"
"fmt"
"syscall"
"unsafe"
)
func GetFilesystemRoots() ([]string, error) {
kernel32, err := syscall.LoadDLL("kernel32.dll")
if err != nil {
return nil, err
}
getLogicalDriveStringsHandle, err := kernel32.FindProc("GetLogicalDriveStringsA")
if err != nil {
return nil, err
}
buffer := [1024]byte{}
bufferSize := uint32(len(buffer))
hr, _, _ := getLogicalDriveStringsHandle.Call(uintptr(unsafe.Pointer(&bufferSize)), uintptr(unsafe.Pointer(&buffer)))
if hr == 0 {
return nil, fmt.Errorf("Syscall failed")
}
var drives []string
parts := bytes.Split(buffer[:], []byte{0})
for _, part := range parts {
if len(part) == 0 {
break
}
drives = append(drives, string(part))
}
return drives, nil
}

17
lib/osutil/glob_unix.go
View File

@ -1,17 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build !windows
package osutil
import (
"path/filepath"
)
func Glob(pattern string) (matches []string, err error) {
return filepath.Glob(pattern)
}

96
lib/osutil/glob_windows.go
View File

@ -1,96 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build windows
package osutil
import (
"os"
"path/filepath"
"runtime"
"sort"
"strings"
)
// Glob implements filepath.Glob, but works with Windows long path prefixes.
// Deals with https://github.com/golang/go/issues/10577
func Glob(pattern string) (matches []string, err error) {
if !hasMeta(pattern) {
if _, err = os.Lstat(pattern); err != nil {
return nil, nil
}
return []string{pattern}, nil
}
dir, file := filepath.Split(filepath.Clean(pattern))
switch dir {
case "":
dir = "."
case string(filepath.Separator):
// nothing
default:
if runtime.GOOS != "windows" || len(dir) < 2 || dir[len(dir)-2] != ':' {
dir = dir[0 : len(dir)-1] // chop off trailing separator, if it's not after the drive letter
}
}
if !hasMeta(dir) {
return glob(dir, file, nil)
}
var m []string
m, err = Glob(dir)
if err != nil {
return
}
for _, d := range m {
matches, err = glob(d, file, matches)
if err != nil {
return
}
}
return
}
func hasMeta(path string) bool {
// Strip off Windows long path prefix if it exists.
if strings.HasPrefix(path, "\\\\?\\") {
path = path[4:]
}
// TODO(niemeyer): Should other magic characters be added here?
return strings.IndexAny(path, "*?[") >= 0
}
func glob(dir, pattern string, matches []string) (m []string, e error) {
m = matches
fi, err := os.Stat(dir)
if err != nil {
return
}
if !fi.IsDir() {
return
}
d, err := os.Open(dir)
if err != nil {
return
}
defer d.Close()
names, _ := d.Readdirnames(-1)
sort.Strings(names)
for _, n := range names {
matched, err := filepath.Match(pattern, n)
if err != nil {
return m, err
}
if matched {
m = append(m, filepath.Join(dir, n))
}
}
return
}

29
lib/osutil/glob_windows_test.go
View File

@ -1,29 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build windows
package osutil_test
import (
"testing"
"github.com/syncthing/syncthing/lib/osutil"
)
func TestGlob(t *testing.T) {
testcases := []string{
`C:\*`,
`\\?\C:\*`,
`\\?\C:\Users`,
`\\?\\\?\C:\Users`,
}
for _, tc := range testcases {
if _, err := osutil.Glob(tc); err != nil {
t.Fatalf("pattern %s failed: %v", tc, err)
}
}
}

8
lib/osutil/hidden_unix.go
View File

@ -8,12 +8,4 @@
package osutil
func HideFile(path string) error {
return nil
}
func ShowFile(path string) error {
return nil
}
func HideConsole() {}

30
lib/osutil/hidden_windows.go
View File

@ -10,36 +10,6 @@ package osutil
import "syscall"
func HideFile(path string) error {
p, err := syscall.UTF16PtrFromString(path)
if err != nil {
return err
}
attrs, err := syscall.GetFileAttributes(p)
if err != nil {
return err
}
attrs |= syscall.FILE_ATTRIBUTE_HIDDEN
return syscall.SetFileAttributes(p, attrs)
}
func ShowFile(path string) error {
p, err := syscall.UTF16PtrFromString(path)
if err != nil {
return err
}
attrs, err := syscall.GetFileAttributes(p)
if err != nil {
return err
}
attrs &^= syscall.FILE_ATTRIBUTE_HIDDEN
return syscall.SetFileAttributes(p, attrs)
}
func HideConsole() {
getConsoleWindow := syscall.NewLazyDLL("kernel32.dll").NewProc("GetConsoleWindow")
showWindow := syscall.NewLazyDLL("user32.dll").NewProc("ShowWindow")

29
lib/osutil/lstat_broken.go
View File

@ -1,29 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build linux android
package osutil
import (
"os"
"syscall"
"time"
)
// Lstat is like os.Lstat, except lobotomized for Android. See
// https://forum.syncthing.net/t/2395
func Lstat(name string) (fi os.FileInfo, err error) {
for i := 0; i < 10; i++ { // We have to draw the line somewhere
fi, err = os.Lstat(name)
if err, ok := err.(*os.PathError); ok && err.Err == syscall.EINTR {
time.Sleep(time.Duration(i+1) * time.Millisecond)
continue
}
return
}
return
}

15
lib/osutil/lstat_ok.go
View File

@ -1,15 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build !linux,!android
package osutil
import "os"
func Lstat(name string) (fi os.FileInfo, err error) {
return os.Lstat(name)
}

17
lib/osutil/mkdirall.go
View File

@ -1,17 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build !windows
package osutil
import (
"os"
)
func MkdirAll(path string, perm os.FileMode) error {
return os.MkdirAll(path, perm)
}

93
lib/osutil/mkdirall_windows.go
View File

@ -1,93 +0,0 @@
// Copyright 2009 The Go Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Modified by Zillode to fix https://github.com/syncthing/syncthing/issues/1822
// Sync with https://github.com/golang/go/blob/master/src/os/path.go
// See https://github.com/golang/go/issues/10900
package osutil
import (
"os"
"path/filepath"
"syscall"
)
// MkdirAll creates a directory named path, along with any necessary parents,
// and returns nil, or else returns an error.
// The permission bits perm are used for all directories that MkdirAll creates.
// If path is already a directory, MkdirAll does nothing and returns nil.
func MkdirAll(path string, perm os.FileMode) error {
// Fast path: if we can tell whether path is a directory or file, stop with success or error.
dir, err := os.Stat(path)
if err == nil {
if dir.IsDir() {
return nil
}
return &os.PathError{
Op: "mkdir",
Path: path,
Err: syscall.ENOTDIR,
}
}
// Slow path: make sure parent exists and then call Mkdir for path.
i := len(path)
for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
i--
}
j := i
for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
j--
}
if j > 1 {
// Create parent
parent := path[0 : j-1]
if parent != filepath.VolumeName(parent) {
err = MkdirAll(parent, perm)
if err != nil {
return err
}
}
}
// Parent now exists; invoke Mkdir and use its result.
err = os.Mkdir(path, perm)
if err != nil {
// Handle arguments like "foo/." by
// double-checking that directory doesn't exist.
dir, err1 := os.Lstat(path)
if err1 == nil && dir.IsDir() {
return nil
}
return err
}
return nil
}

56
lib/osutil/net.go Normal file
View File

@ -0,0 +1,56 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package osutil
import (
"bytes"
"net"
)
// ResolveInterfaceAddresses returns available addresses of the given network
// type for a given interface.
func ResolveInterfaceAddresses(network, nameOrMac string) []string {
intf, err := net.InterfaceByName(nameOrMac)
if err == nil {
return interfaceAddresses(network, intf)
}
mac, err := net.ParseMAC(nameOrMac)
if err != nil {
return []string{nameOrMac}
}
intfs, err := net.Interfaces()
if err != nil {
return []string{nameOrMac}
}
for _, intf := range intfs {
if bytes.Equal(intf.HardwareAddr, mac) {
return interfaceAddresses(network, &intf)
}
}
return []string{nameOrMac}
}
func interfaceAddresses(network string, intf *net.Interface) []string {
var out []string
addrs, err := intf.Addrs()
if err != nil {
return out
}
for _, addr := range addrs {
ipnet, ok := addr.(*net.IPNet)
if ok && (network == "tcp" || (network == "tcp4" && len(ipnet.IP) == net.IPv4len) || (network == "tcp6" && len(ipnet.IP) == net.IPv6len)) {
out = append(out, ipnet.IP.String())
}
}
return out
}

98
lib/osutil/osutil.go
View File

@ -9,19 +9,16 @@ package osutil
import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"strings"
"github.com/calmh/du"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/sync"
)
var errNoHome = errors.New("no home directory found - set $HOME (or the platform equivalent)")
// Try to keep this entire operation atomic-like. We shouldn't be doing this
// often enough that there is any contention on this lock.
var renameLock = sync.NewMutex()
@ -29,12 +26,12 @@ var renameLock = sync.NewMutex()
// TryRename renames a file, leaving source file intact in case of failure.
// Tries hard to succeed on various systems by temporarily tweaking directory
// permissions and removing the destination file when necessary.
func TryRename(from, to string) error {
func TryRename(filesystem fs.Filesystem, from, to string) error {
renameLock.Lock()
defer renameLock.Unlock()
return withPreparedTarget(from, to, func() error {
return os.Rename(from, to)
return withPreparedTarget(filesystem, from, to, func() error {
return filesystem.Rename(from, to)
})
}
@ -43,28 +40,28 @@ func TryRename(from, to string) error {
// for situations like committing a temp file to it's final location.
// Tries hard to succeed on various systems by temporarily tweaking directory
// permissions and removing the destination file when necessary.
func Rename(from, to string) error {
func Rename(filesystem fs.Filesystem, from, to string) error {
// Don't leave a dangling temp file in case of rename error
if !(runtime.GOOS == "windows" && strings.EqualFold(from, to)) {
defer os.Remove(from)
defer filesystem.Remove(from)
}
return TryRename(from, to)
return TryRename(filesystem, from, to)
}
// Copy copies the file content from source to destination.
// Tries hard to succeed on various systems by temporarily tweaking directory
// permissions and removing the destination file when necessary.
func Copy(from, to string) (err error) {
return withPreparedTarget(from, to, func() error {
return copyFileContents(from, to)
func Copy(filesystem fs.Filesystem, from, to string) (err error) {
return withPreparedTarget(filesystem, from, to, func() error {
return copyFileContents(filesystem, from, to)
})
}
// InWritableDir calls fn(path), while making sure that the directory
// containing `path` is writable for the duration of the call.
func InWritableDir(fn func(string) error, path string) error {
func InWritableDir(fn func(string) error, fs fs.Filesystem, path string) error {
dir := filepath.Dir(path)
info, err := os.Stat(dir)
info, err := fs.Stat(dir)
if err != nil {
return err
}
@ -75,10 +72,10 @@ func InWritableDir(fn func(string) error, path string) error {
// A non-writeable directory (for this user; we assume that's the
// relevant part). Temporarily change the mode so we can delete the
// file or directory inside it.
err = os.Chmod(dir, 0755)
err = fs.Chmod(dir, 0755)
if err == nil {
defer func() {
err = os.Chmod(dir, info.Mode())
err = fs.Chmod(dir, info.Mode())
if err != nil {
// We managed to change the permission bits like a
// millisecond ago, so it'd be bizarre if we couldn't
@ -92,59 +89,22 @@ func InWritableDir(fn func(string) error, path string) error {
return fn(path)
}
func ExpandTilde(path string) (string, error) {
if path == "~" {
return getHomeDir()
}
path = filepath.FromSlash(path)
if !strings.HasPrefix(path, fmt.Sprintf("~%c", os.PathSeparator)) {
return path, nil
}
home, err := getHomeDir()
if err != nil {
return "", err
}
return filepath.Join(home, path[2:]), nil
}
func getHomeDir() (string, error) {
var home string
switch runtime.GOOS {
case "windows":
home = filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath"))
if home == "" {
home = os.Getenv("UserProfile")
}
default:
home = os.Getenv("HOME")
}
if home == "" {
return "", errNoHome
}
return home, nil
}
// Tries hard to succeed on various systems by temporarily tweaking directory
// permissions and removing the destination file when necessary.
func withPreparedTarget(from, to string, f func() error) error {
func withPreparedTarget(filesystem fs.Filesystem, from, to string, f func() error) error {
// Make sure the destination directory is writeable
toDir := filepath.Dir(to)
if info, err := os.Stat(toDir); err == nil && info.IsDir() && info.Mode()&0200 == 0 {
os.Chmod(toDir, 0755)
defer os.Chmod(toDir, info.Mode())
if info, err := filesystem.Stat(toDir); err == nil && info.IsDir() && info.Mode()&0200 == 0 {
filesystem.Chmod(toDir, 0755)
defer filesystem.Chmod(toDir, info.Mode())
}
// On Windows, make sure the destination file is writeable (or we can't delete it)
if runtime.GOOS == "windows" {
os.Chmod(to, 0666)
filesystem.Chmod(to, 0666)
if !strings.EqualFold(from, to) {
err := os.Remove(to)
if err != nil && !os.IsNotExist(err) {
err := filesystem.Remove(to)
if err != nil && !fs.IsNotExist(err) {
return err
}
}
@ -156,13 +116,13 @@ func withPreparedTarget(from, to string, f func() error) error {
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
func copyFileContents(src, dst string) (err error) {
in, err := os.Open(src)
func copyFileContents(filesystem fs.Filesystem, src, dst string) (err error) {
in, err := filesystem.Open(src)
if err != nil {
return
}
defer in.Close()
out, err := os.Create(dst)
out, err := filesystem.Create(dst)
if err != nil {
return
}
@ -193,13 +153,3 @@ func init() {
func IsWindowsExecutable(path string) bool {
return execExts[strings.ToLower(filepath.Ext(path))]
}
func DiskFreeBytes(path string) (free int64, err error) {
u, err := du.Get(path)
return u.FreeBytes, err
}
func DiskFreePercentage(path string) (freePct float64, err error) {
u, err := du.Get(path)
return (float64(u.FreeBytes) / float64(u.TotalBytes)) * 100, err
}

42
lib/osutil/osutil_test.go
View File

@ -11,6 +11,7 @@ import (
"runtime"
"testing"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
)
@ -21,6 +22,8 @@ func TestInWriteableDir(t *testing.T) {
}
defer os.RemoveAll("testdata")
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, ".")
os.Mkdir("testdata", 0700)
os.Mkdir("testdata/rw", 0700)
os.Mkdir("testdata/ro", 0500)
@ -36,35 +39,35 @@ func TestInWriteableDir(t *testing.T) {
// These should succeed
err = osutil.InWritableDir(create, "testdata/file")
err = osutil.InWritableDir(create, fs, "testdata/file")
if err != nil {
t.Error("testdata/file:", err)
}
err = osutil.InWritableDir(create, "testdata/rw/foo")
err = osutil.InWritableDir(create, fs, "testdata/rw/foo")
if err != nil {
t.Error("testdata/rw/foo:", err)
}
err = osutil.InWritableDir(os.Remove, "testdata/rw/foo")
err = osutil.InWritableDir(os.Remove, fs, "testdata/rw/foo")
if err != nil {
t.Error("testdata/rw/foo:", err)
}
err = osutil.InWritableDir(create, "testdata/ro/foo")
err = osutil.InWritableDir(create, fs, "testdata/ro/foo")
if err != nil {
t.Error("testdata/ro/foo:", err)
}
err = osutil.InWritableDir(os.Remove, "testdata/ro/foo")
err = osutil.InWritableDir(os.Remove, fs, "testdata/ro/foo")
if err != nil {
t.Error("testdata/ro/foo:", err)
}
// These should not
err = osutil.InWritableDir(create, "testdata/nonexistent/foo")
err = osutil.InWritableDir(create, fs, "testdata/nonexistent/foo")
if err == nil {
t.Error("testdata/nonexistent/foo returned nil error")
}
err = osutil.InWritableDir(create, "testdata/file/foo")
err = osutil.InWritableDir(create, fs, "testdata/file/foo")
if err == nil {
t.Error("testdata/file/foo returned nil error")
}
@ -101,8 +104,10 @@ func TestInWritableDirWindowsRemove(t *testing.T) {
create("testdata/windows/ro/readonly")
os.Chmod("testdata/windows/ro/readonly", 0500)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, ".")
for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
err := osutil.InWritableDir(os.Remove, path)
err := osutil.InWritableDir(os.Remove, fs, path)
if err != nil {
t.Errorf("Unexpected error %s: %s", path, err)
}
@ -174,6 +179,8 @@ func TestInWritableDirWindowsRename(t *testing.T) {
create("testdata/windows/ro/readonly")
os.Chmod("testdata/windows/ro/readonly", 0500)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, ".")
for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
err := os.Rename(path, path+"new")
if err == nil {
@ -183,11 +190,11 @@ func TestInWritableDirWindowsRename(t *testing.T) {
}
rename := func(path string) error {
return osutil.Rename(path, path+"new")
return osutil.Rename(fs, path, path+"new")
}
for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
err := osutil.InWritableDir(rename, path)
err := osutil.InWritableDir(rename, fs, path)
if err != nil {
t.Errorf("Unexpected error %s: %s", path, err)
}
@ -197,18 +204,3 @@ func TestInWritableDirWindowsRename(t *testing.T) {
}
}
}
func TestDiskUsage(t *testing.T) {
free, err := osutil.DiskFreePercentage(".")
if err != nil {
if runtime.GOOS == "netbsd" ||
runtime.GOOS == "openbsd" ||
runtime.GOOS == "solaris" {
t.Skip()
}
t.Errorf("Unexpected error: %s", err)
}
if free < 1 {
t.Error("Disk is full?", free)
}
}

34
lib/osutil/sync.go
View File

@ -1,34 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package osutil
import (
"os"
"runtime"
)
func SyncFile(path string) error {
flag := 0
if runtime.GOOS == "windows" {
flag = os.O_WRONLY
}
fd, err := os.OpenFile(path, flag, 0)
if err != nil {
return err
}
defer fd.Close()
// MacOS and Windows do not flush the disk cache
return fd.Sync()
}
func SyncDir(path string) error {
if runtime.GOOS == "windows" {
// not supported by Windows
return nil
}
return SyncFile(path)
}

63
lib/osutil/tempfile.go Normal file
View File

@ -0,0 +1,63 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package osutil
import (
"os"
"path/filepath"
"strconv"
"sync"
"time"
"github.com/syncthing/syncthing/lib/fs"
)
var rand uint32
var randmu sync.Mutex
func reseed() uint32 {
return uint32(time.Now().UnixNano() + int64(os.Getpid()))
}
func nextSuffix() string {
randmu.Lock()
r := rand
if r == 0 {
r = reseed()
}
r = r*1664525 + 1013904223 // constants from Numerical Recipes
rand = r
randmu.Unlock()
return strconv.Itoa(int(1e9 + r%1e9))[1:]
}
// TempFile creates a new temporary file in the directory dir
// with a name beginning with prefix, opens the file for reading
// and writing, and returns the resulting *os.File.
// If dir is the empty string, TempFile uses the default directory
// for temporary files (see os.TempDir).
// Multiple programs calling TempFile simultaneously
// will not choose the same file. The caller can use f.Name()
// to find the pathname of the file. It is the caller's responsibility
// to remove the file when no longer needed.
func TempFile(filesystem fs.Filesystem, dir, prefix string) (f fs.File, err error) {
nconflict := 0
for i := 0; i < 10000; i++ {
name := filepath.Join(dir, prefix+nextSuffix())
f, err = filesystem.OpenFile(name, fs.OptReadWrite|fs.OptCreate|fs.OptExclusive, 0600)
if fs.IsExist(err) {
if nconflict++; nconflict > 10 {
randmu.Lock()
rand = reseed()
randmu.Unlock()
}
continue
}
break
}
return
}

16
lib/osutil/traversessymlink.go
View File

@ -8,9 +8,10 @@ package osutil
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/syncthing/syncthing/lib/fs"
)
// TraversesSymlinkError is an error indicating symlink traversal
@ -34,9 +35,10 @@ func (e NotADirectoryError) Error() string {
// TraversesSymlink returns an error if base and any path component of name up to and
// including filepath.Join(base, name) traverses a symlink.
// Base and name must both be clean and name must be relative to base.
func TraversesSymlink(base, name string) error {
func TraversesSymlink(filesystem fs.Filesystem, name string) error {
base := "."
path := base
info, err := Lstat(path)
info, err := filesystem.Lstat(path)
if err != nil {
return err
}
@ -51,17 +53,17 @@ func TraversesSymlink(base, name string) error {
return nil
}
parts := strings.Split(name, string(os.PathSeparator))
parts := strings.Split(name, string(fs.PathSeparator))
for _, part := range parts {
path = filepath.Join(path, part)
info, err := Lstat(path)
info, err := filesystem.Lstat(path)
if err != nil {
if os.IsNotExist(err) {
if fs.IsNotExist(err) {
return nil
}
return err
}
if info.Mode()&os.ModeSymlink != 0 {
if info.IsSymlink() {
return &TraversesSymlinkError{
path: strings.TrimPrefix(path, base),
}

16
lib/osutil/traversessymlink_test.go
View File

@ -12,17 +12,20 @@ import (
"os"
"testing"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
)
func TestTraversesSymlink(t *testing.T) {
os.RemoveAll("testdata")
defer os.RemoveAll("testdata")
os.MkdirAll("testdata/a/b/c", 0755)
os.Symlink("b", "testdata/a/l")
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata")
fs.MkdirAll("a/b/c", 0755)
fs.CreateSymlink("b", "a/l")
// a/l -> b, so a/l/c should resolve by normal stat
info, err := osutil.Lstat("testdata/a/l/c")
info, err := fs.Lstat("a/l/c")
if err != nil {
t.Fatal("unexpected error", err)
}
@ -52,7 +55,7 @@ func TestTraversesSymlink(t *testing.T) {
}
for _, tc := range cases {
if res := osutil.TraversesSymlink("testdata", tc.name); tc.traverses == (res == nil) {
if res := osutil.TraversesSymlink(fs, tc.name); tc.traverses == (res == nil) {
t.Errorf("TraversesSymlink(%q) = %v, should be %v", tc.name, res, tc.traverses)
}
}
@ -63,10 +66,11 @@ var traversesSymlinkResult error
func BenchmarkTraversesSymlink(b *testing.B) {
os.RemoveAll("testdata")
defer os.RemoveAll("testdata")
os.MkdirAll("testdata/a/b/c", 0755)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata")
fs.MkdirAll("a/b/c", 0755)
for i := 0; i < b.N; i++ {
traversesSymlinkResult = osutil.TraversesSymlink("testdata", "a/b/c")
traversesSymlinkResult = osutil.TraversesSymlink(fs, "a/b/c")
}
b.ReportAllocs()

7
lib/scanner/blockqueue.go
View File

@ -9,7 +9,6 @@ package scanner
import (
"context"
"errors"
"path/filepath"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
@ -64,7 +63,6 @@ func HashFile(ctx context.Context, fs fs.Filesystem, path string, blockSize int,
// is closed and all items handled.
type parallelHasher struct {
fs fs.Filesystem
dir string
blockSize int
workers int
outbox chan<- protocol.FileInfo
@ -75,10 +73,9 @@ type parallelHasher struct {
wg sync.WaitGroup
}
func newParallelHasher(ctx context.Context, fs fs.Filesystem, dir string, blockSize, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) {
func newParallelHasher(ctx context.Context, fs fs.Filesystem, blockSize, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) {
ph := &parallelHasher{
fs: fs,
dir: dir,
blockSize: blockSize,
workers: workers,
outbox: outbox,
@ -111,7 +108,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) {
panic("Bug. Asked to hash a directory or a deleted file.")
}
blocks, err := HashFile(ctx, ph.fs, filepath.Join(ph.dir, f.Name), ph.blockSize, ph.counter, ph.useWeakHashes)
blocks, err := HashFile(ctx, ph.fs, f.Name, ph.blockSize, ph.counter, ph.useWeakHashes)
if err != nil {
l.Debugln("hash error:", f.Name, err)
continue

29
lib/scanner/infinitefs_test.go
View File

@ -19,6 +19,7 @@ import (
)
type infiniteFS struct {
fs.Filesystem
width int // number of files and directories per level
depth int // number of tree levels to simulate
filesize int64 // size of each file in bytes
@ -50,18 +51,6 @@ func (i infiniteFS) Open(name string) (fs.File, error) {
return &fakeFile{name, i.filesize, 0}, nil
}
func (infiniteFS) Chmod(name string, mode fs.FileMode) error { return errNotSupp }
func (infiniteFS) Chtimes(name string, atime time.Time, mtime time.Time) error { return errNotSupp }
func (infiniteFS) Create(name string) (fs.File, error) { return nil, errNotSupp }
func (infiniteFS) CreateSymlink(name, target string) error { return errNotSupp }
func (infiniteFS) Mkdir(name string, perm fs.FileMode) error { return errNotSupp }
func (infiniteFS) ReadSymlink(name string) (string, error) { return "", errNotSupp }
func (infiniteFS) Remove(name string) error { return errNotSupp }
func (infiniteFS) Rename(oldname, newname string) error { return errNotSupp }
func (infiniteFS) Stat(name string) (fs.FileInfo, error) { return nil, errNotSupp }
func (infiniteFS) SymlinksSupported() bool { return false }
func (infiniteFS) Walk(root string, walkFn fs.WalkFunc) error { return errNotSupp }
type fakeInfo struct {
name string
size int64
@ -71,7 +60,7 @@ func (f fakeInfo) Name() string { return f.name }
func (f fakeInfo) Mode() fs.FileMode { return 0755 }
func (f fakeInfo) Size() int64 { return f.size }
func (f fakeInfo) ModTime() time.Time { return time.Unix(1234567890, 0) }
func (f fakeInfo) IsDir() bool { return strings.Contains(filepath.Base(f.name), "dir") }
func (f fakeInfo) IsDir() bool { return strings.Contains(filepath.Base(f.name), "dir") || f.name == "." }
func (f fakeInfo) IsRegular() bool { return !f.IsDir() }
func (f fakeInfo) IsSymlink() bool { return false }
@ -81,6 +70,10 @@ type fakeFile struct {
readOffset int64
}
func (f *fakeFile) Name() string {
return f.name
}
func (f *fakeFile) Read(bs []byte) (int, error) {
remaining := f.size - f.readOffset
if remaining == 0 {
@ -98,6 +91,10 @@ func (f *fakeFile) Stat() (fs.FileInfo, error) {
return fakeInfo{f.name, f.size}, nil
}
func (f *fakeFile) WriteAt(bs []byte, offs int64) (int, error) { return 0, errNotSupp }
func (f *fakeFile) Close() error { return nil }
func (f *fakeFile) Truncate(size int64) error { return errNotSupp }
func (f *fakeFile) Write([]byte) (int, error) { return 0, errNotSupp }
func (f *fakeFile) WriteAt([]byte, int64) (int, error) { return 0, errNotSupp }
func (f *fakeFile) Close() error { return nil }
func (f *fakeFile) Truncate(size int64) error { return errNotSupp }
func (f *fakeFile) ReadAt([]byte, int64) (int, error) { return 0, errNotSupp }
func (f *fakeFile) Seek(int64, int) (int64, error) { return 0, errNotSupp }
func (f *fakeFile) Sync() error { return nil }

106
lib/scanner/walk.go
View File

@ -9,7 +9,6 @@ package scanner
import (
"context"
"errors"
"path/filepath"
"runtime"
"sync/atomic"
"time"
@ -42,8 +41,6 @@ func init() {
type Config struct {
// Folder for which the walker has been created
Folder string
// Dir is the base directory for the walk
Dir string
// Limit walking to these paths within Dir, or no limit if Sub is empty
Subs []string
// BlockSize controls the size of the block used when hashing.
@ -86,7 +83,7 @@ func Walk(ctx context.Context, cfg Config) (chan protocol.FileInfo, error) {
w.CurrentFiler = noCurrentFiler{}
}
if w.Filesystem == nil {
w.Filesystem = fs.DefaultFilesystem
panic("no filesystem specified")
}
return w.walk(ctx)
@ -99,7 +96,7 @@ type walker struct {
// Walk returns the list of files found in the local folder by scanning the
// file system. Files are blockwise hashed.
func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
l.Debugln("Walk", w.Dir, w.Subs, w.BlockSize, w.Matcher)
l.Debugln("Walk", w.Subs, w.BlockSize, w.Matcher)
if err := w.checkDir(); err != nil {
return nil, err
@ -113,10 +110,10 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
go func() {
hashFiles := w.walkAndHashFiles(ctx, toHashChan, finishedChan)
if len(w.Subs) == 0 {
w.Filesystem.Walk(w.Dir, hashFiles)
w.Filesystem.Walk(".", hashFiles)
} else {
for _, sub := range w.Subs {
w.Filesystem.Walk(filepath.Join(w.Dir, sub), hashFiles)
w.Filesystem.Walk(sub, hashFiles)
}
}
close(toHashChan)
@ -125,7 +122,7 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
// We're not required to emit scan progress events, just kick off hashers,
// and feed inputs directly from the walker.
if w.ProgressTickIntervalS < 0 {
newParallelHasher(ctx, w.Filesystem, w.Dir, w.BlockSize, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes)
newParallelHasher(ctx, w.Filesystem, w.BlockSize, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes)
return finishedChan, nil
}
@ -156,7 +153,7 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
done := make(chan struct{})
progress := newByteCounter()
newParallelHasher(ctx, w.Filesystem, w.Dir, w.BlockSize, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes)
newParallelHasher(ctx, w.Filesystem, w.BlockSize, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes)
// A routine which actually emits the FolderScanProgress events
// every w.ProgressTicker ticks, until the hasher routines terminate.
@ -166,13 +163,13 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
for {
select {
case <-done:
l.Debugln("Walk progress done", w.Dir, w.Subs, w.BlockSize, w.Matcher)
l.Debugln("Walk progress done", w.Folder, w.Subs, w.BlockSize, w.Matcher)
ticker.Stop()
return
case <-ticker.C:
current := progress.Total()
rate := progress.Rate()
l.Debugf("Walk %s %s current progress %d/%d at %.01f MiB/s (%d%%)", w.Dir, w.Subs, current, total, rate/1024/1024, current*100/total)
l.Debugf("Walk %s %s current progress %d/%d at %.01f MiB/s (%d%%)", w.Folder, w.Subs, current, total, rate/1024/1024, current*100/total)
events.Default.Log(events.FolderScanProgress, map[string]interface{}{
"folder": w.Folder,
"current": current,
@ -203,7 +200,7 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
func (w *walker) walkAndHashFiles(ctx context.Context, fchan, dchan chan protocol.FileInfo) fs.WalkFunc {
now := time.Now()
return func(absPath string, info fs.FileInfo, err error) error {
return func(path string, info fs.FileInfo, err error) error {
select {
case <-ctx.Done():
return ctx.Err()
@ -219,58 +216,52 @@ func (w *walker) walkAndHashFiles(ctx context.Context, fchan, dchan chan protoco
}
if err != nil {
l.Debugln("error:", absPath, info, err)
l.Debugln("error:", path, info, err)
return skip
}
relPath, err := filepath.Rel(w.Dir, absPath)
if err != nil {
l.Debugln("rel error:", absPath, err)
return skip
}
if relPath == "." {
if path == "." {
return nil
}
info, err = w.Filesystem.Lstat(absPath)
info, err = w.Filesystem.Lstat(path)
// An error here would be weird as we've already gotten to this point, but act on it nonetheless
if err != nil {
return skip
}
if ignore.IsTemporary(relPath) {
l.Debugln("temporary:", relPath)
if ignore.IsTemporary(path) {
l.Debugln("temporary:", path)
if info.IsRegular() && info.ModTime().Add(w.TempLifetime).Before(now) {
w.Filesystem.Remove(absPath)
l.Debugln("removing temporary:", relPath, info.ModTime())
w.Filesystem.Remove(path)
l.Debugln("removing temporary:", path, info.ModTime())
}
return nil
}
if ignore.IsInternal(relPath) {
l.Debugln("ignored (internal):", relPath)
if ignore.IsInternal(path) {
l.Debugln("ignored (internal):", path)
return skip
}
if w.Matcher.Match(relPath).IsIgnored() {
l.Debugln("ignored (patterns):", relPath)
if w.Matcher.Match(path).IsIgnored() {
l.Debugln("ignored (patterns):", path)
return skip
}
if !utf8.ValidString(relPath) {
l.Warnf("File name %q is not in UTF8 encoding; skipping.", relPath)
if !utf8.ValidString(path) {
l.Warnf("File name %q is not in UTF8 encoding; skipping.", path)
return skip
}
relPath, shouldSkip := w.normalizePath(absPath, relPath)
path, shouldSkip := w.normalizePath(path)
if shouldSkip {
return skip
}
switch {
case info.IsSymlink():
if err := w.walkSymlink(ctx, absPath, relPath, dchan); err != nil {
if err := w.walkSymlink(ctx, path, dchan); err != nil {
return err
}
if info.IsDir() {
@ -280,10 +271,10 @@ func (w *walker) walkAndHashFiles(ctx context.Context, fchan, dchan chan protoco
return nil
case info.IsDir():
err = w.walkDir(ctx, relPath, info, dchan)
err = w.walkDir(ctx, path, info, dchan)
case info.IsRegular():
err = w.walkRegular(ctx, relPath, info, fchan)
err = w.walkRegular(ctx, path, info, fchan)
}
return err
@ -375,7 +366,7 @@ func (w *walker) walkDir(ctx context.Context, relPath string, info fs.FileInfo,
// walkSymlink returns nil or an error, if the error is of the nature that
// it should stop the entire walk.
func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan chan protocol.FileInfo) error {
func (w *walker) walkSymlink(ctx context.Context, relPath string, dchan chan protocol.FileInfo) error {
// Symlinks are not supported on Windows. We ignore instead of returning
// an error.
if runtime.GOOS == "windows" {
@ -387,9 +378,9 @@ func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan
// checking that their existing blocks match with the blocks in
// the index.
target, err := w.Filesystem.ReadSymlink(absPath)
target, err := w.Filesystem.ReadSymlink(relPath)
if err != nil {
l.Debugln("readlink error:", absPath, err)
l.Debugln("readlink error:", relPath, err)
return nil
}
@ -413,7 +404,7 @@ func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan
SymlinkTarget: target,
}
l.Debugln("symlink changedb:", absPath, f)
l.Debugln("symlink changedb:", relPath, f)
select {
case dchan <- f:
@ -426,55 +417,58 @@ func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan
// normalizePath returns the normalized relative path (possibly after fixing
// it on disk), or skip is true.
func (w *walker) normalizePath(absPath, relPath string) (normPath string, skip bool) {
func (w *walker) normalizePath(path string) (normPath string, skip bool) {
if runtime.GOOS == "darwin" {
// Mac OS X file names should always be NFD normalized.
normPath = norm.NFD.String(relPath)
normPath = norm.NFD.String(path)
} else {
// Every other OS in the known universe uses NFC or just plain
// doesn't bother to define an encoding. In our case *we* do care,
// so we enforce NFC regardless.
normPath = norm.NFC.String(relPath)
normPath = norm.NFC.String(path)
}
if relPath != normPath {
if path != normPath {
// The file name was not normalized.
if !w.AutoNormalize {
// We're not authorized to do anything about it, so complain and skip.
l.Warnf("File name %q is not in the correct UTF8 normalization form; skipping.", relPath)
l.Warnf("File name %q is not in the correct UTF8 normalization form; skipping.", path)
return "", true
}
// We will attempt to normalize it.
normalizedPath := filepath.Join(w.Dir, normPath)
if _, err := w.Filesystem.Lstat(normalizedPath); fs.IsNotExist(err) {
if _, err := w.Filesystem.Lstat(normPath); fs.IsNotExist(err) {
// Nothing exists with the normalized filename. Good.
if err = w.Filesystem.Rename(absPath, normalizedPath); err != nil {
l.Infof(`Error normalizing UTF8 encoding of file "%s": %v`, relPath, err)
if err = w.Filesystem.Rename(path, normPath); err != nil {
l.Infof(`Error normalizing UTF8 encoding of file "%s": %v`, path, err)
return "", true
}
l.Infof(`Normalized UTF8 encoding of file name "%s".`, relPath)
l.Infof(`Normalized UTF8 encoding of file name "%s".`, path)
} else {
// There is something already in the way at the normalized
// file name.
l.Infof(`File "%s" has UTF8 encoding conflict with another file; ignoring.`, relPath)
l.Infof(`File "%s" path has UTF8 encoding conflict with another file; ignoring.`, path)
return "", true
}
}
return normPath, false
return path, false
}
func (w *walker) checkDir() error {
if info, err := w.Filesystem.Lstat(w.Dir); err != nil {
info, err := w.Filesystem.Lstat(".")
if err != nil {
return err
} else if !info.IsDir() {
return errors.New(w.Dir + ": not a directory")
} else {
l.Debugln("checkDir", w.Dir, info)
}
if !info.IsDir() {
return errors.New(w.Filesystem.URI() + ": not a directory")
}
l.Debugln("checkDir", w.Filesystem.Type(), w.Filesystem.URI(), info)
return nil
}

68
lib/scanner/walk_test.go
View File

@ -23,7 +23,6 @@ import (
"github.com/d4l3k/messagediff"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"golang.org/x/text/unicode/norm"
)
@ -54,18 +53,18 @@ func init() {
}
func TestWalkSub(t *testing.T) {
ignores := ignore.New()
ignores := ignore.New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."))
err := ignores.Load("testdata/.stignore")
if err != nil {
t.Fatal(err)
}
fchan, err := Walk(context.TODO(), Config{
Dir: "testdata",
Subs: []string{"dir2"},
BlockSize: 128 * 1024,
Matcher: ignores,
Hashers: 2,
Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
Subs: []string{"dir2"},
BlockSize: 128 * 1024,
Matcher: ignores,
Hashers: 2,
})
var files []protocol.FileInfo
for f := range fchan {
@ -90,7 +89,7 @@ func TestWalkSub(t *testing.T) {
}
func TestWalk(t *testing.T) {
ignores := ignore.New()
ignores := ignore.New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."))
err := ignores.Load("testdata/.stignore")
if err != nil {
t.Fatal(err)
@ -98,10 +97,10 @@ func TestWalk(t *testing.T) {
t.Log(ignores)
fchan, err := Walk(context.TODO(), Config{
Dir: "testdata",
BlockSize: 128 * 1024,
Matcher: ignores,
Hashers: 2,
Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
BlockSize: 128 * 1024,
Matcher: ignores,
Hashers: 2,
})
if err != nil {
@ -122,9 +121,9 @@ func TestWalk(t *testing.T) {
func TestWalkError(t *testing.T) {
_, err := Walk(context.TODO(), Config{
Dir: "testdata-missing",
BlockSize: 128 * 1024,
Hashers: 2,
Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata-missing"),
BlockSize: 128 * 1024,
Hashers: 2,
})
if err == nil {
@ -132,8 +131,8 @@ func TestWalkError(t *testing.T) {
}
_, err = Walk(context.TODO(), Config{
Dir: "testdata/bar",
BlockSize: 128 * 1024,
Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata/bar"),
BlockSize: 128 * 1024,
})
if err == nil {
@ -220,9 +219,11 @@ func TestNormalization(t *testing.T) {
numValid := len(tests) - numInvalid
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, ".")
for _, s1 := range tests {
// Create a directory for each of the interesting strings above
if err := osutil.MkdirAll(filepath.Join("testdata/normalization", s1), 0755); err != nil {
if err := fs.MkdirAll(filepath.Join("testdata/normalization", s1), 0755); err != nil {
t.Fatal(err)
}
@ -231,10 +232,10 @@ func TestNormalization(t *testing.T) {
// file names. Ensure that the file doesn't exist when it's
// created. This detects and fails if there's file name
// normalization stuff at the filesystem level.
if fd, err := os.OpenFile(filepath.Join("testdata/normalization", s1, s2), os.O_CREATE|os.O_EXCL, 0644); err != nil {
if fd, err := fs.OpenFile(filepath.Join("testdata/normalization", s1, s2), os.O_CREATE|os.O_EXCL, 0644); err != nil {
t.Fatal(err)
} else {
fd.WriteString("test")
fd.Write([]byte("test"))
fd.Close()
}
}
@ -245,11 +246,11 @@ func TestNormalization(t *testing.T) {
// make sure it all gets done. In production, things will be correct
// eventually...
_, err := walkDir("testdata/normalization")
_, err := walkDir(fs, "testdata/normalization")
if err != nil {
t.Fatal(err)
}
tmp, err := walkDir("testdata/normalization")
tmp, err := walkDir(fs, "testdata/normalization")
if err != nil {
t.Fatal(err)
}
@ -299,8 +300,8 @@ func TestWalkSymlinkUnix(t *testing.T) {
// Scan it
fchan, err := Walk(context.TODO(), Config{
Dir: "_symlinks",
BlockSize: 128 * 1024,
Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"),
BlockSize: 128 * 1024,
})
if err != nil {
@ -344,8 +345,8 @@ func TestWalkSymlinkWindows(t *testing.T) {
// Scan it
fchan, err := Walk(context.TODO(), Config{
Dir: "_symlinks",
BlockSize: 128 * 1024,
Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"),
BlockSize: 128 * 1024,
})
if err != nil {
@ -364,9 +365,10 @@ func TestWalkSymlinkWindows(t *testing.T) {
}
}
func walkDir(dir string) ([]protocol.FileInfo, error) {
func walkDir(fs fs.Filesystem, dir string) ([]protocol.FileInfo, error) {
fchan, err := Walk(context.TODO(), Config{
Dir: dir,
Filesystem: fs,
Subs: []string{dir},
BlockSize: 128 * 1024,
AutoNormalize: true,
Hashers: 2,
@ -435,7 +437,7 @@ func BenchmarkHashFile(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := HashFile(context.TODO(), fs.DefaultFilesystem, testdataName, protocol.BlockSize, nil, true); err != nil {
if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.BlockSize, nil, true); err != nil {
b.Fatal(err)
}
}
@ -467,15 +469,17 @@ func TestStopWalk(t *testing.T) {
// many directories. It'll take a while to scan, giving us time to
// cancel it and make sure the scan stops.
fs := fs.NewWalkFilesystem(&infiniteFS{100, 100, 1e6})
// Use an errorFs as the backing fs for the rest of the interface
// The way we get it is a bit hacky tho.
errorFs := fs.NewFilesystem(fs.FilesystemType(-1), ".")
fs := fs.NewWalkFilesystem(&infiniteFS{errorFs, 100, 100, 1e6})
const numHashers = 4
ctx, cancel := context.WithCancel(context.Background())
fchan, err := Walk(ctx, Config{
Dir: "testdir",
Filesystem: fs,
BlockSize: 128 * 1024,
Hashers: numHashers,
Filesystem: fs,
ProgressTickIntervalS: -1, // Don't attempt to build the full list of files before starting to scan...
})

46
lib/versioner/external.go
View File

@ -10,10 +10,11 @@ import (
"errors"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/fs"
"github.com/kballard/go-shellquote"
)
func init() {
@ -23,15 +24,15 @@ func init() {
type External struct {
command string
folderPath string
filesystem fs.Filesystem
}
func NewExternal(folderID, folderPath string, params map[string]string) Versioner {
func NewExternal(folderID string, filesystem fs.Filesystem, params map[string]string) Versioner {
command := params["command"]
s := External{
command: command,
folderPath: folderPath,
filesystem: filesystem,
}
l.Debugf("instantiated %#v", s)
@ -41,29 +42,41 @@ func NewExternal(folderID, folderPath string, params map[string]string) Versione
// Archive moves the named file away to a version archive. If this function
// returns nil, the named file does not exist any more (has been archived).
func (v External) Archive(filePath string) error {
info, err := osutil.Lstat(filePath)
if os.IsNotExist(err) {
info, err := v.filesystem.Lstat(filePath)
if fs.IsNotExist(err) {
l.Debugln("not archiving nonexistent file", filePath)
return nil
} else if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 {
if info.IsSymlink() {
panic("bug: attempting to version a symlink")
}
l.Debugln("archiving", filePath)
inFolderPath, err := filepath.Rel(v.folderPath, filePath)
if err != nil {
return err
}
if v.command == "" {
return errors.New("Versioner: command is empty, please enter a valid command")
}
cmd := exec.Command(v.command, v.folderPath, inFolderPath)
words, err := shellquote.Split(v.command)
if err != nil {
return errors.New("Versioner: command is invalid: " + err.Error())
}
context := map[string]string{
"%FOLDER_FILESYSTEM%": v.filesystem.Type().String(),
"%FOLDER_PATH%": v.filesystem.URI(),
"%FILE_PATH%": filePath,
}
for i, word := range words {
if replacement, ok := context[word]; ok {
words[i] = replacement
}
}
cmd := exec.Command(words[0], words[1:]...)
env := os.Environ()
// filter STGUIAUTH and STGUIAPIKEY from environment variables
filteredEnv := []string{}
@ -73,13 +86,14 @@ func (v External) Archive(filePath string) error {
}
}
cmd.Env = filteredEnv
err = cmd.Run()
combinedOutput, err := cmd.CombinedOutput()
l.Debugln("external command output:", string(combinedOutput))
if err != nil {
return err
}
// return error if the file was not removed
if _, err = osutil.Lstat(filePath); os.IsNotExist(err) {
if _, err = v.filesystem.Lstat(filePath); fs.IsNotExist(err) {
return nil
}
return errors.New("Versioner: file was not removed by external script")

12
lib/versioner/external_test.go
View File

@ -12,6 +12,8 @@ import (
"path/filepath"
"runtime"
"testing"
"github.com/syncthing/syncthing/lib/fs"
)
func TestExternalNoCommand(t *testing.T) {
@ -28,8 +30,8 @@ func TestExternalNoCommand(t *testing.T) {
// The versioner should fail due to missing command.
e := External{
filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "."),
command: "nonexistent command",
folderPath: "testdata/folder path",
}
if err := e.Archive(file); err == nil {
t.Error("Command should have failed")
@ -43,12 +45,12 @@ func TestExternalNoCommand(t *testing.T) {
}
func TestExternal(t *testing.T) {
cmd := "./_external_test/external.sh"
cmd := "./_external_test/external.sh %FOLDER_PATH% %FILE_PATH%"
if runtime.GOOS == "windows" {
cmd = `.\_external_test\external.bat`
cmd = `.\\_external_test\\external.bat %FOLDER_PATH% %FILE_PATH%`
}
file := "testdata/folder path/dir (parens)/long filename (parens).txt"
file := filepath.Join("testdata", "folder path", "dir (parens)", "/long filename (parens).txt")
prepForRemoval(t, file)
defer os.RemoveAll("testdata")
@ -61,8 +63,8 @@ func TestExternal(t *testing.T) {
// The versioner should run successfully.
e := External{
filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "."),
command: cmd,
folderPath: "testdata/folder path",
}
if err := e.Archive(file); err != nil {
t.Fatal(err)

49
lib/versioner/simple.go
View File

@ -7,10 +7,10 @@
package versioner
import (
"os"
"path/filepath"
"strconv"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/util"
)
@ -21,19 +21,19 @@ func init() {
}
type Simple struct {
keep int
folderPath string
keep int
fs fs.Filesystem
}
func NewSimple(folderID, folderPath string, params map[string]string) Versioner {
func NewSimple(folderID string, fs fs.Filesystem, params map[string]string) Versioner {
keep, err := strconv.Atoi(params["keep"])
if err != nil {
keep = 5 // A reasonable default
}
s := Simple{
keep: keep,
folderPath: folderPath,
keep: keep,
fs: fs,
}
l.Debugf("instantiated %#v", s)
@ -43,24 +43,24 @@ func NewSimple(folderID, folderPath string, params map[string]string) Versioner
// Archive moves the named file away to a version archive. If this function
// returns nil, the named file does not exist any more (has been archived).
func (v Simple) Archive(filePath string) error {
fileInfo, err := osutil.Lstat(filePath)
if os.IsNotExist(err) {
info, err := v.fs.Lstat(filePath)
if fs.IsNotExist(err) {
l.Debugln("not archiving nonexistent file", filePath)
return nil
} else if err != nil {
return err
}
if fileInfo.Mode()&os.ModeSymlink != 0 {
if info.IsSymlink() {
panic("bug: attempting to version a symlink")
}
versionsDir := filepath.Join(v.folderPath, ".stversions")
_, err = os.Stat(versionsDir)
versionsDir := ".stversions"
_, err = v.fs.Stat(versionsDir)
if err != nil {
if os.IsNotExist(err) {
l.Debugln("creating versions dir", versionsDir)
osutil.MkdirAll(versionsDir, 0755)
osutil.HideFile(versionsDir)
if fs.IsNotExist(err) {
l.Debugln("creating versions dir .stversions")
v.fs.Mkdir(versionsDir, 0755)
v.fs.Hide(versionsDir)
} else {
return err
}
@ -69,28 +69,25 @@ func (v Simple) Archive(filePath string) error {
l.Debugln("archiving", filePath)
file := filepath.Base(filePath)
inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath))
if err != nil {
return err
}
inFolderPath := filepath.Dir(filePath)
dir := filepath.Join(versionsDir, inFolderPath)
err = osutil.MkdirAll(dir, 0755)
if err != nil && !os.IsExist(err) {
err = v.fs.MkdirAll(dir, 0755)
if err != nil && !fs.IsExist(err) {
return err
}
ver := taggedFilename(file, fileInfo.ModTime().Format(TimeFormat))
ver := taggedFilename(file, info.ModTime().Format(TimeFormat))
dst := filepath.Join(dir, ver)
l.Debugln("moving to", dst)
err = osutil.Rename(filePath, dst)
err = osutil.Rename(v.fs, filePath, dst)
if err != nil {
return err
}
// Glob according to the new file~timestamp.ext pattern.
pattern := filepath.Join(dir, taggedFilename(file, TimeGlob))
newVersions, err := osutil.Glob(pattern)
newVersions, err := v.fs.Glob(pattern)
if err != nil {
l.Warnln("globbing:", err, "for", pattern)
return nil
@ -98,7 +95,7 @@ func (v Simple) Archive(filePath string) error {
// Also according to the old file.ext~timestamp pattern.
pattern = filepath.Join(dir, file+"~"+TimeGlob)
oldVersions, err := osutil.Glob(pattern)
oldVersions, err := v.fs.Glob(pattern)
if err != nil {
l.Warnln("globbing:", err, "for", pattern)
return nil
@ -111,7 +108,7 @@ func (v Simple) Archive(filePath string) error {
if len(versions) > v.keep {
for _, toRemove := range versions[:len(versions)-v.keep] {
l.Debugln("cleaning out", toRemove)
err = os.Remove(toRemove)
err = v.fs.Remove(toRemove)
if err != nil {
l.Warnln("removing old version:", err)
}

23
lib/versioner/simple_test.go
View File

@ -9,10 +9,11 @@ package versioner
import (
"io/ioutil"
"math"
"os"
"path/filepath"
"testing"
"time"
"github.com/syncthing/syncthing/lib/fs"
)
func TestTaggedFilename(t *testing.T) {
@ -53,29 +54,28 @@ func TestSimpleVersioningVersionCount(t *testing.T) {
}
dir, err := ioutil.TempDir("", "")
defer os.RemoveAll(dir)
//defer os.RemoveAll(dir)
if err != nil {
t.Error(err)
}
v := NewSimple("", dir, map[string]string{"keep": "2"})
versionDir := filepath.Join(dir, ".stversions")
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
path := filepath.Join(dir, "test")
v := NewSimple("", fs, map[string]string{"keep": "2"})
path := "test"
for i := 1; i <= 3; i++ {
f, err := os.Create(path)
f, err := fs.Create(path)
if err != nil {
t.Error(err)
}
f.Close()
v.Archive(path)
d, err := os.Open(versionDir)
if err != nil {
if err := v.Archive(path); err != nil {
t.Error(err)
}
n, err := d.Readdirnames(-1)
n, err := fs.DirNames(".stversions")
if err != nil {
t.Error(err)
}
@ -83,7 +83,6 @@ func TestSimpleVersioningVersionCount(t *testing.T) {
if float64(len(n)) != math.Min(float64(i), 2) {
t.Error("Wrong count")
}
d.Close()
time.Sleep(time.Second)
}

94
lib/versioner/staggered.go
View File

@ -12,7 +12,7 @@ import (
"strconv"
"time"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/util"
)
@ -28,9 +28,9 @@ type Interval struct {
}
type Staggered struct {
versionsPath string
cleanInterval int64
folderPath string
folderFs fs.Filesystem
versionsFs fs.Filesystem
interval [4]Interval
mutex sync.Mutex
@ -38,7 +38,7 @@ type Staggered struct {
testCleanDone chan struct{}
}
func NewStaggered(folderID, folderPath string, params map[string]string) Versioner {
func NewStaggered(folderID string, folderFs fs.Filesystem, params map[string]string) Versioner {
maxAge, err := strconv.ParseInt(params["maxAge"], 10, 0)
if err != nil {
maxAge = 31536000 // Default: ~1 year
@ -49,22 +49,20 @@ func NewStaggered(folderID, folderPath string, params map[string]string) Version
}
// Use custom path if set, otherwise .stversions in folderPath
var versionsDir string
var versionsFs fs.Filesystem
if params["versionsPath"] == "" {
versionsDir = filepath.Join(folderPath, ".stversions")
l.Debugln("using default dir .stversions")
versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), ".stversions"))
} else if filepath.IsAbs(params["versionsPath"]) {
l.Debugln("using dir", params["versionsPath"])
versionsDir = params["versionsPath"]
versionsFs = fs.NewFilesystem(folderFs.Type(), params["versionsPath"])
} else {
versionsDir = filepath.Join(folderPath, params["versionsPath"])
l.Debugln("using dir", versionsDir)
versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), params["versionsPath"]))
}
l.Debugln("%s folder using %s (%s) staggered versioner dir", folderID, versionsFs.URI(), versionsFs.Type())
s := &Staggered{
versionsPath: versionsDir,
cleanInterval: cleanInterval,
folderPath: folderPath,
folderFs: folderFs,
versionsFs: versionsFs,
interval: [4]Interval{
{30, 3600}, // first hour -> 30 sec between versions
{3600, 86400}, // next day -> 1 h between versions
@ -102,12 +100,12 @@ func (v *Staggered) Stop() {
}
func (v *Staggered) clean() {
l.Debugln("Versioner clean: Waiting for lock on", v.versionsPath)
l.Debugln("Versioner clean: Waiting for lock on", v.versionsFs)
v.mutex.Lock()
defer v.mutex.Unlock()
l.Debugln("Versioner clean: Cleaning", v.versionsPath)
l.Debugln("Versioner clean: Cleaning", v.versionsFs)
if _, err := os.Stat(v.versionsPath); os.IsNotExist(err) {
if _, err := v.versionsFs.Stat("."); fs.IsNotExist(err) {
// There is no need to clean a nonexistent dir.
return
}
@ -115,14 +113,14 @@ func (v *Staggered) clean() {
versionsPerFile := make(map[string][]string)
filesPerDir := make(map[string]int)
err := filepath.Walk(v.versionsPath, func(path string, f os.FileInfo, err error) error {
err := v.versionsFs.Walk(".", func(path string, f fs.FileInfo, err error) error {
if err != nil {
return err
}
if f.Mode().IsDir() && f.Mode()&os.ModeSymlink == 0 {
if f.IsDir() && !f.IsSymlink() {
filesPerDir[path] = 0
if path != v.versionsPath {
if path != "." {
dir := filepath.Dir(path)
filesPerDir[dir]++
}
@ -155,25 +153,20 @@ func (v *Staggered) clean() {
continue
}
if path == v.versionsPath {
l.Debugln("Cleaner: versions dir is empty, don't delete", path)
continue
}
l.Debugln("Cleaner: deleting empty directory", path)
err = os.Remove(path)
err = v.versionsFs.Remove(path)
if err != nil {
l.Warnln("Versioner: can't remove directory", path, err)
}
}
l.Debugln("Cleaner: Finished cleaning", v.versionsPath)
l.Debugln("Cleaner: Finished cleaning", v.versionsFs)
}
func (v *Staggered) expire(versions []string) {
l.Debugln("Versioner: Expiring versions", versions)
for _, file := range v.toRemove(versions, time.Now()) {
if fi, err := osutil.Lstat(file); err != nil {
if fi, err := v.versionsFs.Lstat(file); err != nil {
l.Warnln("versioner:", err)
continue
} else if fi.IsDir() {
@ -181,7 +174,7 @@ func (v *Staggered) expire(versions []string) {
continue
}
if err := os.Remove(file); err != nil {
if err := v.versionsFs.Remove(file); err != nil {
l.Warnf("Versioner: can't remove %q: %v", file, err)
}
}
@ -203,7 +196,7 @@ func (v *Staggered) toRemove(versions []string, now time.Time) []string {
// If the file is older than the max age of the last interval, remove it
if lastIntv := v.interval[len(v.interval)-1]; lastIntv.end > 0 && age > lastIntv.end {
l.Debugln("Versioner: File over maximum age -> delete ", file)
err = os.Remove(file)
err = v.versionsFs.Remove(file)
if err != nil {
l.Warnf("Versioner: can't remove %q: %v", file, err)
}
@ -240,26 +233,26 @@ func (v *Staggered) toRemove(versions []string, now time.Time) []string {
// Archive moves the named file away to a version archive. If this function
// returns nil, the named file does not exist any more (has been archived).
func (v *Staggered) Archive(filePath string) error {
l.Debugln("Waiting for lock on ", v.versionsPath)
l.Debugln("Waiting for lock on ", v.versionsFs)
v.mutex.Lock()
defer v.mutex.Unlock()
info, err := osutil.Lstat(filePath)
if os.IsNotExist(err) {
info, err := v.folderFs.Lstat(filePath)
if fs.IsNotExist(err) {
l.Debugln("not archiving nonexistent file", filePath)
return nil
} else if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 {
if info.IsSymlink() {
panic("bug: attempting to version a symlink")
}
if _, err := os.Stat(v.versionsPath); err != nil {
if os.IsNotExist(err) {
l.Debugln("creating versions dir", v.versionsPath)
osutil.MkdirAll(v.versionsPath, 0755)
osutil.HideFile(v.versionsPath)
if _, err := v.versionsFs.Stat("."); err != nil {
if fs.IsNotExist(err) {
l.Debugln("creating versions dir", v.versionsFs)
v.versionsFs.MkdirAll(".", 0755)
v.versionsFs.Hide(".")
} else {
return err
}
@ -268,36 +261,41 @@ func (v *Staggered) Archive(filePath string) error {
l.Debugln("archiving", filePath)
file := filepath.Base(filePath)
inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath))
inFolderPath := filepath.Dir(filePath)
if err != nil {
return err
}
dir := filepath.Join(v.versionsPath, inFolderPath)
err = osutil.MkdirAll(dir, 0755)
if err != nil && !os.IsExist(err) {
err = v.versionsFs.MkdirAll(inFolderPath, 0755)
if err != nil && !fs.IsExist(err) {
return err
}
ver := taggedFilename(file, time.Now().Format(TimeFormat))
dst := filepath.Join(dir, ver)
dst := filepath.Join(inFolderPath, ver)
l.Debugln("moving to", dst)
err = osutil.Rename(filePath, dst)
/// TODO: Fix this when we have an alternative filesystem implementation
if v.versionsFs.Type() != fs.FilesystemTypeBasic {
panic("bug: staggered versioner used with unsupported filesystem")
}
err = os.Rename(filepath.Join(v.folderFs.URI(), filePath), filepath.Join(v.versionsFs.URI(), dst))
if err != nil {
return err
}
// Glob according to the new file~timestamp.ext pattern.
pattern := filepath.Join(dir, taggedFilename(file, TimeGlob))
newVersions, err := osutil.Glob(pattern)
pattern := filepath.Join(inFolderPath, taggedFilename(file, TimeGlob))
newVersions, err := v.versionsFs.Glob(pattern)
if err != nil {
l.Warnln("globbing:", err, "for", pattern)
return nil
}
// Also according to the old file.ext~timestamp pattern.
pattern = filepath.Join(dir, file+"~"+TimeGlob)
oldVersions, err := osutil.Glob(pattern)
pattern = filepath.Join(inFolderPath, file+"~"+TimeGlob)
oldVersions, err := v.versionsFs.Glob(pattern)
if err != nil {
l.Warnln("globbing:", err, "for", pattern)
return nil

3
lib/versioner/staggered_test.go
View File

@ -14,6 +14,7 @@ import (
"time"
"github.com/d4l3k/messagediff"
"github.com/syncthing/syncthing/lib/fs"
)
func TestStaggeredVersioningVersionCount(t *testing.T) {
@ -62,7 +63,7 @@ func TestStaggeredVersioningVersionCount(t *testing.T) {
os.MkdirAll("testdata/.stversions", 0755)
defer os.RemoveAll("testdata")
v := NewStaggered("", "testdata", map[string]string{"maxAge": strconv.Itoa(365 * 86400)}).(*Staggered)
v := NewStaggered("", fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), map[string]string{"maxAge": strconv.Itoa(365 * 86400)}).(*Staggered)
v.testCleanDone = make(chan struct{})
defer v.Stop()
go v.Serve()

51
lib/versioner/trashcan.go
View File

@ -8,11 +8,11 @@ package versioner
import (
"fmt"
"os"
"path/filepath"
"strconv"
"time"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
)
@ -22,17 +22,17 @@ func init() {
}
type Trashcan struct {
folderPath string
fs fs.Filesystem
cleanoutDays int
stop chan struct{}
}
func NewTrashcan(folderID, folderPath string, params map[string]string) Versioner {
func NewTrashcan(folderID string, fs fs.Filesystem, params map[string]string) Versioner {
cleanoutDays, _ := strconv.Atoi(params["cleanoutDays"])
// On error we default to 0, "do not clean out the trash can"
s := &Trashcan{
folderPath: folderPath,
fs: fs,
cleanoutDays: cleanoutDays,
stop: make(chan struct{}),
}
@ -44,52 +44,47 @@ func NewTrashcan(folderID, folderPath string, params map[string]string) Versione
// Archive moves the named file away to a version archive. If this function
// returns nil, the named file does not exist any more (has been archived).
func (t *Trashcan) Archive(filePath string) error {
info, err := osutil.Lstat(filePath)
if os.IsNotExist(err) {
info, err := t.fs.Lstat(filePath)
if fs.IsNotExist(err) {
l.Debugln("not archiving nonexistent file", filePath)
return nil
} else if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 {
if info.IsSymlink() {
panic("bug: attempting to version a symlink")
}
versionsDir := filepath.Join(t.folderPath, ".stversions")
if _, err := os.Stat(versionsDir); err != nil {
if !os.IsNotExist(err) {
versionsDir := ".stversions"
if _, err := t.fs.Stat(versionsDir); err != nil {
if !fs.IsNotExist(err) {
return err
}
l.Debugln("creating versions dir", versionsDir)
if err := osutil.MkdirAll(versionsDir, 0777); err != nil {
if err := t.fs.MkdirAll(versionsDir, 0777); err != nil {
return err
}
osutil.HideFile(versionsDir)
t.fs.Hide(versionsDir)
}
l.Debugln("archiving", filePath)
relativePath, err := filepath.Rel(t.folderPath, filePath)
if err != nil {
return err
}
archivedPath := filepath.Join(versionsDir, relativePath)
if err := osutil.MkdirAll(filepath.Dir(archivedPath), 0777); err != nil && !os.IsExist(err) {
archivedPath := filepath.Join(versionsDir, filePath)
if err := t.fs.MkdirAll(filepath.Dir(archivedPath), 0777); err != nil && !fs.IsExist(err) {
return err
}
l.Debugln("moving to", archivedPath)
if err := osutil.Rename(filePath, archivedPath); err != nil {
if err := osutil.Rename(t.fs, filePath, archivedPath); err != nil {
return err
}
// Set the mtime to the time the file was deleted. This is used by the
// cleanout routine. If this fails things won't work optimally but there's
// not much we can do about it so we ignore the error.
os.Chtimes(archivedPath, time.Now(), time.Now())
t.fs.Chtimes(archivedPath, time.Now(), time.Now())
return nil
}
@ -129,15 +124,15 @@ func (t *Trashcan) String() string {
}
func (t *Trashcan) cleanoutArchive() error {
versionsDir := filepath.Join(t.folderPath, ".stversions")
if _, err := osutil.Lstat(versionsDir); os.IsNotExist(err) {
versionsDir := ".stversions"
if _, err := t.fs.Lstat(versionsDir); fs.IsNotExist(err) {
return nil
}
cutoff := time.Now().Add(time.Duration(-24*t.cleanoutDays) * time.Hour)
currentDir := ""
filesInDir := 0
walkFn := func(path string, info os.FileInfo, err error) error {
walkFn := func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
@ -147,7 +142,7 @@ func (t *Trashcan) cleanoutArchive() error {
// directory was empty and try to remove it. We ignore failure for
// the time being.
if currentDir != "" && filesInDir == 0 {
os.Remove(currentDir)
t.fs.Remove(currentDir)
}
currentDir = path
filesInDir = 0
@ -156,7 +151,7 @@ func (t *Trashcan) cleanoutArchive() error {
if info.ModTime().Before(cutoff) {
// The file is too old; remove it.
os.Remove(path)
t.fs.Remove(path)
} else {
// Keep this file, and remember it so we don't unnecessarily try
// to remove this directory.
@ -165,14 +160,14 @@ func (t *Trashcan) cleanoutArchive() error {
return nil
}
if err := filepath.Walk(versionsDir, walkFn); err != nil {
if err := t.fs.Walk(versionsDir, walkFn); err != nil {
return err
}
// The last directory seen by the walkFn may not have been removed as it
// should be.
if currentDir != "" && filesInDir == 0 {
os.Remove(currentDir)
t.fs.Remove(currentDir)
}
return nil
}

4
lib/versioner/trashcan_test.go
View File

@ -12,6 +12,8 @@ import (
"path/filepath"
"testing"
"time"
"github.com/syncthing/syncthing/lib/fs"
)
func TestTrashcanCleanout(t *testing.T) {
@ -49,7 +51,7 @@ func TestTrashcanCleanout(t *testing.T) {
}
}
versioner := NewTrashcan("default", "testdata", map[string]string{"cleanoutDays": "7"}).(*Trashcan)
versioner := NewTrashcan("default", fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), map[string]string{"cleanoutDays": "7"}).(*Trashcan)
if err := versioner.cleanoutArchive(); err != nil {
t.Fatal(err)
}

4
lib/versioner/versioner.go
View File

@ -8,11 +8,13 @@
// simple default versioning scheme.
package versioner
import "github.com/syncthing/syncthing/lib/fs"
type Versioner interface {
Archive(filePath string) error
}
var Factories = map[string]func(folderID string, folderDir string, params map[string]string) Versioner{}
var Factories = map[string]func(folderID string, filesystem fs.Filesystem, params map[string]string) Versioner{}
const (
TimeFormat = "20060102-150405"

28
lib/weakhash/weakhash.go
View File

@ -9,7 +9,6 @@ package weakhash
import (
"bufio"
"io"
"os"
"github.com/chmduquesne/rollinghash/adler32"
)
@ -72,27 +71,21 @@ func Find(ir io.Reader, hashesToFind []uint32, size int) (map[uint32][]int64, er
return offsets, nil
}
func NewFinder(path string, size int, hashesToFind []uint32) (*Finder, error) {
file, err := os.Open(path)
func NewFinder(ir io.ReadSeeker, size int, hashesToFind []uint32) (*Finder, error) {
offsets, err := Find(ir, hashesToFind, size)
if err != nil {
return nil, err
}
offsets, err := Find(file, hashesToFind, size)
if err != nil {
file.Close()
return nil, err
}
return &Finder{
file: file,
reader: ir,
size: size,
offsets: offsets,
}, nil
}
type Finder struct {
file *os.File
reader io.ReadSeeker
size int
offsets map[uint32][]int64
}
@ -106,7 +99,11 @@ func (h *Finder) Iterate(hash uint32, buf []byte, iterFunc func(int64) bool) (bo
}
for _, offset := range h.offsets[hash] {
_, err := h.file.ReadAt(buf, offset)
_, err := h.reader.Seek(offset, io.SeekStart)
if err != nil {
return false, err
}
_, err = h.reader.Read(buf)
if err != nil {
return false, err
}
@ -116,10 +113,3 @@ func (h *Finder) Iterate(hash uint32, buf []byte, iterFunc func(int64) bool) (bo
}
return false, nil
}
// Close releases any resource associated with the finder
func (h *Finder) Close() {
if h != nil {
h.file.Close()
}
}

7
lib/weakhash/weakhash_test.go
View File

@ -11,6 +11,7 @@ package weakhash
import (
"bytes"
"io"
"io/ioutil"
"os"
"reflect"
@ -30,13 +31,15 @@ func TestFinder(t *testing.T) {
if _, err := f.Write(payload); err != nil {
t.Error(err)
}
if _, err := f.Seek(0, io.SeekStart); err != nil {
t.Error(err)
}
hashes := []uint32{65143183, 65798547}
finder, err := NewFinder(f.Name(), 4, hashes)
finder, err := NewFinder(f, 4, hashes)
if err != nil {
t.Error(err)
}
defer finder.Close()
expected := map[uint32][]int64{
65143183: {1, 27, 53, 79},

19
vendor/github.com/kballard/go-shellquote/LICENSE generated vendored Normal file
View File

@ -0,0 +1,19 @@
Copyright (C) 2014 Kevin Ballard
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

3
vendor/github.com/kballard/go-shellquote/doc.go generated vendored Normal file
View File

@ -0,0 +1,3 @@
// Shellquote provides utilities for joining/splitting strings using sh's
// word-splitting rules.
package shellquote

102
vendor/github.com/kballard/go-shellquote/quote.go generated vendored Normal file
View File

@ -0,0 +1,102 @@
package shellquote
import (
"bytes"
"strings"
"unicode/utf8"
)
// Join quotes each argument and joins them with a space.
// If passed to /bin/sh, the resulting string will be split back into the
// original arguments.
func Join(args ...string) string {
var buf bytes.Buffer
for i, arg := range args {
if i != 0 {
buf.WriteByte(' ')
}
quote(arg, &buf)
}
return buf.String()
}
const (
specialChars = "\\'\"`${[|&;<>()*?!"
extraSpecialChars = " \t\n"
prefixChars = "~"
)
func quote(word string, buf *bytes.Buffer) {
// We want to try to produce a "nice" output. As such, we will
// backslash-escape most characters, but if we encounter a space, or if we
// encounter an extra-special char (which doesn't work with
// backslash-escaping) we switch over to quoting the whole word. We do this
// with a space because it's typically easier for people to read multi-word
// arguments when quoted with a space rather than with ugly backslashes
// everywhere.
origLen := buf.Len()
if len(word) == 0 {
// oops, no content
buf.WriteString("''")
return
}
cur, prev := word, word
atStart := true
for len(cur) > 0 {
c, l := utf8.DecodeRuneInString(cur)
cur = cur[l:]
if strings.ContainsRune(specialChars, c) || (atStart && strings.ContainsRune(prefixChars, c)) {
// copy the non-special chars up to this point
if len(cur) < len(prev) {
buf.WriteString(prev[0 : len(prev)-len(cur)-l])
}
buf.WriteByte('\\')
buf.WriteRune(c)
prev = cur
} else if strings.ContainsRune(extraSpecialChars, c) {
// start over in quote mode
buf.Truncate(origLen)
goto quote
}
atStart = false
}
if len(prev) > 0 {
buf.WriteString(prev)
}
return
quote:
// quote mode
// Use single-quotes, but if we find a single-quote in the word, we need
// to terminate the string, emit an escaped quote, and start the string up
// again
inQuote := false
for len(word) > 0 {
i := strings.IndexRune(word, '\'')
if i == -1 {
break
}
if i > 0 {
if !inQuote {
buf.WriteByte('\'')
inQuote = true
}
buf.WriteString(word[0:i])
}
word = word[i+1:]
if inQuote {
buf.WriteByte('\'')
inQuote = false
}
buf.WriteString("\\'")
}
if len(word) > 0 {
if !inQuote {
buf.WriteByte('\'')
}
buf.WriteString(word)
buf.WriteByte('\'')
}
}

144
vendor/github.com/kballard/go-shellquote/unquote.go generated vendored Normal file
View File

@ -0,0 +1,144 @@
package shellquote
import (
"bytes"
"errors"
"strings"
"unicode/utf8"
)
var (
UnterminatedSingleQuoteError = errors.New("Unterminated single-quoted string")
UnterminatedDoubleQuoteError = errors.New("Unterminated double-quoted string")
UnterminatedEscapeError = errors.New("Unterminated backslash-escape")
)
var (
splitChars = " \n\t"
singleChar = '\''
doubleChar = '"'
escapeChar = '\\'
doubleEscapeChars = "$`\"\n\\"
)
// Split splits a string according to /bin/sh's word-splitting rules. It
// supports backslash-escapes, single-quotes, and double-quotes. Notably it does
// not support the $'' style of quoting. It also doesn't attempt to perform any
// other sort of expansion, including brace expansion, shell expansion, or
// pathname expansion.
//
// If the given input has an unterminated quoted string or ends in a
// backslash-escape, one of UnterminatedSingleQuoteError,
// UnterminatedDoubleQuoteError, or UnterminatedEscapeError is returned.
func Split(input string) (words []string, err error) {
var buf bytes.Buffer
words = make([]string, 0)
for len(input) > 0 {
// skip any splitChars at the start
c, l := utf8.DecodeRuneInString(input)
if strings.ContainsRune(splitChars, c) {
input = input[l:]
continue
}
var word string
word, input, err = splitWord(input, &buf)
if err != nil {
return
}
words = append(words, word)
}
return
}
func splitWord(input string, buf *bytes.Buffer) (word string, remainder string, err error) {
buf.Reset()
raw:
{
cur := input
for len(cur) > 0 {
c, l := utf8.DecodeRuneInString(cur)
cur = cur[l:]
if c == singleChar {
buf.WriteString(input[0 : len(input)-len(cur)-l])
input = cur
goto single
} else if c == doubleChar {
buf.WriteString(input[0 : len(input)-len(cur)-l])
input = cur
goto double
} else if c == escapeChar {
buf.WriteString(input[0 : len(input)-len(cur)-l])
input = cur
goto escape
} else if strings.ContainsRune(splitChars, c) {
buf.WriteString(input[0 : len(input)-len(cur)-l])
return buf.String(), cur, nil
}
}
if len(input) > 0 {
buf.WriteString(input)
input = ""
}
goto done
}
escape:
{
if len(input) == 0 {
return "", "", UnterminatedEscapeError
}
c, l := utf8.DecodeRuneInString(input)
if c == '\n' {
// a backslash-escaped newline is elided from the output entirely
} else {
buf.WriteString(input[:l])
}
input = input[l:]
}
goto raw
single:
{
i := strings.IndexRune(input, singleChar)
if i == -1 {
return "", "", UnterminatedSingleQuoteError
}
buf.WriteString(input[0:i])
input = input[i+1:]
goto raw
}
double:
{
cur := input
for len(cur) > 0 {
c, l := utf8.DecodeRuneInString(cur)
cur = cur[l:]
if c == doubleChar {
buf.WriteString(input[0 : len(input)-len(cur)-l])
input = cur
goto raw
} else if c == escapeChar {
// bash only supports certain escapes in double-quoted strings
c2, l2 := utf8.DecodeRuneInString(cur)
cur = cur[l2:]
if strings.ContainsRune(doubleEscapeChars, c2) {
buf.WriteString(input[0 : len(input)-len(cur)-l-l2])
if c2 == '\n' {
// newline is special, skip the backslash entirely
} else {
buf.WriteRune(c2)
}
input = cur
}
}
}
return "", "", UnterminatedDoubleQuoteError
}
done:
return buf.String(), input, nil
}

8
vendor/manifest vendored
View File

@ -249,6 +249,14 @@
"branch": "master",
"notests": true
},
{
"importpath": "github.com/kballard/go-shellquote",
"repository": "https://github.com/kballard/go-shellquote",
"vcs": "git",
"revision": "cd60e84ee657ff3dc51de0b4f55dd299a3e136f2",
"branch": "master",
"notests": true
},
{
"importpath": "github.com/klauspost/cpuid",
"repository": "https://github.com/klauspost/cpuid",