octoleo/syncthing
1
0
Fork 0
mirror of https://github.com/octoleo/syncthing.git synced 2024-09-19 21:29:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

lib/model: Cleanup redundant filesystem variables in folders (#7237)

Browse Source
This commit is contained in:
Simon Frei 2020-12-27 22:26:25 +01:00 committed by GitHub
parent 5440d1dc3b
commit a05dc6cc47
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 79 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
lib/model/folder.go
View File

@ -45,6 +45,7 @@ type folder struct {
shortID protocol.ShortID shortID protocol.ShortID
fset *db.FileSet fset *db.FileSet
ignores *ignore.Matcher ignores *ignore.Matcher
mtimefs fs.Filesystem
modTimeWindow time.Duration modTimeWindow time.Duration
ctx context.Context // used internally, only accessible on serve lifetime ctx context.Context // used internally, only accessible on serve lifetime
done chan struct{} // used externally, accessible regardless of serve done chan struct{} // used externally, accessible regardless of serve
@ -100,6 +101,7 @@ func newFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg conf
shortID: model.shortID, shortID: model.shortID,
fset: fset, fset: fset,
ignores: ignores, ignores: ignores,
mtimefs: fset.MtimeFS(),
modTimeWindow: cfg.ModTimeWindow(), modTimeWindow: cfg.ModTimeWindow(),
done: make(chan struct{}), done: make(chan struct{}),
@ -457,7 +459,6 @@ func (f *folder) scanSubdirs(subDirs []string) error {
// to be cancelled. // to be cancelled.
scanCtx, scanCancel := context.WithCancel(f.ctx) scanCtx, scanCancel := context.WithCancel(f.ctx)
defer scanCancel() defer scanCancel()
mtimefs := f.fset.MtimeFS()
scanConfig := scanner.Config{ scanConfig := scanner.Config{
Folder: f.ID, Folder: f.ID,
@ -465,7 +466,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
Matcher: f.ignores, Matcher: f.ignores,
TempLifetime: time.Duration(f.model.cfg.Options().KeepTemporariesH) * time.Hour, TempLifetime: time.Duration(f.model.cfg.Options().KeepTemporariesH) * time.Hour,
CurrentFiler: cFiler{snap}, CurrentFiler: cFiler{snap},
Filesystem: mtimefs, Filesystem: f.mtimefs,
IgnorePerms: f.IgnorePerms, IgnorePerms: f.IgnorePerms,
AutoNormalize: f.AutoNormalize, AutoNormalize: f.AutoNormalize,
Hashers: f.model.numHashers(f.ID), Hashers: f.model.numHashers(f.ID),
@ -530,8 +531,8 @@ func (f *folder) scanSubdirs(subDirs []string) error {
// We don't track it, but check if anything still exists // We don't track it, but check if anything still exists
// within and delete it otherwise. // within and delete it otherwise.
if fi.IsDirectory() && protocol.IsEncryptedParent(fi.Name) { if fi.IsDirectory() && protocol.IsEncryptedParent(fi.Name) {
if names, err := mtimefs.DirNames(fi.Name); err == nil && len(names) == 0 { if names, err := f.mtimefs.DirNames(fi.Name); err == nil && len(names) == 0 {
mtimefs.Remove(fi.Name) f.mtimefs.Remove(fi.Name)
} }
changes-- changes--
return return
@ -570,7 +571,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
switch f.Type { switch f.Type {
case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted: case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted:
default: default:
if nf, ok := f.findRename(snap, mtimefs, res.File, alreadyUsed); ok { if nf, ok := f.findRename(snap, res.File, alreadyUsed); ok {
batchAppend(nf, snap) batchAppend(nf, snap)
changes++ changes++
} }
@ -658,7 +659,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
// it's still here. Simply stat:ing it wont do as there are // it's still here. Simply stat:ing it wont do as there are
// tons of corner cases (e.g. parent dir->symlink, missing // tons of corner cases (e.g. parent dir->symlink, missing
// permissions) // permissions)
if !osutil.IsDeleted(mtimefs, file.Name) { if !osutil.IsDeleted(f.mtimefs, file.Name) {
if ignoredParent != "" { if ignoredParent != "" {
// Don't ignore parents of this not ignored item // Don't ignore parents of this not ignored item
toIgnore = toIgnore[:0] toIgnore = toIgnore[:0]
@ -727,7 +728,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
return nil return nil
} }
func (f *folder) findRename(snap *db.Snapshot, mtimefs fs.Filesystem, file protocol.FileInfo, alreadyUsed map[string]struct{}) (protocol.FileInfo, bool) { func (f *folder) findRename(snap *db.Snapshot, file protocol.FileInfo, alreadyUsed map[string]struct{}) (protocol.FileInfo, bool) {
if len(file.Blocks) == 0 || file.Size == 0 { if len(file.Blocks) == 0 || file.Size == 0 {
return protocol.FileInfo{}, false return protocol.FileInfo{}, false
} }
@ -763,7 +764,7 @@ func (f *folder) findRename(snap *db.Snapshot, mtimefs fs.Filesystem, file proto
return true return true
} }
if !osutil.IsDeleted(mtimefs, fi.Name) { if !osutil.IsDeleted(f.mtimefs, fi.Name) {
return true return true
} }

6
lib/model/folder_recvenc.go
View File

@ -27,8 +27,8 @@ type receiveEncryptedFolder struct {
*sendReceiveFolder *sendReceiveFolder
} }
func newReceiveEncryptedFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem, evLogger events.Logger, ioLimiter *byteSemaphore) service { func newReceiveEncryptedFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *byteSemaphore) service {
return &receiveEncryptedFolder{newSendReceiveFolder(model, fset, ignores, cfg, ver, fs, evLogger, ioLimiter).(*sendReceiveFolder)} return &receiveEncryptedFolder{newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)}
} }
func (f *receiveEncryptedFolder) Revert() { func (f *receiveEncryptedFolder) Revert() {
@ -65,7 +65,7 @@ func (f *receiveEncryptedFolder) revert() {
return true return true
} }
if err := f.inWritableDir(f.fs.Remove, fit.Name); err != nil && !fs.IsNotExist(err) { if err := f.inWritableDir(f.mtimefs.Remove, fit.Name); err != nil && !fs.IsNotExist(err) {
f.newScanError(fit.Name, fmt.Errorf("deleting unexpected item: %w", err)) f.newScanError(fit.Name, fmt.Errorf("deleting unexpected item: %w", err))
} }

5
lib/model/folder_recvonly.go
View File

@ -13,7 +13,6 @@ import (
"github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore" "github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/versioner" "github.com/syncthing/syncthing/lib/versioner"
@ -57,8 +56,8 @@ type receiveOnlyFolder struct {
*sendReceiveFolder *sendReceiveFolder
} }
func newReceiveOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem, evLogger events.Logger, ioLimiter *byteSemaphore) service { func newReceiveOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *byteSemaphore) service {
sr := newSendReceiveFolder(model, fset, ignores, cfg, ver, fs, evLogger, ioLimiter).(*sendReceiveFolder) sr := newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)
sr.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files sr.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files
return &receiveOnlyFolder{sr} return &receiveOnlyFolder{sr}
} }

3
lib/model/folder_sendonly.go
View File

@ -10,7 +10,6 @@ import (
"github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore" "github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/versioner" "github.com/syncthing/syncthing/lib/versioner"
@ -24,7 +23,7 @@ type sendOnlyFolder struct {
folder folder
} }
func newSendOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, _ versioner.Versioner, _ fs.Filesystem, evLogger events.Logger, ioLimiter *byteSemaphore) service { func newSendOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, _ versioner.Versioner, evLogger events.Logger, ioLimiter *byteSemaphore) service {
f := &sendOnlyFolder{ f := &sendOnlyFolder{
folder: newFolder(model, fset, ignores, cfg, evLogger, ioLimiter, nil), folder: newFolder(model, fset, ignores, cfg, evLogger, ioLimiter, nil),
} }

101
lib/model/folder_sendrecv.go
View File

@ -121,8 +121,6 @@ type dbUpdateJob struct {
type sendReceiveFolder struct { type sendReceiveFolder struct {
folder folder
fs fs.Filesystem
queue *jobQueue queue *jobQueue
blockPullReorderer blockPullReorderer blockPullReorderer blockPullReorderer
writeLimiter *byteSemaphore writeLimiter *byteSemaphore
@ -130,10 +128,9 @@ type sendReceiveFolder struct {
tempPullErrors map[string]string // pull errors that might be just transient tempPullErrors map[string]string // pull errors that might be just transient
} }
func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem, evLogger events.Logger, ioLimiter *byteSemaphore) service { func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *byteSemaphore) service {
f := &sendReceiveFolder{ f := &sendReceiveFolder{
folder: newFolder(model, fset, ignores, cfg, evLogger, ioLimiter, ver), folder: newFolder(model, fset, ignores, cfg, evLogger, ioLimiter, ver),
fs: fs,
queue: newJobQueue(), queue: newJobQueue(),
blockPullReorderer: newBlockPullReorderer(cfg.BlockPullOrder, model.id, cfg.DeviceIDs()), blockPullReorderer: newBlockPullReorderer(cfg.BlockPullOrder, model.id, cfg.DeviceIDs()),
writeLimiter: newByteSemaphore(cfg.MaxConcurrentWrites), writeLimiter: newByteSemaphore(cfg.MaxConcurrentWrites),
@ -578,7 +575,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, snap *db.Snapshot,
l.Debugf("need dir\n\t%v\n\t%v", file, curFile) l.Debugf("need dir\n\t%v\n\t%v", file, curFile)
} }
info, err := f.fs.Lstat(file.Name) info, err := f.mtimefs.Lstat(file.Name)
switch { switch {
// There is already something under that name, we need to handle that. // There is already something under that name, we need to handle that.
// Unless it already is a directory, as we only track permissions, // Unless it already is a directory, as we only track permissions,
@ -617,7 +614,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, snap *db.Snapshot,
// we can pass it to InWritableDir. We use a regular Mkdir and // we can pass it to InWritableDir. We use a regular Mkdir and
// not MkdirAll because the parent should already exist. // not MkdirAll because the parent should already exist.
mkdir := func(path string) error { mkdir := func(path string) error {
err = f.fs.Mkdir(path, mode) err = f.mtimefs.Mkdir(path, mode)
if err != nil || f.IgnorePerms || file.NoPermissions { if err != nil || f.IgnorePerms || file.NoPermissions {
return err return err
} }
@ -628,14 +625,14 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, snap *db.Snapshot,
} }
// Stat the directory so we can check its permissions. // Stat the directory so we can check its permissions.
info, err := f.fs.Lstat(path) info, err := f.mtimefs.Lstat(path)
if err != nil { if err != nil {
return err return err
} }
// Mask for the bits we want to preserve and add them in to the // Mask for the bits we want to preserve and add them in to the
// directories permissions. // directories permissions.
return f.fs.Chmod(path, mode|(info.Mode()&retainBits)) return f.mtimefs.Chmod(path, mode|(info.Mode()&retainBits))
} }
if err = f.inWritableDir(mkdir, file.Name); err == nil { if err = f.inWritableDir(mkdir, file.Name); err == nil {
@ -655,7 +652,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, snap *db.Snapshot,
// don't handle modification times on directories, because that sucks...) // don't handle modification times on directories, because that sucks...)
// It's OK to change mode bits on stuff within non-writable directories. // It's OK to change mode bits on stuff within non-writable directories.
if !f.IgnorePerms && !file.NoPermissions { if !f.IgnorePerms && !file.NoPermissions {
if err := f.fs.Chmod(file.Name, mode|(info.Mode()&retainBits)); err != nil { if err := f.mtimefs.Chmod(file.Name, mode|(info.Mode()&retainBits)); err != nil {
f.newPullError(file.Name, err) f.newPullError(file.Name, err)
return return
} }
@ -668,7 +665,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, snap *db.Snapshot,
func (f *sendReceiveFolder) checkParent(file string, scanChan chan<- string) bool { func (f *sendReceiveFolder) checkParent(file string, scanChan chan<- string) bool {
parent := filepath.Dir(file) parent := filepath.Dir(file)
if err := osutil.TraversesSymlink(f.fs, parent); err != nil { if err := osutil.TraversesSymlink(f.mtimefs, parent); err != nil {
f.newPullError(file, errors.Wrap(err, "checking parent dirs")) f.newPullError(file, errors.Wrap(err, "checking parent dirs"))
return false return false
} }
@ -688,12 +685,12 @@ func (f *sendReceiveFolder) checkParent(file string, scanChan chan<- string) boo
// And if this is an encrypted folder: // And if this is an encrypted folder:
// Encrypted files have made-up filenames with two synthetic parent // Encrypted files have made-up filenames with two synthetic parent
// directories which don't have any meaning. Create those if necessary. // directories which don't have any meaning. Create those if necessary.
if _, err := f.fs.Lstat(parent); !fs.IsNotExist(err) { if _, err := f.mtimefs.Lstat(parent); !fs.IsNotExist(err) {
l.Debugf("%v parent not missing %v", f, file) l.Debugf("%v parent not missing %v", f, file)
return true return true
} }
l.Debugf("%v creating parent directory of %v", f, file) l.Debugf("%v creating parent directory of %v", f, file)
if err := f.fs.MkdirAll(parent, 0755); err != nil { if err := f.mtimefs.MkdirAll(parent, 0755); err != nil {
f.newPullError(file, errors.Wrap(err, "creating parent dir")) f.newPullError(file, errors.Wrap(err, "creating parent dir"))
return false return false
} }
@ -746,7 +743,7 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, snap *db.Snaps
// We declare a function that acts on only the path name, so // We declare a function that acts on only the path name, so
// we can pass it to InWritableDir. // we can pass it to InWritableDir.
createLink := func(path string) error { createLink := func(path string) error {
if err := f.fs.CreateSymlink(file.SymlinkTarget, path); err != nil { if err := f.mtimefs.CreateSymlink(file.SymlinkTarget, path); err != nil {
return err return err
} }
return f.maybeCopyOwner(path) return f.maybeCopyOwner(path)
@ -761,7 +758,7 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, snap *db.Snaps
func (f *sendReceiveFolder) handleSymlinkCheckExisting(file protocol.FileInfo, snap *db.Snapshot, scanChan chan<- string) error { func (f *sendReceiveFolder) handleSymlinkCheckExisting(file protocol.FileInfo, snap *db.Snapshot, scanChan chan<- string) error {
// If there is already something under that name, we need to handle that. // If there is already something under that name, we need to handle that.
info, err := f.fs.Lstat(file.Name) info, err := f.mtimefs.Lstat(file.Name)
if err != nil { if err != nil {
if fs.IsNotExist(err) { if fs.IsNotExist(err) {
return nil return nil
@ -892,7 +889,7 @@ func (f *sendReceiveFolder) deleteFileWithCurrent(file, cur protocol.FileInfo, h
if f.versioner != nil && !cur.IsSymlink() { if f.versioner != nil && !cur.IsSymlink() {
err = f.inWritableDir(f.versioner.Archive, file.Name) err = f.inWritableDir(f.versioner.Archive, file.Name)
} else { } else {
err = f.inWritableDir(f.fs.Remove, file.Name) err = f.inWritableDir(f.mtimefs.Remove, file.Name)
} }
if err == nil || fs.IsNotExist(err) { if err == nil || fs.IsNotExist(err) {
@ -901,7 +898,7 @@ func (f *sendReceiveFolder) deleteFileWithCurrent(file, cur protocol.FileInfo, h
return return
} }
if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) { if _, serr := f.mtimefs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) {
// We get an error just looking at the file, and it's not a permission // We get an error just looking at the file, and it's not a permission
// problem. Lets assume the error is in fact some variant of "file // problem. Lets assume the error is in fact some variant of "file
// does not exist" (possibly expressed as some parent being a file and // does not exist" (possibly expressed as some parent being a file and
@ -956,7 +953,7 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, sn
} }
// Check that the target corresponds to what we have in the DB // Check that the target corresponds to what we have in the DB
curTarget, ok := snap.Get(protocol.LocalDeviceID, target.Name) curTarget, ok := snap.Get(protocol.LocalDeviceID, target.Name)
switch stat, serr := f.fs.Lstat(target.Name); { switch stat, serr := f.mtimefs.Lstat(target.Name); {
case serr != nil: case serr != nil:
var caseErr *fs.ErrCaseConflict var caseErr *fs.ErrCaseConflict
switch { switch {
@ -983,7 +980,7 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, sn
err = errModified err = errModified
default: default:
var fi protocol.FileInfo var fi protocol.FileInfo
if fi, err = scanner.CreateFileInfo(stat, target.Name, f.fs); err == nil { if fi, err = scanner.CreateFileInfo(stat, target.Name, f.mtimefs); err == nil {
if !fi.IsEquivalentOptional(curTarget, f.modTimeWindow, f.IgnorePerms, true, protocol.LocalAllFlags) { if !fi.IsEquivalentOptional(curTarget, f.modTimeWindow, f.IgnorePerms, true, protocol.LocalAllFlags) {
// Target changed // Target changed
scanChan <- target.Name scanChan <- target.Name
@ -1000,13 +997,13 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, sn
if f.versioner != nil { if f.versioner != nil {
err = f.CheckAvailableSpace(uint64(source.Size)) err = f.CheckAvailableSpace(uint64(source.Size))
if err == nil { if err == nil {
err = osutil.Copy(f.CopyRangeMethod, f.fs, f.fs, source.Name, tempName) err = osutil.Copy(f.CopyRangeMethod, f.mtimefs, f.mtimefs, source.Name, tempName)
if err == nil { if err == nil {
err = f.inWritableDir(f.versioner.Archive, source.Name) err = f.inWritableDir(f.versioner.Archive, source.Name)
} }
} }
} else { } else {
err = osutil.RenameOrCopy(f.CopyRangeMethod, f.fs, f.fs, source.Name, tempName) err = osutil.RenameOrCopy(f.CopyRangeMethod, f.mtimefs, f.mtimefs, source.Name, tempName)
} }
if err != nil { if err != nil {
return err return err
@ -1081,12 +1078,12 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, snap *db.Snapshot
// Check for an old temporary file which might have some blocks we could // Check for an old temporary file which might have some blocks we could
// reuse. // reuse.
tempBlocks, err := scanner.HashFile(f.ctx, f.fs, tempName, file.BlockSize(), nil, false) tempBlocks, err := scanner.HashFile(f.ctx, f.mtimefs, tempName, file.BlockSize(), nil, false)
if err != nil { if err != nil {
var caseErr *fs.ErrCaseConflict var caseErr *fs.ErrCaseConflict
if errors.As(err, &caseErr) { if errors.As(err, &caseErr) {
if rerr := f.fs.Rename(caseErr.Real, tempName); rerr == nil { if rerr := f.mtimefs.Rename(caseErr.Real, tempName); rerr == nil {
tempBlocks, err = scanner.HashFile(f.ctx, f.fs, tempName, file.BlockSize(), nil, false) tempBlocks, err = scanner.HashFile(f.ctx, f.mtimefs, tempName, file.BlockSize(), nil, false)
} }
} }
} }
@ -1116,7 +1113,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, snap *db.Snapshot
// Otherwise, discard the file ourselves in order for the // Otherwise, discard the file ourselves in order for the
// sharedpuller not to panic when it fails to exclusively create a // sharedpuller not to panic when it fails to exclusively create a
// file which already exists // file which already exists
f.inWritableDir(f.fs.Remove, tempName) f.inWritableDir(f.mtimefs.Remove, tempName)
} }
} else { } else {
// Copy the blocks, as we don't want to shuffle them on the FileInfo // Copy the blocks, as we don't want to shuffle them on the FileInfo
@ -1133,7 +1130,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, snap *db.Snapshot
"action": "update", "action": "update",
}) })
s := newSharedPullerState(file, f.fs, f.folderID, tempName, blocks, reused, f.IgnorePerms || file.NoPermissions, hasCurFile, curFile, !f.DisableSparseFiles, !f.DisableFsync) s := newSharedPullerState(file, f.mtimefs, f.folderID, tempName, blocks, reused, f.IgnorePerms || file.NoPermissions, hasCurFile, curFile, !f.DisableSparseFiles, !f.DisableFsync)
l.Debugf("%v need file %s; copy %d, reused %v", f, file.Name, len(blocks), len(reused)) l.Debugf("%v need file %s; copy %d, reused %v", f, file.Name, len(blocks), len(reused))
@ -1208,13 +1205,13 @@ func (f *sendReceiveFolder) shortcutFile(file, curFile protocol.FileInfo, dbUpda
f.queue.Done(file.Name) f.queue.Done(file.Name)
if !f.IgnorePerms && !file.NoPermissions { if !f.IgnorePerms && !file.NoPermissions {
if err = f.fs.Chmod(file.Name, fs.FileMode(file.Permissions&0777)); err != nil { if err = f.mtimefs.Chmod(file.Name, fs.FileMode(file.Permissions&0777)); err != nil {
f.newPullError(file.Name, err) f.newPullError(file.Name, err)
return return
} }
} }
f.fs.Chtimes(file.Name, file.ModTime(), file.ModTime()) // never fails f.mtimefs.Chtimes(file.Name, file.ModTime(), file.ModTime()) // never fails
dbUpdateChan <- dbUpdateJob{file, dbUpdateShortcutFile} dbUpdateChan <- dbUpdateJob{file, dbUpdateShortcutFile}
} }
@ -1403,7 +1400,7 @@ func (f *sendReceiveFolder) initWeakHashFinder(state copyBlocksState) (*weakhash
return nil, nil return nil, nil
} }
file, err := f.fs.Open(state.file.Name) file, err := f.mtimefs.Open(state.file.Name)
if err != nil { if err != nil {
l.Debugln("weak hasher", err) l.Debugln("weak hasher", err)
return nil, nil return nil, nil
@ -1551,7 +1548,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCurFile bool, tempName string, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error { func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCurFile bool, tempName string, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error {
// Set the correct permission bits on the new file // Set the correct permission bits on the new file
if !f.IgnorePerms && !file.NoPermissions { if !f.IgnorePerms && !file.NoPermissions {
if err := f.fs.Chmod(tempName, fs.FileMode(file.Permissions&0777)); err != nil { if err := f.mtimefs.Chmod(tempName, fs.FileMode(file.Permissions&0777)); err != nil {
return err return err
} }
} }
@ -1561,7 +1558,7 @@ func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCu
return err return err
} }
if stat, err := f.fs.Lstat(file.Name); err == nil { if stat, err := f.mtimefs.Lstat(file.Name); err == nil {
// There is an old file or directory already in place. We need to // There is an old file or directory already in place. We need to
// handle that. // handle that.
@ -1590,12 +1587,12 @@ func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCu
// Replace the original content with the new one. If it didn't work, // Replace the original content with the new one. If it didn't work,
// leave the temp file in place for reuse. // leave the temp file in place for reuse.
if err := osutil.RenameOrCopy(f.CopyRangeMethod, f.fs, f.fs, tempName, file.Name); err != nil { if err := osutil.RenameOrCopy(f.CopyRangeMethod, f.mtimefs, f.mtimefs, tempName, file.Name); err != nil {
return err return err
} }
// Set the correct timestamp on the new file // Set the correct timestamp on the new file
f.fs.Chtimes(file.Name, file.ModTime(), file.ModTime()) // never fails f.mtimefs.Chtimes(file.Name, file.ModTime(), file.ModTime()) // never fails
// Record the updated file in the index // Record the updated file in the index
dbUpdateChan <- dbUpdateJob{file, dbUpdateHandleFile} dbUpdateChan <- dbUpdateJob{file, dbUpdateHandleFile}
@ -1671,7 +1668,7 @@ func (f *sendReceiveFolder) dbUpdaterRoutine(dbUpdateChan <-chan dbUpdateJob) {
for dir := range changedDirs { for dir := range changedDirs {
delete(changedDirs, dir) delete(changedDirs, dir)
if !f.FolderConfiguration.DisableFsync { if !f.FolderConfiguration.DisableFsync {
fd, err := f.fs.Open(dir) fd, err := f.mtimefs.Open(dir)
if err != nil { if err != nil {
l.Debugf("fsync %q failed: %v", dir, err) l.Debugf("fsync %q failed: %v", dir, err)
continue continue
@ -1786,21 +1783,21 @@ func removeAvailability(availabilities []Availability, availability Availability
func (f *sendReceiveFolder) moveForConflict(name, lastModBy string, scanChan chan<- string) error { func (f *sendReceiveFolder) moveForConflict(name, lastModBy string, scanChan chan<- string) error {
if isConflict(name) { if isConflict(name) {
l.Infoln("Conflict for", name, "which is already a conflict copy; not copying again.") l.Infoln("Conflict for", name, "which is already a conflict copy; not copying again.")
if err := f.fs.Remove(name); err != nil && !fs.IsNotExist(err) { if err := f.mtimefs.Remove(name); err != nil && !fs.IsNotExist(err) {
return errors.Wrap(err, contextRemovingOldItem) return errors.Wrap(err, contextRemovingOldItem)
} }
return nil return nil
} }
if f.MaxConflicts == 0 { if f.MaxConflicts == 0 {
if err := f.fs.Remove(name); err != nil && !fs.IsNotExist(err) { if err := f.mtimefs.Remove(name); err != nil && !fs.IsNotExist(err) {
return errors.Wrap(err, contextRemovingOldItem) return errors.Wrap(err, contextRemovingOldItem)
} }
return nil return nil
} }
newName := conflictName(name, lastModBy) newName := conflictName(name, lastModBy)
err := f.fs.Rename(name, newName) err := f.mtimefs.Rename(name, newName)
if fs.IsNotExist(err) { if fs.IsNotExist(err) {
// We were supposed to move a file away but it does not exist. Either // We were supposed to move a file away but it does not exist. Either
// the user has already moved it away, or the conflict was between a // the user has already moved it away, or the conflict was between a
@ -1809,11 +1806,11 @@ func (f *sendReceiveFolder) moveForConflict(name, lastModBy string, scanChan cha
err = nil err = nil
} }
if f.MaxConflicts > -1 { if f.MaxConflicts > -1 {
matches := existingConflicts(name, f.fs) matches := existingConflicts(name, f.mtimefs)
if len(matches) > f.MaxConflicts { if len(matches) > f.MaxConflicts {
sort.Sort(sort.Reverse(sort.StringSlice(matches))) sort.Sort(sort.Reverse(sort.StringSlice(matches)))
for _, match := range matches[f.MaxConflicts:] { for _, match := range matches[f.MaxConflicts:] {
if gerr := f.fs.Remove(match); gerr != nil { if gerr := f.mtimefs.Remove(match); gerr != nil {
l.Debugln(f, "removing extra conflict", gerr) l.Debugln(f, "removing extra conflict", gerr)
} }
} }
@ -1871,17 +1868,17 @@ func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, snap *db.Sn
return f.inWritableDir(f.versioner.Archive, item.Name) return f.inWritableDir(f.versioner.Archive, item.Name)
} }
return f.inWritableDir(f.fs.Remove, item.Name) return f.inWritableDir(f.mtimefs.Remove, item.Name)
} }
// deleteDirOnDisk attempts to delete a directory. It checks for files/dirs inside // deleteDirOnDisk attempts to delete a directory. It checks for files/dirs inside
// the directory and removes them if possible or returns an error if it fails // the directory and removes them if possible or returns an error if it fails
func (f *sendReceiveFolder) deleteDirOnDisk(dir string, snap *db.Snapshot, scanChan chan<- string) error { func (f *sendReceiveFolder) deleteDirOnDisk(dir string, snap *db.Snapshot, scanChan chan<- string) error {
if err := osutil.TraversesSymlink(f.fs, filepath.Dir(dir)); err != nil { if err := osutil.TraversesSymlink(f.mtimefs, filepath.Dir(dir)); err != nil {
return err return err
} }
files, _ := f.fs.DirNames(dir) files, _ := f.mtimefs.DirNames(dir)
toBeDeleted := make([]string, 0, len(files)) toBeDeleted := make([]string, 0, len(files))
@ -1910,10 +1907,10 @@ func (f *sendReceiveFolder) deleteDirOnDisk(dir string, snap *db.Snapshot, scanC
hasReceiveOnlyChanged = true hasReceiveOnlyChanged = true
continue continue
} }
info, err := f.fs.Lstat(fullDirFile) info, err := f.mtimefs.Lstat(fullDirFile)
var diskFile protocol.FileInfo var diskFile protocol.FileInfo
if err == nil { if err == nil {
diskFile, err = scanner.CreateFileInfo(info, fullDirFile, f.fs) diskFile, err = scanner.CreateFileInfo(info, fullDirFile, f.mtimefs)
} }
if err != nil { if err != nil {
// Lets just assume the file has changed. // Lets just assume the file has changed.
@ -1950,15 +1947,15 @@ func (f *sendReceiveFolder) deleteDirOnDisk(dir string, snap *db.Snapshot, scanC
} }
for _, del := range toBeDeleted { for _, del := range toBeDeleted {
f.fs.RemoveAll(del) f.mtimefs.RemoveAll(del)
} }
err := f.inWritableDir(f.fs.Remove, dir) err := f.inWritableDir(f.mtimefs.Remove, dir)
if err == nil || fs.IsNotExist(err) { if err == nil || fs.IsNotExist(err) {
// It was removed or it doesn't exist to start with // It was removed or it doesn't exist to start with
return nil return nil
} }
if _, serr := f.fs.Lstat(dir); serr != nil && !fs.IsPermission(serr) { if _, serr := f.mtimefs.Lstat(dir); serr != nil && !fs.IsPermission(serr) {
// We get an error just looking at the directory, and it's not a // We get an error just looking at the directory, and it's not a
// permission problem. Lets assume the error is in fact some variant // permission problem. Lets assume the error is in fact some variant
// of "file does not exist" (possibly expressed as some parent being a // of "file does not exist" (possibly expressed as some parent being a
@ -1988,7 +1985,7 @@ func (f *sendReceiveFolder) scanIfItemChanged(name string, stat fs.FileInfo, ite
// to the database. If there's a mismatch here, there might be local // to the database. If there's a mismatch here, there might be local
// changes that we don't know about yet and we should scan before // changes that we don't know about yet and we should scan before
// touching the item. // touching the item.
statItem, err := scanner.CreateFileInfo(stat, item.Name, f.fs) statItem, err := scanner.CreateFileInfo(stat, item.Name, f.mtimefs)
if err != nil { if err != nil {
return errors.Wrap(err, "comparing item on disk to db") return errors.Wrap(err, "comparing item on disk to db")
} }
@ -2004,12 +2001,12 @@ func (f *sendReceiveFolder) scanIfItemChanged(name string, stat fs.FileInfo, ite
// in the DB before the caller proceeds with actually deleting it. // in the DB before the caller proceeds with actually deleting it.
// I.e. non-nil error status means "Do not delete!" or "is already deleted". // I.e. non-nil error status means "Do not delete!" or "is already deleted".
func (f *sendReceiveFolder) checkToBeDeleted(file, cur protocol.FileInfo, hasCur bool, scanChan chan<- string) error { func (f *sendReceiveFolder) checkToBeDeleted(file, cur protocol.FileInfo, hasCur bool, scanChan chan<- string) error {
if err := osutil.TraversesSymlink(f.fs, filepath.Dir(file.Name)); err != nil { if err := osutil.TraversesSymlink(f.mtimefs, filepath.Dir(file.Name)); err != nil {
l.Debugln(f, "not deleting item behind symlink on disk, but update db", file.Name) l.Debugln(f, "not deleting item behind symlink on disk, but update db", file.Name)
return fs.ErrNotExist return fs.ErrNotExist
} }
stat, err := f.fs.Lstat(file.Name) stat, err := f.mtimefs.Lstat(file.Name)
deleted := fs.IsNotExist(err) || fs.IsErrCaseConflict(err) deleted := fs.IsNotExist(err) || fs.IsErrCaseConflict(err)
if !deleted && err != nil { if !deleted && err != nil {
return err return err
@ -2036,18 +2033,18 @@ func (f *sendReceiveFolder) maybeCopyOwner(path string) error {
return nil return nil
} }
info, err := f.fs.Lstat(filepath.Dir(path)) info, err := f.mtimefs.Lstat(filepath.Dir(path))
if err != nil { if err != nil {
return errors.Wrap(err, "copy owner from parent") return errors.Wrap(err, "copy owner from parent")
} }
if err := f.fs.Lchown(path, info.Owner(), info.Group()); err != nil { if err := f.mtimefs.Lchown(path, info.Owner(), info.Group()); err != nil {
return errors.Wrap(err, "copy owner from parent") return errors.Wrap(err, "copy owner from parent")
} }
return nil return nil
} }
func (f *sendReceiveFolder) inWritableDir(fn func(string) error, path string) error { func (f *sendReceiveFolder) inWritableDir(fn func(string) error, path string) error {
return inWritableDir(fn, f.fs, path, f.IgnorePerms) return inWritableDir(fn, f.mtimefs, path, f.IgnorePerms)
} }
func (f *sendReceiveFolder) limitedWriteAt(fd io.WriterAt, data []byte, offset int64) error { func (f *sendReceiveFolder) limitedWriteAt(fd io.WriterAt, data []byte, offset int64) error {

23
lib/model/folder_sendrecv_test.go
View File

@ -807,12 +807,13 @@ func TestCopyOwner(t *testing.T) {
f.folder.FolderConfiguration = config.NewFolderConfiguration(m.id, f.ID, f.Label, fs.FilesystemTypeFake, "/TestCopyOwner") f.folder.FolderConfiguration = config.NewFolderConfiguration(m.id, f.ID, f.Label, fs.FilesystemTypeFake, "/TestCopyOwner")
f.folder.FolderConfiguration.CopyOwnershipFromParent = true f.folder.FolderConfiguration.CopyOwnershipFromParent = true
f.fs = f.Filesystem() f.fset = newFileSet(t, f.ID, f.Filesystem(), m.db)
f.mtimefs = f.fset.MtimeFS()
// Create a parent dir with a certain owner/group. // Create a parent dir with a certain owner/group.
f.fs.Mkdir("foo", 0755) f.mtimefs.Mkdir("foo", 0755)
f.fs.Lchown("foo", expOwner, expGroup) f.mtimefs.Lchown("foo", expOwner, expGroup)
dir := protocol.FileInfo{ dir := protocol.FileInfo{
Name: "foo/bar", Name: "foo/bar",
@ -833,7 +834,7 @@ func TestCopyOwner(t *testing.T) {
t.Fatal("Unexpected receive on scanChan:", toScan) t.Fatal("Unexpected receive on scanChan:", toScan)
} }
info, err := f.fs.Lstat("foo/bar") info, err := f.mtimefs.Lstat("foo/bar")
if err != nil { if err != nil {
t.Fatal("Unexpected error (dir):", err) t.Fatal("Unexpected error (dir):", err)
} }
@ -869,7 +870,7 @@ func TestCopyOwner(t *testing.T) {
f.handleFile(file, snap, copierChan) f.handleFile(file, snap, copierChan)
<-dbUpdateChan <-dbUpdateChan
info, err = f.fs.Lstat("foo/bar/baz") info, err = f.mtimefs.Lstat("foo/bar/baz")
if err != nil { if err != nil {
t.Fatal("Unexpected error (file):", err) t.Fatal("Unexpected error (file):", err)
} }
@ -892,7 +893,7 @@ func TestCopyOwner(t *testing.T) {
t.Fatal("Unexpected receive on scanChan:", toScan) t.Fatal("Unexpected receive on scanChan:", toScan)
} }
info, err = f.fs.Lstat("foo/bar/sym") info, err = f.mtimefs.Lstat("foo/bar/sym")
if err != nil { if err != nil {
t.Fatal("Unexpected error (file):", err) t.Fatal("Unexpected error (file):", err)
} }
@ -1215,7 +1216,7 @@ func TestPullTempFileCaseConflict(t *testing.T) {
file := protocol.FileInfo{Name: "foo"} file := protocol.FileInfo{Name: "foo"}
confl := "Foo" confl := "Foo"
tempNameConfl := fs.TempName(confl) tempNameConfl := fs.TempName(confl)
if fd, err := f.fs.Create(tempNameConfl); err != nil { if fd, err := f.mtimefs.Create(tempNameConfl); err != nil {
t.Fatal(err) t.Fatal(err)
} else { } else {
if _, err := fd.Write([]byte("data")); err != nil { if _, err := fd.Write([]byte("data")); err != nil {
@ -1239,7 +1240,7 @@ func TestPullCaseOnlyRename(t *testing.T) {
// tempNameConfl := fs.TempName(confl) // tempNameConfl := fs.TempName(confl)
name := "foo" name := "foo"
if fd, err := f.fs.Create(name); err != nil { if fd, err := f.mtimefs.Create(name); err != nil {
t.Fatal(err) t.Fatal(err)
} else { } else {
if _, err := fd.Write([]byte("data")); err != nil { if _, err := fd.Write([]byte("data")); err != nil {
@ -1280,7 +1281,7 @@ func TestPullSymlinkOverExistingWindows(t *testing.T) {
defer cleanupSRFolder(f, m) defer cleanupSRFolder(f, m)
name := "foo" name := "foo"
if fd, err := f.fs.Create(name); err != nil { if fd, err := f.mtimefs.Create(name); err != nil {
t.Fatal(err) t.Fatal(err)
} else { } else {
if _, err := fd.Write([]byte("data")); err != nil { if _, err := fd.Write([]byte("data")); err != nil {
@ -1308,7 +1309,7 @@ func TestPullSymlinkOverExistingWindows(t *testing.T) {
} else if !file.IsUnsupported() { } else if !file.IsUnsupported() {
t.Error("symlink entry isn't marked as unsupported") t.Error("symlink entry isn't marked as unsupported")
} }
if _, err := f.fs.Lstat(name); err == nil { if _, err := f.mtimefs.Lstat(name); err == nil {
t.Error("old file still exists on disk") t.Error("old file still exists on disk")
} else if !fs.IsNotExist(err) { } else if !fs.IsNotExist(err) {
t.Error(err) t.Error(err)
@ -1324,7 +1325,7 @@ func TestPullDeleteCaseConflict(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1) dbUpdateChan := make(chan dbUpdateJob, 1)
scanChan := make(chan string) scanChan := make(chan string)
if fd, err := f.fs.Create(name); err != nil { if fd, err := f.mtimefs.Create(name); err != nil {
t.Fatal(err) t.Fatal(err)
} else { } else {
if _, err := fd.Write([]byte("data")); err != nil { if _, err := fd.Write([]byte("data")); err != nil {

7
lib/model/model.go
View File

@ -164,7 +164,7 @@ type model struct {
started chan struct{} started chan struct{}
} }
type folderFactory func(*model, *db.FileSet, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, fs.Filesystem, events.Logger, *byteSemaphore) service type folderFactory func(*model, *db.FileSet, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, events.Logger, *byteSemaphore) service
var ( var (
folderFactories = make(map[config.FolderType]folderFactory) folderFactories = make(map[config.FolderType]folderFactory)
@ -384,8 +384,6 @@ func (m *model) addAndStartFolderLockedWithIgnores(cfg config.FolderConfiguratio
} }
} }
ffs := fset.MtimeFS()
if cfg.Type == config.FolderTypeReceiveEncrypted { if cfg.Type == config.FolderTypeReceiveEncrypted {
if encryptionToken, err := readEncryptionToken(cfg); err == nil { if encryptionToken, err := readEncryptionToken(cfg); err == nil {
m.folderEncryptionPasswordTokens[folder] = encryptionToken m.folderEncryptionPasswordTokens[folder] = encryptionToken
@ -395,6 +393,7 @@ func (m *model) addAndStartFolderLockedWithIgnores(cfg config.FolderConfiguratio
} }
// These are our metadata files, and they should always be hidden. // These are our metadata files, and they should always be hidden.
ffs := cfg.Filesystem()
_ = ffs.Hide(config.DefaultMarkerName) _ = ffs.Hide(config.DefaultMarkerName)
_ = ffs.Hide(".stversions") _ = ffs.Hide(".stversions")
_ = ffs.Hide(".stignore") _ = ffs.Hide(".stignore")
@ -409,7 +408,7 @@ func (m *model) addAndStartFolderLockedWithIgnores(cfg config.FolderConfiguratio
} }
m.folderVersioners[folder] = ver m.folderVersioners[folder] = ver
p := folderFactory(m, fset, ignores, cfg, ver, ffs, m.evLogger, m.folderIOLimiter) p := folderFactory(m, fset, ignores, cfg, ver, m.evLogger, m.folderIOLimiter)
m.folderRunners[folder] = p m.folderRunners[folder] = p