diff --git a/gui/default/syncthing/core/syncthingController.js b/gui/default/syncthing/core/syncthingController.js index 9fdcefc8a..b60f6822d 100755 --- a/gui/default/syncthing/core/syncthingController.js +++ b/gui/default/syncthing/core/syncthingController.js @@ -1318,6 +1318,7 @@ angular.module('syncthing.core') rescanIntervalS: 60, minDiskFreePct: 1, maxConflicts: 10, + fsync: true, order: "random", fileVersioningSelector: "none", trashcanClean: 0, @@ -1345,6 +1346,7 @@ angular.module('syncthing.core') rescanIntervalS: 60, minDiskFreePct: 1, maxConflicts: 10, + fsync: true, order: "random", fileVersioningSelector: "none", trashcanClean: 0, diff --git a/lib/config/config.go b/lib/config/config.go index b59590542..090ef2e01 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -26,7 +26,7 @@ import ( const ( OldestHandledVersion = 10 - CurrentVersion = 16 + CurrentVersion = 17 MaxRescanIntervalS = 365 * 24 * 60 * 60 ) @@ -254,6 +254,9 @@ func (cfg *Configuration) clean() error { if cfg.Version == 15 { convertV15V16(cfg) } + if cfg.Version == 16 { + convertV16V17(cfg) + } // Build a list of available devices existingDevices := make(map[protocol.DeviceID]bool) @@ -327,6 +330,14 @@ func convertV15V16(cfg *Configuration) { cfg.Version = 16 } +func convertV16V17(cfg *Configuration) { + for i := range cfg.Folders { + cfg.Folders[i].Fsync = true + } + + cfg.Version = 17 +} + func convertV13V14(cfg *Configuration) { // Not using the ignore cache is the new default. Disable it on existing // configurations. diff --git a/lib/config/config_test.go b/lib/config/config_test.go index 1e8767628..70c50de58 100644 --- a/lib/config/config_test.go +++ b/lib/config/config_test.go @@ -104,6 +104,7 @@ func TestDeviceConfig(t *testing.T) { AutoNormalize: true, MinDiskFreePct: 1, MaxConflicts: -1, + Fsync: true, Versioning: VersioningConfiguration{ Params: map[string]string{}, }, diff --git a/lib/config/folderconfiguration.go b/lib/config/folderconfiguration.go index 2a7bec92d..355b31e1a 100644 --- a/lib/config/folderconfiguration.go +++ b/lib/config/folderconfiguration.go @@ -38,6 +38,7 @@ type FolderConfiguration struct { MaxConflicts int `xml:"maxConflicts" json:"maxConflicts"` DisableSparseFiles bool `xml:"disableSparseFiles" json:"disableSparseFiles"` DisableTempIndexes bool `xml:"disableTempIndexes" json:"disableTempIndexes"` + Fsync bool `xml:"fsync" json:"fsync"` cachedPath string @@ -85,6 +86,9 @@ func (f *FolderConfiguration) CreateMarker() error { return err } fd.Close() + if err := osutil.SyncDir(filepath.Dir(marker)); err != nil { + l.Infof("fsync %q failed: %v", filepath.Dir(marker), err) + } osutil.HideFile(marker) } diff --git a/lib/config/testdata/v17.xml b/lib/config/testdata/v17.xml new file mode 100644 index 000000000..52e29a2b1 --- /dev/null +++ b/lib/config/testdata/v17.xml @@ -0,0 +1,15 @@ + + + + + 1 + -1 + true + + +
tcp://a
+ + +
tcp://b
+ + diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go index 5e6ce3612..1b178c739 100644 --- a/lib/model/rwfolder.go +++ b/lib/model/rwfolder.go @@ -91,6 +91,7 @@ type rwFolder struct { allowSparse bool checkFreeSpace bool ignoreDelete bool + fsync bool copiers int pullers int @@ -126,6 +127,7 @@ func newRWFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Ver allowSparse: !cfg.DisableSparseFiles, checkFreeSpace: cfg.MinDiskFreePct != 0, ignoreDelete: cfg.IgnoreDelete, + fsync: cfg.Fsync, queue: newJobQueue(), pullTimer: time.NewTimer(time.Second), @@ -1372,12 +1374,50 @@ func (f *rwFolder) dbUpdaterRoutine() { tick := time.NewTicker(maxBatchTime) defer tick.Stop() + var changedFiles []string + var changedDirs []string + if f.fsync { + changedFiles = make([]string, 0, maxBatchSize) + changedDirs = make([]string, 0, maxBatchSize) + } + + syncFilesOnce := func(files []string, syncFn func(string) error) { + sort.Strings(files) + var lastFile string + for _, file := range files { + if lastFile == file { + continue + } + lastFile = file + if err := syncFn(file); err != nil { + l.Infof("fsync %q failed: %v", file, err) + } + } + } + handleBatch := func() { found := false var lastFile protocol.FileInfo for _, job := range batch { files = append(files, job.file) + if f.fsync { + // collect changed files and dirs + switch job.jobType { + case dbUpdateHandleFile, dbUpdateShortcutFile: + // fsyncing symlinks is only supported by MacOS + if !job.file.IsSymlink() { + changedFiles = append(changedFiles, + filepath.Join(f.dir, job.file.Name)) + } + case dbUpdateHandleDir: + changedDirs = append(changedDirs, filepath.Join(f.dir, job.file.Name)) + } + if job.jobType != dbUpdateShortcutFile { + changedDirs = append(changedDirs, + filepath.Dir(filepath.Join(f.dir, job.file.Name))) + } + } if job.file.IsInvalid() || (job.file.IsDirectory() && !job.file.IsSymlink()) { continue } @@ -1390,6 +1430,14 @@ func (f *rwFolder) dbUpdaterRoutine() { lastFile = job.file } + if f.fsync { + // sync files and dirs to disk + syncFilesOnce(changedFiles, osutil.SyncFile) + changedFiles = changedFiles[:0] + syncFilesOnce(changedDirs, osutil.SyncDir) + changedDirs = changedDirs[:0] + } + // All updates to file/folder objects that originated remotely // (across the network) use this call to updateLocals f.model.updateLocalsFromPulling(f.folderID, files) diff --git a/lib/osutil/atomic.go b/lib/osutil/atomic.go index c244823aa..a5385c95f 100644 --- a/lib/osutil/atomic.go +++ b/lib/osutil/atomic.go @@ -77,6 +77,11 @@ func (w *AtomicWriter) Close() error { // Try to not leave temp file around, but ignore error. defer os.Remove(w.next.Name()) + if err := w.next.Sync(); err != nil { + w.err = err + return err + } + if err := w.next.Close(); err != nil { w.err = err return err @@ -97,6 +102,8 @@ func (w *AtomicWriter) Close() error { return err } + SyncDir(filepath.Dir(w.next.Name())) + // Set w.err to return appropriately for any future operations. w.err = ErrClosed diff --git a/lib/osutil/sync.go b/lib/osutil/sync.go new file mode 100644 index 000000000..97f3ec890 --- /dev/null +++ b/lib/osutil/sync.go @@ -0,0 +1,37 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package osutil + +import ( + "os" + "runtime" +) + +func SyncFile(path string) error { + flag := 0 + if runtime.GOOS == "windows" { + flag = os.O_WRONLY + } + fd, err := os.OpenFile(path, flag, 0) + if err != nil { + return err + } + defer fd.Close() + // MacOS and Windows do not flush the disk cache + if err := fd.Sync(); err != nil { + return err + } + return nil +} + +func SyncDir(path string) error { + if runtime.GOOS == "windows" { + // not supported by Windows + return nil + } + return SyncFile(path) +}