From dfaa99929180a0e64f688a9a0a39b83dde0fd1c4 Mon Sep 17 00:00:00 2001 From: Lode Hoste Date: Thu, 16 Jul 2015 12:52:36 +0200 Subject: [PATCH] Stop folder when running out of disk space (fixes #2057) & tweaks by calmh --- Godeps/Godeps.json | 4 +++ .../src/github.com/calmh/du/LICENSE | 24 +++++++++++++++++ .../src/github.com/calmh/du/README.md | 14 ++++++++++ .../src/github.com/calmh/du/cmd/du/main.go | 21 +++++++++++++++ .../src/github.com/calmh/du/diskusage.go | 8 ++++++ .../github.com/calmh/du/diskusage_posix.go | 24 +++++++++++++++++ .../calmh/du/diskusage_unsupported.go | 13 +++++++++ .../github.com/calmh/du/diskusage_windows.go | 27 +++++++++++++++++++ cmd/syncthing/main.go | 1 + internal/config/testdata/v11.xml | 13 +++++++++ lib/config/config.go | 14 +++++++++- lib/config/config_test.go | 1 + lib/config/wrapper.go | 4 +++ lib/model/model.go | 11 ++++++++ lib/model/rwfolder.go | 13 +++++++++ lib/osutil/osutil.go | 11 ++++++++ lib/osutil/osutil_test.go | 15 +++++++++++ 17 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 Godeps/_workspace/src/github.com/calmh/du/LICENSE create mode 100644 Godeps/_workspace/src/github.com/calmh/du/README.md create mode 100644 Godeps/_workspace/src/github.com/calmh/du/cmd/du/main.go create mode 100644 Godeps/_workspace/src/github.com/calmh/du/diskusage.go create mode 100644 Godeps/_workspace/src/github.com/calmh/du/diskusage_posix.go create mode 100644 Godeps/_workspace/src/github.com/calmh/du/diskusage_unsupported.go create mode 100644 Godeps/_workspace/src/github.com/calmh/du/diskusage_windows.go create mode 100644 internal/config/testdata/v11.xml diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index bd2f7fc55..33fb3b19a 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -9,6 +9,10 @@ "ImportPath": "github.com/bkaradzic/go-lz4", "Rev": "4f7c2045dbd17b802370e2e6022200468abf02ba" }, + { + "ImportPath": "github.com/calmh/du", + "Rev": "3c0690cca16228b97741327b1b6781397afbdb24" + }, { "ImportPath": "github.com/calmh/logger", "Rev": "c96f6a1a8c7b6bf2f4860c667867d90174799eb2" diff --git a/Godeps/_workspace/src/github.com/calmh/du/LICENSE b/Godeps/_workspace/src/github.com/calmh/du/LICENSE new file mode 100644 index 000000000..cf1ab25da --- /dev/null +++ b/Godeps/_workspace/src/github.com/calmh/du/LICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/Godeps/_workspace/src/github.com/calmh/du/README.md b/Godeps/_workspace/src/github.com/calmh/du/README.md new file mode 100644 index 000000000..0cb5e5d73 --- /dev/null +++ b/Godeps/_workspace/src/github.com/calmh/du/README.md @@ -0,0 +1,14 @@ +du +== + +Get total and available disk space on a given volume. + +Documentation +------------- + +http://godoc.org/github.com/calmh/du + +License +------- + +Public Domain diff --git a/Godeps/_workspace/src/github.com/calmh/du/cmd/du/main.go b/Godeps/_workspace/src/github.com/calmh/du/cmd/du/main.go new file mode 100644 index 000000000..3b8b54467 --- /dev/null +++ b/Godeps/_workspace/src/github.com/calmh/du/cmd/du/main.go @@ -0,0 +1,21 @@ +package main + +import ( + "fmt" + "log" + "os" + + "github.com/calmh/du" +) + +var KB = int64(1024) + +func main() { + usage, err := du.Get(os.Args[1]) + if err != nil { + log.Fatal(err) + } + fmt.Println("Free:", usage.FreeBytes/(KB*KB), "MiB") + fmt.Println("Available:", usage.AvailBytes/(KB*KB), "MiB") + fmt.Println("Size:", usage.TotalBytes/(KB*KB), "MiB") +} diff --git a/Godeps/_workspace/src/github.com/calmh/du/diskusage.go b/Godeps/_workspace/src/github.com/calmh/du/diskusage.go new file mode 100644 index 000000000..f6bb975ca --- /dev/null +++ b/Godeps/_workspace/src/github.com/calmh/du/diskusage.go @@ -0,0 +1,8 @@ +package du + +// Usage holds information about total and available storage on a volume. +type Usage struct { + TotalBytes int64 // Size of volume + FreeBytes int64 // Unused size + AvailBytes int64 // Available to a non-privileged user +} diff --git a/Godeps/_workspace/src/github.com/calmh/du/diskusage_posix.go b/Godeps/_workspace/src/github.com/calmh/du/diskusage_posix.go new file mode 100644 index 000000000..6502bdb68 --- /dev/null +++ b/Godeps/_workspace/src/github.com/calmh/du/diskusage_posix.go @@ -0,0 +1,24 @@ +// +build !windows,!netbsd,!openbsd,!solaris + +package du + +import ( + "path/filepath" + "syscall" +) + +// Get returns the Usage of a given path, or an error if usage data is +// unavailable. +func Get(path string) (Usage, error) { + var stat syscall.Statfs_t + err := syscall.Statfs(filepath.Clean(path), &stat) + if err != nil { + return Usage{}, err + } + u := Usage{ + FreeBytes: int64(stat.Bfree) * int64(stat.Bsize), + TotalBytes: int64(stat.Blocks) * int64(stat.Bsize), + AvailBytes: int64(stat.Bavail) * int64(stat.Bsize), + } + return u, nil +} diff --git a/Godeps/_workspace/src/github.com/calmh/du/diskusage_unsupported.go b/Godeps/_workspace/src/github.com/calmh/du/diskusage_unsupported.go new file mode 100644 index 000000000..044e77480 --- /dev/null +++ b/Godeps/_workspace/src/github.com/calmh/du/diskusage_unsupported.go @@ -0,0 +1,13 @@ +// +build netbsd openbsd solaris + +package du + +import "errors" + +var ErrUnsupported = errors.New("unsupported platform") + +// Get returns the Usage of a given path, or an error if usage data is +// unavailable. +func Get(path string) (Usage, error) { + return Usage{}, ErrUnsupported +} diff --git a/Godeps/_workspace/src/github.com/calmh/du/diskusage_windows.go b/Godeps/_workspace/src/github.com/calmh/du/diskusage_windows.go new file mode 100644 index 000000000..6ed5e52e2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/calmh/du/diskusage_windows.go @@ -0,0 +1,27 @@ +package du + +import ( + "syscall" + "unsafe" +) + +// Get returns the Usage of a given path, or an error if usage data is +// unavailable. +func Get(path string) (Usage, error) { + h := syscall.MustLoadDLL("kernel32.dll") + c := h.MustFindProc("GetDiskFreeSpaceExW") + + var u Usage + + ret, _, err := c.Call( + uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))), + uintptr(unsafe.Pointer(&u.FreeBytes)), + uintptr(unsafe.Pointer(&u.TotalBytes)), + uintptr(unsafe.Pointer(&u.AvailBytes))) + + if ret == 0 { + return u, err + } + + return u, nil +} diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index 19591880d..26c329273 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -837,6 +837,7 @@ func defaultConfig(myName string) config.Configuration { ID: "default", RawPath: locations[locDefFolder], RescanIntervalS: 60, + MinDiskFreePct: 1, Devices: []config.FolderDeviceConfiguration{{DeviceID: myID}}, }, } diff --git a/internal/config/testdata/v11.xml b/internal/config/testdata/v11.xml new file mode 100644 index 000000000..d6d60999c --- /dev/null +++ b/internal/config/testdata/v11.xml @@ -0,0 +1,13 @@ + + + + + 1 + + +
a
+ + +
b
+ + diff --git a/lib/config/config.go b/lib/config/config.go index 9eeea1cca..0d2dbd553 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -26,7 +26,7 @@ import ( const ( OldestHandledVersion = 5 - CurrentVersion = 10 + CurrentVersion = 11 MaxRescanIntervalS = 365 * 24 * 60 * 60 ) @@ -74,6 +74,7 @@ type FolderConfiguration struct { RescanIntervalS int `xml:"rescanIntervalS,attr" json:"rescanIntervalS"` IgnorePerms bool `xml:"ignorePerms,attr" json:"ignorePerms"` AutoNormalize bool `xml:"autoNormalize,attr" json:"autoNormalize"` + MinDiskFreePct int `xml:"minDiskFreePct" json:"minDiskFreePct"` Versioning VersioningConfiguration `xml:"versioning" json:"versioning"` Copiers int `xml:"copiers" json:"copiers"` // This defines how many files are handled concurrently. Pullers int `xml:"pullers" json:"pullers"` // Defines how many blocks are fetched at the same time, possibly between separate copier routines. @@ -364,6 +365,9 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) { if cfg.Version == 9 { convertV9V10(cfg) } + if cfg.Version == 10 { + convertV10V11(cfg) + } // Hash old cleartext passwords if len(cfg.GUI.Password) > 0 && cfg.GUI.Password[0] != '$' { @@ -460,6 +464,14 @@ func ChangeRequiresRestart(from, to Configuration) bool { return false } +func convertV10V11(cfg *Configuration) { + // Set minimum disk free of existing folders to 1% + for i := range cfg.Folders { + cfg.Folders[i].MinDiskFreePct = 1 + } + cfg.Version = 11 +} + func convertV9V10(cfg *Configuration) { // Enable auto normalization on existing folders. for i := range cfg.Folders { diff --git a/lib/config/config_test.go b/lib/config/config_test.go index 6220bce4c..fcadeef82 100644 --- a/lib/config/config_test.go +++ b/lib/config/config_test.go @@ -92,6 +92,7 @@ func TestDeviceConfig(t *testing.T) { Pullers: 16, Hashers: 0, AutoNormalize: true, + MinDiskFreePct: 1, }, } expectedDevices := []DeviceConfiguration{ diff --git a/lib/config/wrapper.go b/lib/config/wrapper.go index 25ea792bf..84799728e 100644 --- a/lib/config/wrapper.go +++ b/lib/config/wrapper.go @@ -96,6 +96,10 @@ func Load(path string, myID protocol.DeviceID) (*Wrapper, error) { return Wrap(path, cfg), nil } +func (w *Wrapper) ConfigPath() string { + return w.path +} + // Stop stops the Serve() loop. Set and Replace operations will panic after a // Stop. func (w *Wrapper) Stop() { diff --git a/lib/model/model.go b/lib/model/model.go index e8afc16ac..18fc2a5d9 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -45,6 +45,7 @@ const ( indexBatchSize = 1000 // Either way, don't include more files than this reqValidationTime = time.Hour // How long to cache validation entries for Request messages reqValidationCacheSize = 1000 // How many entries to aim for in the validation cache size + minHomeDiskFreePct = 1.0 // Stop when less space than this is available on the home (config & db) disk ) type service interface { @@ -1230,6 +1231,10 @@ func (m *Model) internalScanFolderSubs(folder string, subs []string) error { return errors.New("no such folder") } + if err := m.CheckFolderHealth(folder); err != nil { + return err + } + _ = ignores.Load(filepath.Join(folderCfg.Path(), ".stignore")) // Ignore error, there might not be an .stignore // Required to make sure that we start indexing at a directory we're already @@ -1658,6 +1663,10 @@ func (m *Model) BringToFront(folder, file string) { // CheckFolderHealth checks the folder for common errors and returns the // current folder error, or nil if the folder is healthy. func (m *Model) CheckFolderHealth(id string) error { + if free, err := osutil.DiskFreePercentage(m.cfg.ConfigPath()); err == nil && free < minHomeDiskFreePct { + return errors.New("out of disk space") + } + folder, ok := m.cfg.Folders()[id] if !ok { return errors.New("folder does not exist") @@ -1673,6 +1682,8 @@ func (m *Model) CheckFolderHealth(id string) error { err = errors.New("folder path missing") } else if !folder.HasMarker() { err = errors.New("folder marker missing") + } else if free, errDfp := osutil.DiskFreePercentage(folder.Path()); errDfp == nil && free < float64(folder.MinDiskFreePct) { + err = errors.New("out of disk space") } } else if os.IsNotExist(err) { // If we don't have any files in the index, and the directory diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go index af5942a51..8abfb95e1 100644 --- a/lib/model/rwfolder.go +++ b/lib/model/rwfolder.go @@ -437,6 +437,7 @@ func (p *rwFolder) pullerIteration(ignores *ignore.Matcher) int { // !!! changed := 0 + pullFileSize := int64(0) fileDeletions := map[string]protocol.FileInfo{} dirDeletions := []protocol.FileInfo{} @@ -485,6 +486,7 @@ func (p *rwFolder) pullerIteration(ignores *ignore.Matcher) int { default: // A new or changed file or symlink. This is the only case where we // do stuff concurrently in the background + pullFileSize += file.Size() p.queue.Push(file.Name, file.Size(), file.Modified) } @@ -492,6 +494,17 @@ func (p *rwFolder) pullerIteration(ignores *ignore.Matcher) int { return true }) + // Check if we are able to store all files on disk + if pullFileSize > 0 { + folder, ok := p.model.cfg.Folders()[p.folder] + if ok { + if free, err := osutil.DiskFreeBytes(folder.Path()); err == nil && free < pullFileSize { + l.Infof("Puller (folder %q): insufficient disk space available to pull %d files (%.2fMB)", p.folder, changed, float64(pullFileSize)/1024/1024) + return 0 + } + } + } + // Reorder the file queue according to configuration switch p.order { diff --git a/lib/osutil/osutil.go b/lib/osutil/osutil.go index d97e611f9..64802518e 100644 --- a/lib/osutil/osutil.go +++ b/lib/osutil/osutil.go @@ -16,6 +16,7 @@ import ( "runtime" "strings" + "github.com/calmh/du" "github.com/syncthing/syncthing/lib/sync" ) @@ -210,3 +211,13 @@ func init() { func IsWindowsExecutable(path string) bool { return execExts[strings.ToLower(filepath.Ext(path))] } + +func DiskFreeBytes(path string) (free int64, err error) { + u, err := du.Get(path) + return u.FreeBytes, err +} + +func DiskFreePercentage(path string) (freePct float64, err error) { + u, err := du.Get(path) + return (float64(u.FreeBytes) / float64(u.TotalBytes)) * 100, err +} diff --git a/lib/osutil/osutil_test.go b/lib/osutil/osutil_test.go index 5dfdce182..d4d5c15bc 100644 --- a/lib/osutil/osutil_test.go +++ b/lib/osutil/osutil_test.go @@ -164,3 +164,18 @@ func TestInWritableDirWindowsRename(t *testing.T) { } } } + +func TestDiskUsage(t *testing.T) { + free, err := osutil.DiskFreePercentage(".") + if err != nil { + if runtime.GOOS == "netbsd" || + runtime.GOOS == "openbsd" || + runtime.GOOS == "solaris" { + t.Skip() + } + t.Errorf("Unexpected error: %s", err) + } + if free < 1 { + t.Error("Disk is full?", free) + } +}