From f98c21b68ea6a4968d007c6fd0c7b48c7900cc95 Mon Sep 17 00:00:00 2001 From: Michael Ploujnikov Date: Fri, 20 Oct 2017 14:52:55 +0000 Subject: [PATCH] all: Add filesystem notification support GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3986 --- cmd/syncthing/main.go | 1 + gui/default/index.html | 8 +- .../syncthing/core/syncthingController.js | 2 +- jenkins/build-macos.bash | 3 + lib/config/config.go | 13 +- lib/config/config_test.go | 26 +- lib/config/folderconfiguration.go | 7 + lib/config/testdata/v25.xml | 16 + lib/fs/basicfs_test.go | 22 + lib/fs/basicfs_watch.go | 116 ++++ lib/fs/basicfs_watch_errors_linux.go | 18 + lib/fs/basicfs_watch_errors_others.go | 13 + lib/fs/basicfs_watch_eventtypes_fen.go | 17 + lib/fs/basicfs_watch_eventtypes_inotify.go | 17 + lib/fs/basicfs_watch_eventtypes_kqueue.go | 17 + lib/fs/basicfs_watch_eventtypes_other.go | 21 + lib/fs/basicfs_watch_eventtypes_readdcw.go | 17 + lib/fs/basicfs_watch_test.go | 295 +++++++++ lib/fs/basicfs_watch_unsupported.go | 15 + lib/fs/errorfs.go | 8 +- lib/fs/filesystem.go | 40 +- lib/fs/logfs.go | 7 + lib/model/folder.go | 36 ++ lib/model/model.go | 4 +- lib/model/rofolder.go | 13 + lib/model/rwfolder.go | 14 + lib/watchaggregator/aggregator.go | 438 +++++++++++++ lib/watchaggregator/aggregator_test.go | 281 +++++++++ lib/watchaggregator/debug.go | 24 + man/syncthing-rest-api.7 | 1 + vendor/github.com/zillode/notify/LICENSE | 21 + vendor/github.com/zillode/notify/debug.go | 11 + .../github.com/zillode/notify/debug_debug.go | 43 ++ vendor/github.com/zillode/notify/doc.go | 43 ++ vendor/github.com/zillode/notify/event.go | 143 +++++ vendor/github.com/zillode/notify/event_fen.go | 57 ++ .../zillode/notify/event_fsevents.go | 71 +++ .../zillode/notify/event_inotify.go | 75 +++ .../github.com/zillode/notify/event_kqueue.go | 59 ++ .../zillode/notify/event_readdcw.go | 118 ++++ .../github.com/zillode/notify/event_stub.go | 31 + .../zillode/notify/event_trigger.go | 22 + vendor/github.com/zillode/notify/node.go | 272 ++++++++ vendor/github.com/zillode/notify/notify.go | 83 +++ vendor/github.com/zillode/notify/tree.go | 22 + .../zillode/notify/tree_nonrecursive.go | 303 +++++++++ .../zillode/notify/tree_recursive.go | 355 +++++++++++ vendor/github.com/zillode/notify/util.go | 150 +++++ vendor/github.com/zillode/notify/watcher.go | 85 +++ .../github.com/zillode/notify/watcher_fen.go | 161 +++++ .../zillode/notify/watcher_fen_cgo.go | 141 +++++ .../zillode/notify/watcher_fsevents.go | 311 ++++++++++ .../zillode/notify/watcher_fsevents_cgo.go | 190 ++++++ .../zillode/notify/watcher_inotify.go | 405 ++++++++++++ .../zillode/notify/watcher_kqueue.go | 189 ++++++ .../zillode/notify/watcher_readdcw.go | 582 ++++++++++++++++++ .../github.com/zillode/notify/watcher_stub.go | 23 + .../zillode/notify/watcher_trigger.go | 449 ++++++++++++++ .../github.com/zillode/notify/watchpoint.go | 103 ++++ .../zillode/notify/watchpoint_other.go | 23 + .../zillode/notify/watchpoint_readdcw.go | 38 ++ vendor/manifest | 8 + 62 files changed, 6079 insertions(+), 18 deletions(-) create mode 100644 lib/config/testdata/v25.xml create mode 100644 lib/fs/basicfs_watch.go create mode 100644 lib/fs/basicfs_watch_errors_linux.go create mode 100644 lib/fs/basicfs_watch_errors_others.go create mode 100644 lib/fs/basicfs_watch_eventtypes_fen.go create mode 100644 lib/fs/basicfs_watch_eventtypes_inotify.go create mode 100644 lib/fs/basicfs_watch_eventtypes_kqueue.go create mode 100644 lib/fs/basicfs_watch_eventtypes_other.go create mode 100644 lib/fs/basicfs_watch_eventtypes_readdcw.go create mode 100644 lib/fs/basicfs_watch_test.go create mode 100644 lib/fs/basicfs_watch_unsupported.go create mode 100644 lib/watchaggregator/aggregator.go create mode 100644 lib/watchaggregator/aggregator_test.go create mode 100644 lib/watchaggregator/debug.go create mode 100644 vendor/github.com/zillode/notify/LICENSE create mode 100644 vendor/github.com/zillode/notify/debug.go create mode 100644 vendor/github.com/zillode/notify/debug_debug.go create mode 100644 vendor/github.com/zillode/notify/doc.go create mode 100644 vendor/github.com/zillode/notify/event.go create mode 100644 vendor/github.com/zillode/notify/event_fen.go create mode 100644 vendor/github.com/zillode/notify/event_fsevents.go create mode 100644 vendor/github.com/zillode/notify/event_inotify.go create mode 100644 vendor/github.com/zillode/notify/event_kqueue.go create mode 100644 vendor/github.com/zillode/notify/event_readdcw.go create mode 100644 vendor/github.com/zillode/notify/event_stub.go create mode 100644 vendor/github.com/zillode/notify/event_trigger.go create mode 100644 vendor/github.com/zillode/notify/node.go create mode 100644 vendor/github.com/zillode/notify/notify.go create mode 100644 vendor/github.com/zillode/notify/tree.go create mode 100644 vendor/github.com/zillode/notify/tree_nonrecursive.go create mode 100644 vendor/github.com/zillode/notify/tree_recursive.go create mode 100644 vendor/github.com/zillode/notify/util.go create mode 100644 vendor/github.com/zillode/notify/watcher.go create mode 100644 vendor/github.com/zillode/notify/watcher_fen.go create mode 100644 vendor/github.com/zillode/notify/watcher_fen_cgo.go create mode 100644 vendor/github.com/zillode/notify/watcher_fsevents.go create mode 100644 vendor/github.com/zillode/notify/watcher_fsevents_cgo.go create mode 100644 vendor/github.com/zillode/notify/watcher_inotify.go create mode 100644 vendor/github.com/zillode/notify/watcher_kqueue.go create mode 100644 vendor/github.com/zillode/notify/watcher_readdcw.go create mode 100644 vendor/github.com/zillode/notify/watcher_stub.go create mode 100644 vendor/github.com/zillode/notify/watcher_trigger.go create mode 100644 vendor/github.com/zillode/notify/watchpoint.go create mode 100644 vendor/github.com/zillode/notify/watchpoint_other.go create mode 100644 vendor/github.com/zillode/notify/watchpoint_readdcw.go diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index e86024686..891830fd7 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -1095,6 +1095,7 @@ func defaultConfig(myName string) config.Configuration { defaultFolder = config.NewFolderConfiguration("default", fs.FilesystemTypeBasic, locations[locDefFolder]) defaultFolder.Label = "Default Folder" defaultFolder.RescanIntervalS = 60 + defaultFolder.FSWatcherDelayS = 10 defaultFolder.MinDiskFree = config.Size{Value: 1, Unit: "%"} defaultFolder.Devices = []config.FolderDeviceConfiguration{{DeviceID: myID}} defaultFolder.AutoNormalize = true diff --git a/gui/default/index.html b/gui/default/index.html index c1f4d7116..a84e7c409 100644 --- a/gui/default/index.html +++ b/gui/default/index.html @@ -368,7 +368,13 @@ Yes - + +  Filesystem Notifications + + Yes + + +  Rescan Interval {{folder.rescanIntervalS}} s diff --git a/gui/default/syncthing/core/syncthingController.js b/gui/default/syncthing/core/syncthingController.js index c833415cd..64aa1c464 100755 --- a/gui/default/syncthing/core/syncthingController.js +++ b/gui/default/syncthing/core/syncthingController.js @@ -50,7 +50,6 @@ angular.module('syncthing.core') $scope.neededPageSize = 10; $scope.failed = {}; $scope.failedCurrentPage = 1; - $scope.failedCurrentFolder = undefined; $scope.failedPageSize = 10; $scope.scanProgress = {}; $scope.themes = []; @@ -66,6 +65,7 @@ angular.module('syncthing.core') selectedDevices: {}, type: "readwrite", rescanIntervalS: 60, + fsWatcherDelayS: 10, minDiskFree: {value: 1, unit: "%"}, maxConflicts: 10, fsync: true, diff --git a/jenkins/build-macos.bash b/jenkins/build-macos.bash index bc960b13f..9528ca19b 100755 --- a/jenkins/build-macos.bash +++ b/jenkins/build-macos.bash @@ -25,6 +25,9 @@ platforms=( darwin-amd64 darwin-386 ) +# Mac builds always require cgo +export CGO_ENABLED=1 + echo Building for plat in "${platforms[@]}"; do echo Building "$plat" diff --git a/lib/config/config.go b/lib/config/config.go index fec0ecb54..61ceb3ec6 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -32,7 +32,7 @@ import ( const ( OldestHandledVersion = 10 - CurrentVersion = 24 + CurrentVersion = 25 MaxRescanIntervalS = 365 * 24 * 60 * 60 ) @@ -326,6 +326,9 @@ func (cfg *Configuration) clean() error { if cfg.Version == 23 { convertV23V24(cfg) } + if cfg.Version == 24 { + convertV24V25(cfg) + } // Build a list of available devices existingDevices := make(map[protocol.DeviceID]bool) @@ -375,6 +378,14 @@ func (cfg *Configuration) clean() error { return nil } +func convertV24V25(cfg *Configuration) { + for i := range cfg.Folders { + cfg.Folders[i].FSWatcherDelayS = 10 + } + + cfg.Version = 25 +} + func convertV23V24(cfg *Configuration) { cfg.Options.URSeen = 2 diff --git a/lib/config/config_test.go b/lib/config/config_test.go index 07117449f..2cd5b5335 100644 --- a/lib/config/config_test.go +++ b/lib/config/config_test.go @@ -102,18 +102,20 @@ func TestDeviceConfig(t *testing.T) { expectedFolders := []FolderConfiguration{ { - ID: "test", - FilesystemType: fs.FilesystemTypeBasic, - Path: "testdata", - Devices: []FolderDeviceConfiguration{{DeviceID: device1}, {DeviceID: device4}}, - Type: FolderTypeSendOnly, - RescanIntervalS: 600, - Copiers: 0, - Pullers: 0, - Hashers: 0, - AutoNormalize: true, - MinDiskFree: Size{1, "%"}, - MaxConflicts: -1, + ID: "test", + FilesystemType: fs.FilesystemTypeBasic, + Path: "testdata", + Devices: []FolderDeviceConfiguration{{DeviceID: device1}, {DeviceID: device4}}, + Type: FolderTypeSendOnly, + RescanIntervalS: 600, + FSWatcherEnabled: false, + FSWatcherDelayS: 10, + Copiers: 0, + Pullers: 0, + Hashers: 0, + AutoNormalize: true, + MinDiskFree: Size{1, "%"}, + MaxConflicts: -1, Versioning: VersioningConfiguration{ Params: map[string]string{}, }, diff --git a/lib/config/folderconfiguration.go b/lib/config/folderconfiguration.go index 8e84e0db8..756ed333b 100644 --- a/lib/config/folderconfiguration.go +++ b/lib/config/folderconfiguration.go @@ -22,6 +22,8 @@ type FolderConfiguration struct { Type FolderType `xml:"type,attr" json:"type"` Devices []FolderDeviceConfiguration `xml:"device" json:"devices"` RescanIntervalS int `xml:"rescanIntervalS,attr" json:"rescanIntervalS"` + FSWatcherEnabled bool `xml:"fsWatcherEnabled,attr" json:"fsWatcherEnabled"` + FSWatcherDelayS int `xml:"fsWatcherDelayS,attr" json:"fsWatcherDelayS"` IgnorePerms bool `xml:"ignorePerms,attr" json:"ignorePerms"` AutoNormalize bool `xml:"autoNormalize,attr" json:"autoNormalize"` MinDiskFree Size `xml:"minDiskFree" json:"minDiskFree"` @@ -157,6 +159,11 @@ func (f *FolderConfiguration) prepare() { f.RescanIntervalS = 0 } + if f.FSWatcherDelayS <= 0 { + f.FSWatcherEnabled = false + f.FSWatcherDelayS = 10 + } + if f.Versioning.Params == nil { f.Versioning.Params = make(map[string]string) } diff --git a/lib/config/testdata/v25.xml b/lib/config/testdata/v25.xml new file mode 100644 index 000000000..f5e0551c7 --- /dev/null +++ b/lib/config/testdata/v25.xml @@ -0,0 +1,16 @@ + + + basic + + + 1 + -1 + true + + +
tcp://a
+ + +
tcp://b
+ + diff --git a/lib/fs/basicfs_test.go b/lib/fs/basicfs_test.go index 9d0c4c1ec..74313d62c 100644 --- a/lib/fs/basicfs_test.go +++ b/lib/fs/basicfs_test.go @@ -7,12 +7,14 @@ package fs import ( + "errors" "io/ioutil" "os" "path/filepath" "runtime" "sort" "strings" + "syscall" "testing" "time" ) @@ -484,3 +486,23 @@ func TestRooted(t *testing.T) { } } } + +func TestWatchErrorLinuxInterpretation(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("testing of linux specific error codes") + } + + var errTooManyFiles syscall.Errno = 24 + var errNoSpace syscall.Errno = 28 + + if !reachedMaxUserWatches(errTooManyFiles) { + t.Errorf("Errno %v should be recognised to be about inotify limits.", errTooManyFiles) + } + if !reachedMaxUserWatches(errNoSpace) { + t.Errorf("Errno %v should be recognised to be about inotify limits.", errNoSpace) + } + err := errors.New("Another error") + if reachedMaxUserWatches(err) { + t.Errorf("This error does not concern inotify limits: %#v", err) + } +} diff --git a/lib/fs/basicfs_watch.go b/lib/fs/basicfs_watch.go new file mode 100644 index 000000000..3557f6f9b --- /dev/null +++ b/lib/fs/basicfs_watch.go @@ -0,0 +1,116 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !solaris,!darwin solaris,cgo darwin,cgo + +package fs + +import ( + "context" + "errors" + "path/filepath" + + "github.com/zillode/notify" +) + +// Notify does not block on sending to channel, so the channel must be buffered. +// The actual number is magic. +// Not meant to be changed, but must be changeable for tests +var backendBuffer = 500 + +func (f *BasicFilesystem) Watch(name string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, error) { + absName, err := f.rooted(name) + if err != nil { + return nil, err + } + + absShouldIgnore := func(absPath string) bool { + return ignore.ShouldIgnore(f.unrootedChecked(absPath)) + } + + outChan := make(chan Event) + backendChan := make(chan notify.EventInfo, backendBuffer) + + eventMask := subEventMask + if !ignorePerms { + eventMask |= permEventMask + } + + if err := notify.WatchWithFilter(filepath.Join(absName, "..."), backendChan, absShouldIgnore, eventMask); err != nil { + notify.Stop(backendChan) + if reachedMaxUserWatches(err) { + err = errors.New("failed to install inotify handler. Please increase inotify limits, see https://github.com/syncthing/syncthing-inotify#troubleshooting-for-folders-with-many-files-on-linux for more information") + } + return nil, err + } + + go f.watchLoop(name, absName, backendChan, outChan, ignore, ctx) + + return outChan, nil +} + +func (f *BasicFilesystem) watchLoop(name string, absName string, backendChan chan notify.EventInfo, outChan chan<- Event, ignore Matcher, ctx context.Context) { + for { + // Detect channel overflow + if len(backendChan) == backendBuffer { + outer: + for { + select { + case <-backendChan: + default: + break outer + } + } + // When next scheduling a scan, do it on the entire folder as events have been lost. + outChan <- Event{Name: name, Type: NonRemove} + l.Debugln(f.Type(), f.URI(), "Watch: Event overflow, send \".\"") + } + + select { + case ev := <-backendChan: + relPath := f.unrootedChecked(ev.Path()) + if ignore.ShouldIgnore(relPath) { + l.Debugln(f.Type(), f.URI(), "Watch: Ignoring", relPath) + continue + } + evType := f.eventType(ev.Event()) + select { + case outChan <- Event{Name: relPath, Type: evType}: + l.Debugln(f.Type(), f.URI(), "Watch: Sending", relPath, evType) + case <-ctx.Done(): + notify.Stop(backendChan) + l.Debugln(f.Type(), f.URI(), "Watch: Stopped") + return + } + case <-ctx.Done(): + notify.Stop(backendChan) + l.Debugln(f.Type(), f.URI(), "Watch: Stopped") + return + } + } +} + +func (f *BasicFilesystem) eventType(notifyType notify.Event) EventType { + if notifyType&rmEventMask != 0 { + return Remove + } + return NonRemove +} + +// unrootedChecked returns the path relative to the folder root (same as +// unrooted). It panics if the given path is not a subpath and handles the +// special case when the given path is the folder root without a trailing +// pathseparator. +func (f *BasicFilesystem) unrootedChecked(absPath string) string { + if absPath+string(PathSeparator) == f.root { + return "." + } + relPath := f.unrooted(absPath) + if relPath == absPath { + panic("bug: Notify backend is processing a change outside of the watched path: " + absPath) + } + return relPath +} diff --git a/lib/fs/basicfs_watch_errors_linux.go b/lib/fs/basicfs_watch_errors_linux.go new file mode 100644 index 000000000..d61587a79 --- /dev/null +++ b/lib/fs/basicfs_watch_errors_linux.go @@ -0,0 +1,18 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build linux + +package fs + +import "syscall" + +func reachedMaxUserWatches(err error) bool { + if errno, ok := err.(syscall.Errno); ok { + return errno == 24 || errno == 28 + } + return false +} diff --git a/lib/fs/basicfs_watch_errors_others.go b/lib/fs/basicfs_watch_errors_others.go new file mode 100644 index 000000000..dc3ef38b8 --- /dev/null +++ b/lib/fs/basicfs_watch_errors_others.go @@ -0,0 +1,13 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !linux + +package fs + +func reachedMaxUserWatches(err error) bool { + return false +} diff --git a/lib/fs/basicfs_watch_eventtypes_fen.go b/lib/fs/basicfs_watch_eventtypes_fen.go new file mode 100644 index 000000000..2a25aab7e --- /dev/null +++ b/lib/fs/basicfs_watch_eventtypes_fen.go @@ -0,0 +1,17 @@ +// Copyright (C) 2017 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build solaris,cgo + +package fs + +import "github.com/zillode/notify" + +const ( + subEventMask = notify.Create | notify.FileModified | notify.FileRenameFrom | notify.FileDelete | notify.FileRenameTo + permEventMask = notify.FileAttrib + rmEventMask = notify.FileDelete | notify.FileRenameFrom +) diff --git a/lib/fs/basicfs_watch_eventtypes_inotify.go b/lib/fs/basicfs_watch_eventtypes_inotify.go new file mode 100644 index 000000000..00f6e5d42 --- /dev/null +++ b/lib/fs/basicfs_watch_eventtypes_inotify.go @@ -0,0 +1,17 @@ +// Copyright (C) 2017 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build linux + +package fs + +import "github.com/zillode/notify" + +const ( + subEventMask = notify.InCreate | notify.InMovedTo | notify.InDelete | notify.InDeleteSelf | notify.InModify | notify.InMovedFrom | notify.InMoveSelf + permEventMask = notify.InAttrib + rmEventMask = notify.InDelete | notify.InDeleteSelf | notify.InMovedFrom | notify.InMoveSelf +) diff --git a/lib/fs/basicfs_watch_eventtypes_kqueue.go b/lib/fs/basicfs_watch_eventtypes_kqueue.go new file mode 100644 index 000000000..892c74423 --- /dev/null +++ b/lib/fs/basicfs_watch_eventtypes_kqueue.go @@ -0,0 +1,17 @@ +// Copyright (C) 2017 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build dragonfly freebsd netbsd openbsd + +package fs + +import "github.com/zillode/notify" + +const ( + subEventMask = notify.NoteDelete | notify.NoteWrite | notify.NoteRename + permEventMask = notify.NoteAttrib + rmEventMask = notify.NoteDelete | notify.NoteRename +) diff --git a/lib/fs/basicfs_watch_eventtypes_other.go b/lib/fs/basicfs_watch_eventtypes_other.go new file mode 100644 index 000000000..7503ba510 --- /dev/null +++ b/lib/fs/basicfs_watch_eventtypes_other.go @@ -0,0 +1,21 @@ +// Copyright (C) 2017 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !linux,!windows,!dragonfly,!freebsd,!netbsd,!openbsd,!solaris +// +build !darwin darwin,cgo + +// Catch all platforms that are not specifically handled to use the generic +// event types. + +package fs + +import "github.com/zillode/notify" + +const ( + subEventMask = notify.All + permEventMask = 0 + rmEventMask = notify.Remove | notify.Rename +) diff --git a/lib/fs/basicfs_watch_eventtypes_readdcw.go b/lib/fs/basicfs_watch_eventtypes_readdcw.go new file mode 100644 index 000000000..54fb7a69f --- /dev/null +++ b/lib/fs/basicfs_watch_eventtypes_readdcw.go @@ -0,0 +1,17 @@ +// Copyright (C) 2017 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build windows + +package fs + +import "github.com/zillode/notify" + +const ( + subEventMask = notify.FileNotifyChangeFileName | notify.FileNotifyChangeDirName | notify.FileNotifyChangeSize | notify.FileNotifyChangeCreation + permEventMask = notify.FileNotifyChangeAttributes + rmEventMask = notify.FileActionRemoved | notify.FileActionRenamedOldName +) diff --git a/lib/fs/basicfs_watch_test.go b/lib/fs/basicfs_watch_test.go new file mode 100644 index 000000000..2461dec88 --- /dev/null +++ b/lib/fs/basicfs_watch_test.go @@ -0,0 +1,295 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !solaris,!darwin solaris,cgo darwin,cgo + +package fs + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "strconv" + "testing" + "time" + + "github.com/zillode/notify" +) + +func TestMain(m *testing.M) { + if err := os.RemoveAll(testDir); err != nil { + panic(err) + } + + dir, err := filepath.Abs(".") + if err != nil { + panic("Cannot get absolute path to working dir") + } + dir, err = filepath.EvalSymlinks(dir) + if err != nil { + panic("Cannot get real path to working dir") + } + testDirAbs = filepath.Join(dir, testDir) + testFs = newBasicFilesystem(testDirAbs) + if l.ShouldDebug("filesystem") { + testFs = &logFilesystem{testFs} + } + + backendBuffer = 10 + defer func() { + backendBuffer = 500 + }() + os.Exit(m.Run()) +} + +const ( + testDir = "temporary_test_root" +) + +var ( + testDirAbs string + testFs Filesystem +) + +func TestWatchIgnore(t *testing.T) { + name := "ignore" + + file := "file" + ignored := "ignored" + + testCase := func() { + createTestFile(name, file) + createTestFile(name, ignored) + } + + expectedEvents := []Event{ + {file, NonRemove}, + } + + testScenario(t, name, testCase, expectedEvents, false, ignored) +} + +func TestWatchRename(t *testing.T) { + name := "rename" + + old := createTestFile(name, "oldfile") + new := "newfile" + + testCase := func() { + renameTestFile(name, old, new) + } + + destEvent := Event{new, Remove} + // Only on these platforms the removed file can be differentiated from + // the created file during renaming + if runtime.GOOS == "windows" || runtime.GOOS == "linux" || runtime.GOOS == "solaris" { + destEvent = Event{new, NonRemove} + } + expectedEvents := []Event{ + {old, Remove}, + destEvent, + } + + testScenario(t, name, testCase, expectedEvents, false, "") +} + +// TestWatchOutside checks that no changes from outside the folder make it in +func TestWatchOutside(t *testing.T) { + outChan := make(chan Event) + backendChan := make(chan notify.EventInfo, backendBuffer) + + ctx, cancel := context.WithCancel(context.Background()) + + // testFs is Filesystem, but we need BasicFilesystem here + fs := newBasicFilesystem(testDirAbs) + + go func() { + defer func() { + if recover() == nil { + t.Fatalf("Watch did not panic on receiving event outside of folder") + } + cancel() + }() + fs.watchLoop(".", testDirAbs, backendChan, outChan, fakeMatcher{}, ctx) + }() + + backendChan <- fakeEventInfo(filepath.Join(filepath.Dir(testDirAbs), "outside")) +} + +func TestWatchSubpath(t *testing.T) { + outChan := make(chan Event) + backendChan := make(chan notify.EventInfo, backendBuffer) + + ctx, cancel := context.WithCancel(context.Background()) + + // testFs is Filesystem, but we need BasicFilesystem here + fs := newBasicFilesystem(testDirAbs) + + abs, _ := fs.rooted("sub") + go fs.watchLoop("sub", abs, backendChan, outChan, fakeMatcher{}, ctx) + + backendChan <- fakeEventInfo(filepath.Join(abs, "file")) + + timeout := time.NewTimer(2 * time.Second) + select { + case <-timeout.C: + t.Errorf("Timed out before receiving an event") + cancel() + case ev := <-outChan: + if ev.Name != filepath.Join("sub", "file") { + t.Errorf("While watching a subfolder, received an event with unexpected path %v", ev.Name) + } + } + + cancel() +} + +// TestWatchOverflow checks that an event at the root is sent when maxFiles is reached +func TestWatchOverflow(t *testing.T) { + name := "overflow" + + testCase := func() { + for i := 0; i < 5*backendBuffer; i++ { + createTestFile(name, "file"+strconv.Itoa(i)) + } + } + + expectedEvents := []Event{ + {".", NonRemove}, + } + + testScenario(t, name, testCase, expectedEvents, true, "") +} + +// path relative to folder root, also creates parent dirs if necessary +func createTestFile(name string, file string) string { + joined := filepath.Join(name, file) + if err := testFs.MkdirAll(filepath.Dir(joined), 0755); err != nil { + panic(fmt.Sprintf("Failed to create parent directory for %s: %s", joined, err)) + } + handle, err := testFs.Create(joined) + if err != nil { + panic(fmt.Sprintf("Failed to create test file %s: %s", joined, err)) + } + handle.Close() + return file +} + +func renameTestFile(name string, old string, new string) { + old = filepath.Join(name, old) + new = filepath.Join(name, new) + if err := testFs.Rename(old, new); err != nil { + panic(fmt.Sprintf("Failed to rename %s to %s: %s", old, new, err)) + } +} + +func sleepMs(ms int) { + time.Sleep(time.Duration(ms) * time.Millisecond) +} + +func testScenario(t *testing.T, name string, testCase func(), expectedEvents []Event, allowOthers bool, ignored string) { + if err := testFs.MkdirAll(name, 0755); err != nil { + panic(fmt.Sprintf("Failed to create directory %s: %s", name, err)) + } + + // Tests pick up the previously created files/dirs, probably because + // they get flushed to disk with a delay. + initDelayMs := 500 + if runtime.GOOS == "darwin" { + initDelayMs = 900 + } + sleepMs(initDelayMs) + + ctx, cancel := context.WithCancel(context.Background()) + + if ignored != "" { + ignored = filepath.Join(name, ignored) + } + + eventChan, err := testFs.Watch(name, fakeMatcher{ignored}, ctx, false) + if err != nil { + panic(err) + } + + go testWatchOutput(t, name, eventChan, expectedEvents, allowOthers, ctx, cancel) + + timeout := time.NewTimer(2 * time.Second) + + testCase() + + select { + case <-timeout.C: + t.Errorf("Timed out before receiving all expected events") + cancel() + case <-ctx.Done(): + } + + if err := testFs.RemoveAll(name); err != nil { + panic(fmt.Sprintf("Failed to remove directory %s: %s", name, err)) + } +} + +func testWatchOutput(t *testing.T, name string, in <-chan Event, expectedEvents []Event, allowOthers bool, ctx context.Context, cancel context.CancelFunc) { + var expected = make(map[Event]struct{}) + for _, ev := range expectedEvents { + ev.Name = filepath.Join(name, ev.Name) + expected[ev] = struct{}{} + } + + var received Event + var last Event + for { + if len(expected) == 0 { + cancel() + return + } + + select { + case <-ctx.Done(): + return + case received = <-in: + } + + // apparently the backend sometimes sends repeat events + if last == received { + continue + } + + if _, ok := expected[received]; !ok { + if allowOthers { + sleepMs(100) // To facilitate overflow + continue + } + t.Errorf("Received unexpected event %v expected one of %v", received, expected) + cancel() + return + } + delete(expected, received) + last = received + } +} + +type fakeMatcher struct{ match string } + +func (fm fakeMatcher) ShouldIgnore(name string) bool { + return name == fm.match +} + +type fakeEventInfo string + +func (e fakeEventInfo) Path() string { + return string(e) +} + +func (e fakeEventInfo) Event() notify.Event { + return notify.Write +} + +func (e fakeEventInfo) Sys() interface{} { + return nil +} diff --git a/lib/fs/basicfs_watch_unsupported.go b/lib/fs/basicfs_watch_unsupported.go new file mode 100644 index 000000000..77000a3fd --- /dev/null +++ b/lib/fs/basicfs_watch_unsupported.go @@ -0,0 +1,15 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build solaris,!cgo darwin,!cgo + +package fs + +import "context" + +func (f *BasicFilesystem) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, error) { + return nil, ErrWatchNotSupported +} diff --git a/lib/fs/errorfs.go b/lib/fs/errorfs.go index 8d38561c1..49ec8ad65 100644 --- a/lib/fs/errorfs.go +++ b/lib/fs/errorfs.go @@ -6,7 +6,10 @@ package fs -import "time" +import ( + "context" + "time" +) type errorFilesystem struct { err error @@ -39,3 +42,6 @@ func (fs *errorFilesystem) Roots() ([]string, error) func (fs *errorFilesystem) Usage(name string) (Usage, error) { return Usage{}, fs.err } func (fs *errorFilesystem) Type() FilesystemType { return fs.fsType } func (fs *errorFilesystem) URI() string { return fs.uri } +func (fs *errorFilesystem) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, error) { + return nil, fs.err +} diff --git a/lib/fs/filesystem.go b/lib/fs/filesystem.go index 7c2744fe9..f7f27e7d8 100644 --- a/lib/fs/filesystem.go +++ b/lib/fs/filesystem.go @@ -7,6 +7,7 @@ package fs import ( + "context" "errors" "io" "os" @@ -33,7 +34,8 @@ type Filesystem interface { Rename(oldname, newname string) error Stat(name string) (FileInfo, error) SymlinksSupported() bool - Walk(root string, walkFn WalkFunc) error + Walk(name string, walkFn WalkFunc) error + Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, error) Hide(name string) error Unhide(name string) error Glob(pattern string) ([]string, error) @@ -82,6 +84,42 @@ type Usage struct { Total int64 } +type Matcher interface { + ShouldIgnore(name string) bool +} + +type MatchResult interface { + IsIgnored() bool +} + +type Event struct { + Name string + Type EventType +} + +type EventType int + +const ( + NonRemove EventType = 1 + iota + Remove + Mixed // Should probably not be necessary to be used in filesystem interface implementation +) + +func (evType EventType) String() string { + switch { + case evType == NonRemove: + return "non-remove" + case evType == Remove: + return "remove" + case evType == Mixed: + return "mixed" + default: + panic("bug: Unknown event type") + } +} + +var ErrWatchNotSupported = errors.New("watching is not supported") + // Equivalents from os package. const ModePerm = FileMode(os.ModePerm) diff --git a/lib/fs/logfs.go b/lib/fs/logfs.go index 1a2df6256..b7ce67ec0 100644 --- a/lib/fs/logfs.go +++ b/lib/fs/logfs.go @@ -7,6 +7,7 @@ package fs import ( + "context" "fmt" "path/filepath" "runtime" @@ -127,6 +128,12 @@ func (fs *logFilesystem) Walk(root string, walkFn WalkFunc) error { return err } +func (fs *logFilesystem) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, error) { + evChan, err := fs.Filesystem.Watch(path, ignore, ctx, ignorePerms) + l.Debugln(getCaller(), fs.Type(), fs.URI(), "Watch", path, ignore, ignorePerms, err) + return evChan, err +} + func (fs *logFilesystem) Unhide(name string) error { err := fs.Filesystem.Unhide(name) l.Debugln(getCaller(), fs.Type(), fs.URI(), "Unhide", name, err) diff --git a/lib/model/folder.go b/lib/model/folder.go index e0dc1764f..b43bec8c3 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -11,6 +11,7 @@ import ( "time" "github.com/syncthing/syncthing/lib/config" + "github.com/syncthing/syncthing/lib/watchaggregator" ) type folder struct { @@ -22,6 +23,9 @@ type folder struct { ctx context.Context cancel context.CancelFunc initialScanFinished chan struct{} + watchCancel context.CancelFunc + watchChan chan []string + ignoresUpdated chan struct{} // The ignores changed, we need to restart watcher } func newFolder(model *Model, cfg config.FolderConfiguration) folder { @@ -92,3 +96,35 @@ func (f *folder) scanTimerFired() { f.scan.Reschedule() } + +func (f *folder) startWatcher() { + ctx, cancel := context.WithCancel(f.ctx) + f.model.fmut.RLock() + ignores := f.model.folderIgnores[f.folderID] + f.model.fmut.RUnlock() + eventChan, err := f.Filesystem().Watch(".", ignores, ctx, f.IgnorePerms) + if err != nil { + l.Warnf("Failed to start filesystem watcher for folder %s: %v", f.Description(), err) + } else { + f.watchChan = make(chan []string) + f.watchCancel = cancel + watchaggregator.Aggregate(eventChan, f.watchChan, f.FolderConfiguration, f.model.cfg, ctx) + l.Infoln("Started filesystem watcher for folder", f.Description()) + } +} + +func (f *folder) restartWatcher() { + f.watchCancel() + f.startWatcher() + f.Scan(nil) +} + +func (f *folder) IgnoresUpdated() { + select { + case f.ignoresUpdated <- struct{}{}: + default: + // We might be busy doing a pull and thus not reading from this + // channel. The channel is 1-buffered, so one notification will be + // queued to ensure we recheck after the pull. + } +} diff --git a/lib/model/model.go b/lib/model/model.go index e362479a0..1739b5e43 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -48,6 +48,7 @@ type service interface { BringToFront(string) DelayScan(d time.Duration) IndexUpdated() // Remote index was updated notification + IgnoresUpdated() // ignore matcher was updated notification Jobs() ([]string, []string) // In progress, Queued Scan(subs []string) error Serve() @@ -260,6 +261,7 @@ func (m *Model) startFolderLocked(folder string) config.FolderType { ffs.Hide(".stignore") p := folderFactory(m, cfg, ver, ffs) + m.folderRunners[folder] = p m.warnAboutOverwritingProtectedFiles(folder) @@ -1858,7 +1860,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su defer func() { if ignores.Hash() != oldHash { l.Debugln("Folder", folder, "ignore patterns changed; triggering puller") - runner.IndexUpdated() + runner.IgnoresUpdated() } }() diff --git a/lib/model/rofolder.go b/lib/model/rofolder.go index 27f1d55c4..20023e8cd 100644 --- a/lib/model/rofolder.go +++ b/lib/model/rofolder.go @@ -34,11 +34,20 @@ func (f *sendOnlyFolder) Serve() { f.scan.timer.Stop() }() + if f.FSWatcherEnabled { + f.startWatcher() + } + for { select { case <-f.ctx.Done(): return + case <-f.ignoresUpdated: + if f.FSWatcherEnabled { + f.restartWatcher() + } + case <-f.scan.timer.C: l.Debugln(f, "Scanning subdirectories") f.scanTimerFired() @@ -48,6 +57,10 @@ func (f *sendOnlyFolder) Serve() { case next := <-f.scan.delay: f.scan.timer.Reset(next) + + case fsEvents := <-f.watchChan: + l.Debugln(f, "filesystem notification rescan") + f.scanSubdirs(fsEvents) } } } diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go index 4d92d479c..21886cee9 100644 --- a/lib/model/rwfolder.go +++ b/lib/model/rwfolder.go @@ -164,6 +164,10 @@ func (f *sendReceiveFolder) Serve() { var prevSec int64 var prevIgnoreHash string + if f.FSWatcherEnabled { + f.startWatcher() + } + for { select { case <-f.ctx.Done(): @@ -174,6 +178,12 @@ func (f *sendReceiveFolder) Serve() { f.pullTimer.Reset(0) l.Debugln(f, "remote index updated, rescheduling pull") + case <-f.ignoresUpdated: + if f.FSWatcherEnabled { + f.restartWatcher() + } + f.IndexUpdated() + case <-f.pullTimer.C: select { case <-f.initialScanFinished: @@ -278,6 +288,10 @@ func (f *sendReceiveFolder) Serve() { case next := <-f.scan.delay: f.scan.timer.Reset(next) + + case fsEvents := <-f.watchChan: + l.Debugln(f, "filesystem notification rescan") + f.scanSubdirs(fsEvents) } } } diff --git a/lib/watchaggregator/aggregator.go b/lib/watchaggregator/aggregator.go new file mode 100644 index 000000000..ae09632ef --- /dev/null +++ b/lib/watchaggregator/aggregator.go @@ -0,0 +1,438 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package watchaggregator + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/syncthing/syncthing/lib/config" + "github.com/syncthing/syncthing/lib/events" + "github.com/syncthing/syncthing/lib/fs" +) + +// Not meant to be changed, but must be changeable for tests +var ( + maxFiles = 512 + maxFilesPerDir = 128 +) + +// aggregatedEvent represents potentially multiple events at and/or recursively +// below one path until it times out and a scan is scheduled. +type aggregatedEvent struct { + firstModTime time.Time + lastModTime time.Time + evType fs.EventType +} + +// Stores pointers to both aggregated events directly within this directory and +// child directories recursively containing aggregated events themselves. +type eventDir struct { + events map[string]*aggregatedEvent + dirs map[string]*eventDir +} + +func newEventDir() *eventDir { + return &eventDir{ + events: make(map[string]*aggregatedEvent), + dirs: make(map[string]*eventDir), + } +} + +func (dir *eventDir) eventCount() int { + count := len(dir.events) + for _, dir := range dir.dirs { + count += dir.eventCount() + } + return count +} + +func (dir *eventDir) childCount() int { + return len(dir.events) + len(dir.dirs) +} + +func (dir *eventDir) firstModTime() time.Time { + if dir.childCount() == 0 { + panic("bug: firstModTime must not be used on empty eventDir") + } + firstModTime := time.Now() + for _, childDir := range dir.dirs { + dirTime := childDir.firstModTime() + if dirTime.Before(firstModTime) { + firstModTime = dirTime + } + } + for _, event := range dir.events { + if event.firstModTime.Before(firstModTime) { + firstModTime = event.firstModTime + } + } + return firstModTime +} + +func (dir *eventDir) eventType() fs.EventType { + if dir.childCount() == 0 { + panic("bug: eventType must not be used on empty eventDir") + } + var evType fs.EventType + for _, childDir := range dir.dirs { + evType |= childDir.eventType() + if evType == fs.Mixed { + return fs.Mixed + } + } + for _, event := range dir.events { + evType |= event.evType + if evType == fs.Mixed { + return fs.Mixed + } + } + return evType +} + +type aggregator struct { + folderCfg config.FolderConfiguration + folderCfgUpdate chan config.FolderConfiguration + // Time after which an event is scheduled for scanning when no modifications occur. + notifyDelay time.Duration + // Time after which an event is scheduled for scanning even though modifications occur. + notifyTimeout time.Duration + notifyTimer *time.Timer + notifyTimerNeedsReset bool + notifyTimerResetChan chan time.Duration + ctx context.Context +} + +func new(folderCfg config.FolderConfiguration, ctx context.Context) *aggregator { + a := &aggregator{ + folderCfgUpdate: make(chan config.FolderConfiguration), + notifyTimerNeedsReset: false, + notifyTimerResetChan: make(chan time.Duration), + ctx: ctx, + } + + a.updateConfig(folderCfg) + + return a +} + +func Aggregate(in <-chan fs.Event, out chan<- []string, folderCfg config.FolderConfiguration, cfg *config.Wrapper, ctx context.Context) { + a := new(folderCfg, ctx) + + // Necessary for unit tests where the backend is mocked + go a.mainLoop(in, out, cfg) +} + +func (a *aggregator) mainLoop(in <-chan fs.Event, out chan<- []string, cfg *config.Wrapper) { + a.notifyTimer = time.NewTimer(a.notifyDelay) + defer a.notifyTimer.Stop() + + inProgress := make(map[string]struct{}) + inProgressItemSubscription := events.Default.Subscribe(events.ItemStarted | events.ItemFinished) + + cfg.Subscribe(a) + + rootEventDir := newEventDir() + + for { + select { + case event := <-in: + a.newEvent(event, rootEventDir, inProgress) + case event := <-inProgressItemSubscription.C(): + updateInProgressSet(event, inProgress) + case <-a.notifyTimer.C: + a.actOnTimer(rootEventDir, out) + case interval := <-a.notifyTimerResetChan: + a.resetNotifyTimer(interval) + case folderCfg := <-a.folderCfgUpdate: + a.updateConfig(folderCfg) + case <-a.ctx.Done(): + cfg.Unsubscribe(a) + l.Debugln(a, "Stopped") + return + } + } +} + +func (a *aggregator) newEvent(event fs.Event, rootEventDir *eventDir, inProgress map[string]struct{}) { + if _, ok := rootEventDir.events["."]; ok { + l.Debugln(a, "Will scan entire folder anyway; dropping:", event.Name) + return + } + if _, ok := inProgress[event.Name]; ok { + l.Debugln(a, "Skipping path we modified:", event.Name) + return + } + a.aggregateEvent(event, time.Now(), rootEventDir) +} + +func (a *aggregator) aggregateEvent(event fs.Event, evTime time.Time, rootEventDir *eventDir) { + if event.Name == "." || rootEventDir.eventCount() == maxFiles { + l.Debugln(a, "Scan entire folder") + firstModTime := evTime + if rootEventDir.childCount() != 0 { + event.Type |= rootEventDir.eventType() + firstModTime = rootEventDir.firstModTime() + } + rootEventDir.dirs = make(map[string]*eventDir) + rootEventDir.events = make(map[string]*aggregatedEvent) + rootEventDir.events["."] = &aggregatedEvent{ + firstModTime: firstModTime, + lastModTime: evTime, + evType: event.Type, + } + a.resetNotifyTimerIfNeeded() + return + } + + parentDir := rootEventDir + + // Check if any parent directory is already tracked or will exceed + // events per directory limit bottom up + pathSegments := strings.Split(filepath.ToSlash(event.Name), "/") + + // As root dir cannot be further aggregated, allow up to maxFiles + // children. + localMaxFilesPerDir := maxFiles + var currPath string + for i, name := range pathSegments[:len(pathSegments)-1] { + currPath = filepath.Join(currPath, name) + + if ev, ok := parentDir.events[name]; ok { + ev.lastModTime = evTime + ev.evType |= event.Type + l.Debugf("%v Parent %s (type %s) already tracked: %s", a, currPath, ev.evType, event.Name) + return + } + + if parentDir.childCount() == localMaxFilesPerDir { + l.Debugf("%v Parent dir %s already has %d children, tracking it instead: %s", a, currPath, localMaxFilesPerDir, event.Name) + event.Name = filepath.Dir(currPath) + a.aggregateEvent(event, evTime, rootEventDir) + return + } + + // If there are no events below path, but we need to recurse + // into that path, create eventDir at path. + if newParent, ok := parentDir.dirs[name]; ok { + parentDir = newParent + } else { + l.Debugln(a, "Creating eventDir at:", currPath) + newParent = newEventDir() + parentDir.dirs[name] = newParent + parentDir = newParent + } + + // Reset allowed children count to maxFilesPerDir for non-root + if i == 0 { + localMaxFilesPerDir = maxFilesPerDir + } + } + + name := pathSegments[len(pathSegments)-1] + + if ev, ok := parentDir.events[name]; ok { + ev.lastModTime = evTime + ev.evType |= event.Type + l.Debugf("%v Already tracked (type %v): %s", a, ev.evType, event.Name) + return + } + + childDir, ok := parentDir.dirs[name] + + // If a dir existed at path, it would be removed from dirs, thus + // childCount would not increase. + if !ok && parentDir.childCount() == localMaxFilesPerDir { + l.Debugf("%v Parent dir already has %d children, tracking it instead: %s", a, localMaxFilesPerDir, event.Name) + event.Name = filepath.Dir(event.Name) + a.aggregateEvent(event, evTime, rootEventDir) + return + } + + firstModTime := evTime + if ok { + firstModTime = childDir.firstModTime() + event.Type |= childDir.eventType() + delete(parentDir.dirs, name) + } + l.Debugf("%v Tracking (type %v): %s", a, event.Type, event.Name) + parentDir.events[name] = &aggregatedEvent{ + firstModTime: firstModTime, + lastModTime: evTime, + evType: event.Type, + } + a.resetNotifyTimerIfNeeded() +} + +func (a *aggregator) resetNotifyTimerIfNeeded() { + if a.notifyTimerNeedsReset { + a.resetNotifyTimer(a.notifyDelay) + } +} + +// resetNotifyTimer should only ever be called when notifyTimer has stopped +// and notifyTimer.C been read from. Otherwise, call resetNotifyTimerIfNeeded. +func (a *aggregator) resetNotifyTimer(duration time.Duration) { + l.Debugln(a, "Resetting notifyTimer to", duration.String()) + a.notifyTimerNeedsReset = false + a.notifyTimer.Reset(duration) +} + +func (a *aggregator) actOnTimer(rootEventDir *eventDir, out chan<- []string) { + eventCount := rootEventDir.eventCount() + if eventCount == 0 { + l.Debugln(a, "No tracked events, waiting for new event.") + a.notifyTimerNeedsReset = true + return + } + oldevents := a.popOldEvents(rootEventDir, ".", time.Now()) + if len(oldevents) == 0 { + l.Debugln(a, "No old fs events") + a.resetNotifyTimer(a.notifyDelay) + return + } + // Sending to channel might block for a long time, but we need to keep + // reading from notify backend channel to avoid overflow + go a.notify(oldevents, out) +} + +// Schedule scan for given events dispatching deletes last and reset notification +// afterwards to set up for the next scan scheduling. +func (a *aggregator) notify(oldEvents map[string]*aggregatedEvent, out chan<- []string) { + timeBeforeSending := time.Now() + l.Debugf("%v Notifying about %d fs events", a, len(oldEvents)) + separatedBatches := make(map[fs.EventType][]string) + for path, event := range oldEvents { + separatedBatches[event.evType] = append(separatedBatches[event.evType], path) + } + for _, evType := range [3]fs.EventType{fs.NonRemove, fs.Mixed, fs.Remove} { + currBatch := separatedBatches[evType] + if len(currBatch) != 0 { + select { + case out <- currBatch: + case <-a.ctx.Done(): + return + } + } + } + // If sending to channel blocked for a long time, + // shorten next notifyDelay accordingly. + duration := time.Since(timeBeforeSending) + buffer := time.Millisecond + var nextDelay time.Duration + switch { + case duration < a.notifyDelay/10: + nextDelay = a.notifyDelay + case duration+buffer > a.notifyDelay: + nextDelay = buffer + default: + nextDelay = a.notifyDelay - duration + } + select { + case a.notifyTimerResetChan <- nextDelay: + case <-a.ctx.Done(): + } +} + +// popOldEvents finds events that should be scheduled for scanning recursively in dirs, +// removes those events and empty eventDirs and returns a map with all the removed +// events referenced by their filesystem path +func (a *aggregator) popOldEvents(dir *eventDir, dirPath string, currTime time.Time) map[string]*aggregatedEvent { + oldEvents := make(map[string]*aggregatedEvent) + for childName, childDir := range dir.dirs { + for evPath, event := range a.popOldEvents(childDir, filepath.Join(dirPath, childName), currTime) { + oldEvents[evPath] = event + } + if childDir.childCount() == 0 { + delete(dir.dirs, childName) + } + } + for name, event := range dir.events { + if a.isOld(event, currTime) { + oldEvents[filepath.Join(dirPath, name)] = event + delete(dir.events, name) + } + } + return oldEvents +} + +func (a *aggregator) isOld(ev *aggregatedEvent, currTime time.Time) bool { + // Deletes should always be scanned last, therefore they are always + // delayed by letting them time out (see below). + // An event that has not registered any new modifications recently is scanned. + // a.notifyDelay is the user facing value signifying the normal delay between + // a picking up a modification and scanning it. As scheduling scans happens at + // regular intervals of a.notifyDelay the delay of a single event is not exactly + // a.notifyDelay, but lies in in the range of 0.5 to 1.5 times a.notifyDelay. + if ev.evType == fs.NonRemove && 2*currTime.Sub(ev.lastModTime) > a.notifyDelay { + return true + } + // When an event registers repeat modifications or involves removals it + // is delayed to reduce resource usage, but after a certain time (notifyTimeout) + // passed it is scanned anyway. + return currTime.Sub(ev.firstModTime) > a.notifyTimeout +} + +func (a *aggregator) String() string { + return fmt.Sprintf("aggregator/%s:", a.folderCfg.Description()) +} + +func (a *aggregator) VerifyConfiguration(from, to config.Configuration) error { + return nil +} + +func (a *aggregator) CommitConfiguration(from, to config.Configuration) bool { + for _, folderCfg := range to.Folders { + if folderCfg.ID == a.folderCfg.ID { + select { + case a.folderCfgUpdate <- folderCfg: + case <-a.ctx.Done(): + } + return true + } + } + // Nothing to do, model will soon stop this + return true +} + +func (a *aggregator) updateConfig(folderCfg config.FolderConfiguration) { + a.notifyDelay = time.Duration(folderCfg.FSWatcherDelayS) * time.Second + a.notifyTimeout = notifyTimeout(folderCfg.FSWatcherDelayS) + a.folderCfg = folderCfg +} + +func updateInProgressSet(event events.Event, inProgress map[string]struct{}) { + if event.Type == events.ItemStarted { + path := event.Data.(map[string]string)["item"] + inProgress[path] = struct{}{} + } else if event.Type == events.ItemFinished { + path := event.Data.(map[string]interface{})["item"].(string) + delete(inProgress, path) + } +} + +// Events that involve removals or continuously receive new modifications are +// delayed but must time out at some point. The following numbers come out of thin +// air, they were just considered as a sensible compromise between fast updates and +// saving resources. For short delays the timeout is 6 times the delay, capped at 1 +// minute. For delays longer than 1 minute, the delay and timeout are equal. +func notifyTimeout(eventDelayS int) time.Duration { + shortDelayS := 10 + shortDelayMultiplicator := 6 + longDelayS := 60 + longDelayTimeout := time.Duration(1) * time.Minute + if eventDelayS < shortDelayS { + return time.Duration(eventDelayS*shortDelayMultiplicator) * time.Second + } + if eventDelayS < longDelayS { + return longDelayTimeout + } + return time.Duration(eventDelayS) * time.Second +} diff --git a/lib/watchaggregator/aggregator_test.go b/lib/watchaggregator/aggregator_test.go new file mode 100644 index 000000000..559e7a1f9 --- /dev/null +++ b/lib/watchaggregator/aggregator_test.go @@ -0,0 +1,281 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package watchaggregator + +import ( + "context" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/syncthing/syncthing/lib/config" + "github.com/syncthing/syncthing/lib/events" + "github.com/syncthing/syncthing/lib/fs" +) + +func TestMain(m *testing.M) { + maxFiles = 32 + maxFilesPerDir = 8 + defer func() { + maxFiles = 512 + maxFilesPerDir = 128 + }() + + os.Exit(m.Run()) +} + +const ( + testNotifyDelayS = 1 + testNotifyTimeout = 2 * time.Second +) + +var ( + folderRoot = filepath.Clean("/home/someuser/syncthing") + defaultFolderCfg = config.FolderConfiguration{ + FilesystemType: fs.FilesystemTypeBasic, + Path: folderRoot, + FSWatcherDelayS: testNotifyDelayS, + } + defaultCfg = config.Wrap("", config.Configuration{ + Folders: []config.FolderConfiguration{defaultFolderCfg}, + }) +) + +type expectedBatch struct { + paths []string + afterMs int + beforeMs int +} + +// TestAggregate checks whether maxFilesPerDir+1 events in one dir are +// aggregated to parent dir +func TestAggregate(t *testing.T) { + evDir := newEventDir() + inProgress := make(map[string]struct{}) + + folderCfg := defaultFolderCfg.Copy() + folderCfg.ID = "Aggregate" + ctx, _ := context.WithCancel(context.Background()) + a := new(folderCfg, ctx) + + // checks whether maxFilesPerDir events in one dir are kept as is + for i := 0; i < maxFilesPerDir; i++ { + a.newEvent(fs.Event{filepath.Join("parent", strconv.Itoa(i)), fs.NonRemove}, evDir, inProgress) + } + if len(getEventPaths(evDir, ".", a)) != maxFilesPerDir { + t.Errorf("Unexpected number of events stored") + } + + // checks whether maxFilesPerDir+1 events in one dir are aggregated to parent dir + a.newEvent(fs.Event{filepath.Join("parent", "new"), fs.NonRemove}, evDir, inProgress) + compareBatchToExpected(t, getEventPaths(evDir, ".", a), []string{"parent"}) + + // checks that adding an event below "parent" does not change anything + a.newEvent(fs.Event{filepath.Join("parent", "extra"), fs.NonRemove}, evDir, inProgress) + compareBatchToExpected(t, getEventPaths(evDir, ".", a), []string{"parent"}) + + // again test aggregation in "parent" but with event in subdirs + evDir = newEventDir() + for i := 0; i < maxFilesPerDir; i++ { + a.newEvent(fs.Event{filepath.Join("parent", strconv.Itoa(i)), fs.NonRemove}, evDir, inProgress) + } + a.newEvent(fs.Event{filepath.Join("parent", "sub", "new"), fs.NonRemove}, evDir, inProgress) + compareBatchToExpected(t, getEventPaths(evDir, ".", a), []string{"parent"}) + + // test aggregation in root + evDir = newEventDir() + for i := 0; i < maxFiles; i++ { + a.newEvent(fs.Event{strconv.Itoa(i), fs.NonRemove}, evDir, inProgress) + } + if len(getEventPaths(evDir, ".", a)) != maxFiles { + t.Errorf("Unexpected number of events stored in root") + } + a.newEvent(fs.Event{filepath.Join("parent", "sub", "new"), fs.NonRemove}, evDir, inProgress) + compareBatchToExpected(t, getEventPaths(evDir, ".", a), []string{"."}) + + // checks that adding an event when "." is already stored is a noop + a.newEvent(fs.Event{"anythingelse", fs.NonRemove}, evDir, inProgress) + compareBatchToExpected(t, getEventPaths(evDir, ".", a), []string{"."}) + + // TestOverflow checks that the entire folder is scanned when maxFiles is reached + evDir = newEventDir() + filesPerDir := maxFilesPerDir / 2 + dirs := make([]string, maxFiles/filesPerDir+1) + for i := 0; i < maxFiles/filesPerDir+1; i++ { + dirs[i] = "dir" + strconv.Itoa(i) + } + for _, dir := range dirs { + for i := 0; i < filesPerDir; i++ { + a.newEvent(fs.Event{filepath.Join(dir, strconv.Itoa(i)), fs.NonRemove}, evDir, inProgress) + } + } + compareBatchToExpected(t, getEventPaths(evDir, ".", a), []string{"."}) +} + +// TestInProgress checks that ignoring files currently edited by Syncthing works +func TestInProgress(t *testing.T) { + testCase := func(c chan<- fs.Event) { + events.Default.Log(events.ItemStarted, map[string]string{ + "item": "inprogress", + }) + sleepMs(100) + c <- fs.Event{Name: "inprogress", Type: fs.NonRemove} + sleepMs(1000) + events.Default.Log(events.ItemFinished, map[string]interface{}{ + "item": "inprogress", + }) + sleepMs(100) + c <- fs.Event{Name: "notinprogress", Type: fs.NonRemove} + sleepMs(800) + } + + expectedBatches := []expectedBatch{ + {[]string{"notinprogress"}, 2000, 3500}, + } + + testScenario(t, "InProgress", testCase, expectedBatches) +} + +// TestDelay checks that recurring changes to the same path are delayed +// and different types separated and ordered correctly +func TestDelay(t *testing.T) { + file := filepath.Join("parent", "file") + delayed := "delayed" + del := "deleted" + both := filepath.Join("parent", "sub", "both") + testCase := func(c chan<- fs.Event) { + sleepMs(200) + c <- fs.Event{Name: file, Type: fs.NonRemove} + delay := time.Duration(300) * time.Millisecond + timer := time.NewTimer(delay) + <-timer.C + timer.Reset(delay) + c <- fs.Event{Name: delayed, Type: fs.NonRemove} + c <- fs.Event{Name: both, Type: fs.NonRemove} + c <- fs.Event{Name: both, Type: fs.Remove} + c <- fs.Event{Name: del, Type: fs.Remove} + for i := 0; i < 9; i++ { + <-timer.C + timer.Reset(delay) + c <- fs.Event{Name: delayed, Type: fs.NonRemove} + } + <-timer.C + } + + // batches that we expect to receive with time interval in milliseconds + expectedBatches := []expectedBatch{ + {[]string{file}, 500, 2500}, + {[]string{delayed}, 2500, 4500}, + {[]string{both}, 2500, 4500}, + {[]string{del}, 2500, 4500}, + {[]string{delayed}, 3600, 6500}, + } + + testScenario(t, "Delay", testCase, expectedBatches) +} + +func getEventPaths(dir *eventDir, dirPath string, a *aggregator) []string { + var paths []string + for childName, childDir := range dir.dirs { + for _, path := range getEventPaths(childDir, filepath.Join(dirPath, childName), a) { + paths = append(paths, path) + } + } + for name := range dir.events { + paths = append(paths, filepath.Join(dirPath, name)) + } + return paths +} + +func sleepMs(ms int) { + time.Sleep(time.Duration(ms) * time.Millisecond) +} + +func durationMs(ms int) time.Duration { + return time.Duration(ms) * time.Millisecond +} + +func compareBatchToExpected(t *testing.T, batch []string, expectedPaths []string) { + for _, expected := range expectedPaths { + expected = filepath.Clean(expected) + found := false + for i, received := range batch { + if expected == received { + found = true + batch = append(batch[:i], batch[i+1:]...) + break + } + } + if !found { + t.Errorf("Did not receive event %s", expected) + } + } + for _, received := range batch { + t.Errorf("Received unexpected event %s", received) + } +} + +func testScenario(t *testing.T, name string, testCase func(c chan<- fs.Event), expectedBatches []expectedBatch) { + ctx, cancel := context.WithCancel(context.Background()) + eventChan := make(chan fs.Event) + watchChan := make(chan []string) + + folderCfg := defaultFolderCfg.Copy() + folderCfg.ID = name + a := new(folderCfg, ctx) + a.notifyTimeout = testNotifyTimeout + + startTime := time.Now() + go a.mainLoop(eventChan, watchChan, defaultCfg) + + sleepMs(10) + go testAggregatorOutput(t, watchChan, expectedBatches, startTime, ctx) + + testCase(eventChan) + + timeout := time.NewTimer(time.Duration(expectedBatches[len(expectedBatches)-1].beforeMs+100) * time.Millisecond) + <-timeout.C + cancel() +} + +func testAggregatorOutput(t *testing.T, fsWatchChan <-chan []string, expectedBatches []expectedBatch, startTime time.Time, ctx context.Context) { + var received []string + var elapsedTime time.Duration + batchIndex := 0 + for { + select { + case <-ctx.Done(): + if batchIndex != len(expectedBatches) { + t.Errorf("Received only %d batches (%d expected)", batchIndex, len(expectedBatches)) + } + return + case received = <-fsWatchChan: + } + + if batchIndex >= len(expectedBatches) { + t.Errorf("Received batch %d (only %d expected)", batchIndex+1, len(expectedBatches)) + continue + } + + elapsedTime = time.Since(startTime) + expected := expectedBatches[batchIndex] + switch { + case elapsedTime < durationMs(expected.afterMs): + t.Errorf("Received batch %d after %v (too soon)", batchIndex+1, elapsedTime) + + case elapsedTime > durationMs(expected.beforeMs): + t.Errorf("Received batch %d after %v (too late)", batchIndex+1, elapsedTime) + + case len(received) != len(expected.paths): + t.Errorf("Received %v events instead of %v for batch %v", len(received), len(expected.paths), batchIndex+1) + } + compareBatchToExpected(t, received, expected.paths) + batchIndex++ + } +} diff --git a/lib/watchaggregator/debug.go b/lib/watchaggregator/debug.go new file mode 100644 index 000000000..a9c2212d6 --- /dev/null +++ b/lib/watchaggregator/debug.go @@ -0,0 +1,24 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package watchaggregator + +import ( + "os" + "strings" + + "github.com/syncthing/syncthing/lib/logger" +) + +var facilityName = "watchaggregator" + +var ( + l = logger.DefaultLogger.NewFacility(facilityName, "Filesystem event watcher") +) + +func init() { + l.SetDebug(facilityName, strings.Contains(os.Getenv("STTRACE"), facilityName) || os.Getenv("STTRACE") == "all") +} diff --git a/man/syncthing-rest-api.7 b/man/syncthing-rest-api.7 index e375af4a7..cfc0a660f 100644 --- a/man/syncthing-rest-api.7 +++ b/man/syncthing-rest-api.7 @@ -111,6 +111,7 @@ Returns the current configuration. } ], "rescanIntervalS": 60, + "longRescanIntervalS": 3600, "ignorePerms": false, "autoNormalize": true, "minDiskFreePct": 1, diff --git a/vendor/github.com/zillode/notify/LICENSE b/vendor/github.com/zillode/notify/LICENSE new file mode 100644 index 000000000..3e678817b --- /dev/null +++ b/vendor/github.com/zillode/notify/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2015 The Notify Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/zillode/notify/debug.go b/vendor/github.com/zillode/notify/debug.go new file mode 100644 index 000000000..bd9bc468d --- /dev/null +++ b/vendor/github.com/zillode/notify/debug.go @@ -0,0 +1,11 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build !debug + +package notify + +func dbgprint(...interface{}) {} + +func dbgprintf(string, ...interface{}) {} diff --git a/vendor/github.com/zillode/notify/debug_debug.go b/vendor/github.com/zillode/notify/debug_debug.go new file mode 100644 index 000000000..f0622917f --- /dev/null +++ b/vendor/github.com/zillode/notify/debug_debug.go @@ -0,0 +1,43 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build debug + +package notify + +import ( + "fmt" + "os" + "runtime" + "strings" +) + +func dbgprint(v ...interface{}) { + fmt.Printf("[D] ") + fmt.Print(v...) + fmt.Printf("\n\n") +} + +func dbgprintf(format string, v ...interface{}) { + fmt.Printf("[D] ") + fmt.Printf(format, v...) + fmt.Printf("\n\n") +} + +func dbgcallstack(max int) []string { + pc, stack := make([]uintptr, max), make([]string, 0, max) + runtime.Callers(2, pc) + for _, pc := range pc { + if f := runtime.FuncForPC(pc); f != nil { + fname := f.Name() + idx := strings.LastIndex(fname, string(os.PathSeparator)) + if idx != -1 { + stack = append(stack, fname[idx+1:]) + } else { + stack = append(stack, fname) + } + } + } + return stack +} diff --git a/vendor/github.com/zillode/notify/doc.go b/vendor/github.com/zillode/notify/doc.go new file mode 100644 index 000000000..60543c083 --- /dev/null +++ b/vendor/github.com/zillode/notify/doc.go @@ -0,0 +1,43 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// Package notify implements access to filesystem events. +// +// Notify is a high-level abstraction over filesystem watchers like inotify, +// kqueue, FSEvents, FEN or ReadDirectoryChangesW. Watcher implementations are +// split into two groups: ones that natively support recursive notifications +// (FSEvents and ReadDirectoryChangesW) and ones that do not (inotify, kqueue, FEN). +// For more details see watcher and recursiveWatcher interfaces in watcher.go +// source file. +// +// On top of filesystem watchers notify maintains a watchpoint tree, which provides +// a strategy for creating and closing filesystem watches and dispatching filesystem +// events to user channels. +// +// An event set is just an event list joint using bitwise OR operator +// into a single event value. +// Both the platform-independent (see Constants) and specific events can be used. +// Refer to the event_*.go source files for information about the available +// events. +// +// A filesystem watch or just a watch is platform-specific entity which represents +// a single path registered for notifications for specific event set. Setting a watch +// means using platform-specific API calls for creating / initializing said watch. +// For each watcher the API call is: +// +// - FSEvents: FSEventStreamCreate +// - inotify: notify_add_watch +// - kqueue: kevent +// - ReadDirectoryChangesW: CreateFile+ReadDirectoryChangesW +// - FEN: port_get +// +// To rewatch means to either shrink or expand an event set that was previously +// registered during watch operation for particular filesystem watch. +// +// A watchpoint is a list of user channel and event set pairs for particular +// path (watchpoint tree's node). A single watchpoint can contain multiple +// different user channels registered to listen for one or more events. A single +// user channel can be registered in one or more watchpoints, recursive and +// non-recursive ones as well. +package notify diff --git a/vendor/github.com/zillode/notify/event.go b/vendor/github.com/zillode/notify/event.go new file mode 100644 index 000000000..c12884197 --- /dev/null +++ b/vendor/github.com/zillode/notify/event.go @@ -0,0 +1,143 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +package notify + +import ( + "fmt" + "strings" +) + +// Event represents the type of filesystem action. +// +// Number of available event values is dependent on the target system or the +// watcher implmenetation used (e.g. it's possible to use either kqueue or +// FSEvents on Darwin). +// +// Please consult documentation for your target platform to see list of all +// available events. +type Event uint32 + +// Create, Remove, Write and Rename are the only event values guaranteed to be +// present on all platforms. +const ( + Create = osSpecificCreate + Remove = osSpecificRemove + Write = osSpecificWrite + Rename = osSpecificRename + + // All is handful alias for all platform-independent event values. + All = Create | Remove | Write | Rename +) + +const internal = recursive | omit + +// String implements fmt.Stringer interface. +func (e Event) String() string { + var s []string + for _, strmap := range []map[Event]string{estr, osestr} { + for ev, str := range strmap { + if e&ev == ev { + s = append(s, str) + } + } + } + return strings.Join(s, "|") +} + +// EventInfo describes an event reported by the underlying filesystem notification +// subsystem. +// +// It always describes single event, even if the OS reported a coalesced action. +// Reported path is absolute and clean. +// +// For non-recursive watchpoints its base is always equal to the path passed +// to corresponding Watch call. +// +// The value of Sys if system-dependent and can be nil. +// +// Sys +// +// Under Darwin (FSEvents) Sys() always returns a non-nil *notify.FSEvent value, +// which is defined as: +// +// type FSEvent struct { +// Path string // real path of the file or directory +// ID uint64 // ID of the event (FSEventStreamEventId) +// Flags uint32 // joint FSEvents* flags (FSEventStreamEventFlags) +// } +// +// For possible values of Flags see Darwin godoc for notify or FSEvents +// documentation for FSEventStreamEventFlags constants: +// +// https://developer.apple.com/library/mac/documentation/Darwin/Reference/FSEvents_Ref/index.html#//apple_ref/doc/constant_group/FSEventStreamEventFlags +// +// Under Linux (inotify) Sys() always returns a non-nil *unix.InotifyEvent +// value, defined as: +// +// type InotifyEvent struct { +// Wd int32 // Watch descriptor +// Mask uint32 // Mask describing event +// Cookie uint32 // Unique cookie associating related events (for rename(2)) +// Len uint32 // Size of name field +// Name [0]uint8 // Optional null-terminated name +// } +// +// More information about inotify masks and the usage of inotify_event structure +// can be found at: +// +// http://man7.org/linux/man-pages/man7/inotify.7.html +// +// Under Darwin, DragonFlyBSD, FreeBSD, NetBSD, OpenBSD (kqueue) Sys() always +// returns a non-nil *notify.Kevent value, which is defined as: +// +// type Kevent struct { +// Kevent *syscall.Kevent_t // Kevent is a kqueue specific structure +// FI os.FileInfo // FI describes file/dir +// } +// +// More information about syscall.Kevent_t can be found at: +// +// https://www.freebsd.org/cgi/man.cgi?query=kqueue +// +// Under Windows (ReadDirectoryChangesW) Sys() always returns nil. The documentation +// of watcher's WinAPI function can be found at: +// +// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365465%28v=vs.85%29.aspx +type EventInfo interface { + Event() Event // event value for the filesystem action + Path() string // real path of the file or directory + Sys() interface{} // underlying data source (can return nil) +} + +type isDirer interface { + isDir() (bool, error) +} + +var _ fmt.Stringer = (*event)(nil) +var _ isDirer = (*event)(nil) + +// String implements fmt.Stringer interface. +func (e *event) String() string { + return e.Event().String() + `: "` + e.Path() + `"` +} + +var estr = map[Event]string{ + Create: "notify.Create", + Remove: "notify.Remove", + Write: "notify.Write", + Rename: "notify.Rename", + // Display name for recursive event is added only for debugging + // purposes. It's an internal event after all and won't be exposed to the + // user. Having Recursive event printable is helpful, e.g. for reading + // testing failure messages: + // + // --- FAIL: TestWatchpoint (0.00 seconds) + // watchpoint_test.go:64: want diff=[notify.Remove notify.Create|notify.Remove]; + // got [notify.Remove notify.Remove|notify.Create] (i=1) + // + // Yup, here the diff have Recursive event inside. Go figure. + recursive: "recursive", + omit: "omit", +} diff --git a/vendor/github.com/zillode/notify/event_fen.go b/vendor/github.com/zillode/notify/event_fen.go new file mode 100644 index 000000000..767f04fa8 --- /dev/null +++ b/vendor/github.com/zillode/notify/event_fen.go @@ -0,0 +1,57 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build solaris + +package notify + +const ( + osSpecificCreate Event = 0x00000100 << iota + osSpecificRemove + osSpecificWrite + osSpecificRename + // internal + // recursive is used to distinguish recursive eventsets from non-recursive ones + recursive + // omit is used for dispatching internal events; only those events are sent + // for which both the event and the watchpoint has omit in theirs event sets. + omit +) + +const ( + // FileAccess is an event reported when monitored file/directory was accessed. + FileAccess = fileAccess + // FileModified is an event reported when monitored file/directory was modified. + FileModified = fileModified + // FileAttrib is an event reported when monitored file/directory's ATTRIB + // was changed. + FileAttrib = fileAttrib + // FileDelete is an event reported when monitored file/directory was deleted. + FileDelete = fileDelete + // FileRenameTo to is an event reported when monitored file/directory was renamed. + FileRenameTo = fileRenameTo + // FileRenameFrom is an event reported when monitored file/directory was renamed. + FileRenameFrom = fileRenameFrom + // FileTrunc is an event reported when monitored file/directory was truncated. + FileTrunc = fileTrunc + // FileNoFollow is an flag to indicate not to follow symbolic links. + FileNoFollow = fileNoFollow + // Unmounted is an event reported when monitored filesystem was unmounted. + Unmounted = unmounted + // MountedOver is an event reported when monitored file/directory was mounted on. + MountedOver = mountedOver +) + +var osestr = map[Event]string{ + FileAccess: "notify.FileAccess", + FileModified: "notify.FileModified", + FileAttrib: "notify.FileAttrib", + FileDelete: "notify.FileDelete", + FileRenameTo: "notify.FileRenameTo", + FileRenameFrom: "notify.FileRenameFrom", + FileTrunc: "notify.FileTrunc", + FileNoFollow: "notify.FileNoFollow", + Unmounted: "notify.Unmounted", + MountedOver: "notify.MountedOver", +} diff --git a/vendor/github.com/zillode/notify/event_fsevents.go b/vendor/github.com/zillode/notify/event_fsevents.go new file mode 100644 index 000000000..6ded80b2c --- /dev/null +++ b/vendor/github.com/zillode/notify/event_fsevents.go @@ -0,0 +1,71 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build darwin,!kqueue + +package notify + +const ( + osSpecificCreate = Event(FSEventsCreated) + osSpecificRemove = Event(FSEventsRemoved) + osSpecificWrite = Event(FSEventsModified) + osSpecificRename = Event(FSEventsRenamed) + // internal = Event(0x100000) + // recursive is used to distinguish recursive eventsets from non-recursive ones + recursive = Event(0x200000) + // omit is used for dispatching internal events; only those events are sent + // for which both the event and the watchpoint has omit in theirs event sets. + omit = Event(0x400000) +) + +// FSEvents specific event values. +const ( + FSEventsMustScanSubDirs Event = 0x00001 + FSEventsUserDropped = 0x00002 + FSEventsKernelDropped = 0x00004 + FSEventsEventIdsWrapped = 0x00008 + FSEventsHistoryDone = 0x00010 + FSEventsRootChanged = 0x00020 + FSEventsMount = 0x00040 + FSEventsUnmount = 0x00080 + FSEventsCreated = 0x00100 + FSEventsRemoved = 0x00200 + FSEventsInodeMetaMod = 0x00400 + FSEventsRenamed = 0x00800 + FSEventsModified = 0x01000 + FSEventsFinderInfoMod = 0x02000 + FSEventsChangeOwner = 0x04000 + FSEventsXattrMod = 0x08000 + FSEventsIsFile = 0x10000 + FSEventsIsDir = 0x20000 + FSEventsIsSymlink = 0x40000 +) + +var osestr = map[Event]string{ + FSEventsMustScanSubDirs: "notify.FSEventsMustScanSubDirs", + FSEventsUserDropped: "notify.FSEventsUserDropped", + FSEventsKernelDropped: "notify.FSEventsKernelDropped", + FSEventsEventIdsWrapped: "notify.FSEventsEventIdsWrapped", + FSEventsHistoryDone: "notify.FSEventsHistoryDone", + FSEventsRootChanged: "notify.FSEventsRootChanged", + FSEventsMount: "notify.FSEventsMount", + FSEventsUnmount: "notify.FSEventsUnmount", + FSEventsInodeMetaMod: "notify.FSEventsInodeMetaMod", + FSEventsFinderInfoMod: "notify.FSEventsFinderInfoMod", + FSEventsChangeOwner: "notify.FSEventsChangeOwner", + FSEventsXattrMod: "notify.FSEventsXattrMod", + FSEventsIsFile: "notify.FSEventsIsFile", + FSEventsIsDir: "notify.FSEventsIsDir", + FSEventsIsSymlink: "notify.FSEventsIsSymlink", +} + +type event struct { + fse FSEvent + event Event +} + +func (ei *event) Event() Event { return ei.event } +func (ei *event) Path() string { return ei.fse.Path } +func (ei *event) Sys() interface{} { return &ei.fse } +func (ei *event) isDir() (bool, error) { return ei.fse.Flags&FSEventsIsDir != 0, nil } diff --git a/vendor/github.com/zillode/notify/event_inotify.go b/vendor/github.com/zillode/notify/event_inotify.go new file mode 100644 index 000000000..1f32bb73e --- /dev/null +++ b/vendor/github.com/zillode/notify/event_inotify.go @@ -0,0 +1,75 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build linux + +package notify + +import "golang.org/x/sys/unix" + +// Platform independent event values. +const ( + osSpecificCreate Event = 0x100000 << iota + osSpecificRemove + osSpecificWrite + osSpecificRename + // internal + // recursive is used to distinguish recursive eventsets from non-recursive ones + recursive + // omit is used for dispatching internal events; only those events are sent + // for which both the event and the watchpoint has omit in theirs event sets. + omit +) + +// Inotify specific masks are legal, implemented events that are guaranteed to +// work with notify package on linux-based systems. +const ( + InAccess = Event(unix.IN_ACCESS) // File was accessed + InModify = Event(unix.IN_MODIFY) // File was modified + InAttrib = Event(unix.IN_ATTRIB) // Metadata changed + InCloseWrite = Event(unix.IN_CLOSE_WRITE) // Writtable file was closed + InCloseNowrite = Event(unix.IN_CLOSE_NOWRITE) // Unwrittable file closed + InOpen = Event(unix.IN_OPEN) // File was opened + InMovedFrom = Event(unix.IN_MOVED_FROM) // File was moved from X + InMovedTo = Event(unix.IN_MOVED_TO) // File was moved to Y + InCreate = Event(unix.IN_CREATE) // Subfile was created + InDelete = Event(unix.IN_DELETE) // Subfile was deleted + InDeleteSelf = Event(unix.IN_DELETE_SELF) // Self was deleted + InMoveSelf = Event(unix.IN_MOVE_SELF) // Self was moved +) + +var osestr = map[Event]string{ + InAccess: "notify.InAccess", + InModify: "notify.InModify", + InAttrib: "notify.InAttrib", + InCloseWrite: "notify.InCloseWrite", + InCloseNowrite: "notify.InCloseNowrite", + InOpen: "notify.InOpen", + InMovedFrom: "notify.InMovedFrom", + InMovedTo: "notify.InMovedTo", + InCreate: "notify.InCreate", + InDelete: "notify.InDelete", + InDeleteSelf: "notify.InDeleteSelf", + InMoveSelf: "notify.InMoveSelf", +} + +// Inotify behavior events are not **currently** supported by notify package. +const ( + inDontFollow = Event(unix.IN_DONT_FOLLOW) + inExclUnlink = Event(unix.IN_EXCL_UNLINK) + inMaskAdd = Event(unix.IN_MASK_ADD) + inOneshot = Event(unix.IN_ONESHOT) + inOnlydir = Event(unix.IN_ONLYDIR) +) + +type event struct { + sys unix.InotifyEvent + path string + event Event +} + +func (e *event) Event() Event { return e.event } +func (e *event) Path() string { return e.path } +func (e *event) Sys() interface{} { return &e.sys } +func (e *event) isDir() (bool, error) { return e.sys.Mask&unix.IN_ISDIR != 0, nil } diff --git a/vendor/github.com/zillode/notify/event_kqueue.go b/vendor/github.com/zillode/notify/event_kqueue.go new file mode 100644 index 000000000..422ac87f6 --- /dev/null +++ b/vendor/github.com/zillode/notify/event_kqueue.go @@ -0,0 +1,59 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build darwin,kqueue dragonfly freebsd netbsd openbsd + +package notify + +import "syscall" + +// TODO(pblaszczyk): ensure in runtime notify built-in event values do not +// overlap with platform-defined ones. + +// Platform independent event values. +const ( + osSpecificCreate Event = 0x0100 << iota + osSpecificRemove + osSpecificWrite + osSpecificRename + // internal + // recursive is used to distinguish recursive eventsets from non-recursive ones + recursive + // omit is used for dispatching internal events; only those events are sent + // for which both the event and the watchpoint has omit in theirs event sets. + omit +) + +const ( + // NoteDelete is an event reported when the unlink() system call was called + // on the file referenced by the descriptor. + NoteDelete = Event(syscall.NOTE_DELETE) + // NoteWrite is an event reported when a write occurred on the file + // referenced by the descriptor. + NoteWrite = Event(syscall.NOTE_WRITE) + // NoteExtend is an event reported when the file referenced by the + // descriptor was extended. + NoteExtend = Event(syscall.NOTE_EXTEND) + // NoteAttrib is an event reported when the file referenced + // by the descriptor had its attributes changed. + NoteAttrib = Event(syscall.NOTE_ATTRIB) + // NoteLink is an event reported when the link count on the file changed. + NoteLink = Event(syscall.NOTE_LINK) + // NoteRename is an event reported when the file referenced + // by the descriptor was renamed. + NoteRename = Event(syscall.NOTE_RENAME) + // NoteRevoke is an event reported when access to the file was revoked via + // revoke(2) or the underlying file system was unmounted. + NoteRevoke = Event(syscall.NOTE_REVOKE) +) + +var osestr = map[Event]string{ + NoteDelete: "notify.NoteDelete", + NoteWrite: "notify.NoteWrite", + NoteExtend: "notify.NoteExtend", + NoteAttrib: "notify.NoteAttrib", + NoteLink: "notify.NoteLink", + NoteRename: "notify.NoteRename", + NoteRevoke: "notify.NoteRevoke", +} diff --git a/vendor/github.com/zillode/notify/event_readdcw.go b/vendor/github.com/zillode/notify/event_readdcw.go new file mode 100644 index 000000000..f7b82de0c --- /dev/null +++ b/vendor/github.com/zillode/notify/event_readdcw.go @@ -0,0 +1,118 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build windows + +package notify + +import ( + "os" + "path/filepath" + "syscall" +) + +// Platform independent event values. +const ( + osSpecificCreate Event = 1 << (20 + iota) + osSpecificRemove + osSpecificWrite + osSpecificRename + // recursive is used to distinguish recursive eventsets from non-recursive ones + recursive + // omit is used for dispatching internal events; only those events are sent + // for which both the event and the watchpoint has omit in theirs event sets. + omit + // dirmarker TODO(pknap) + dirmarker +) + +// ReadDirectoryChangesW filters +// On Windows the following events can be passed to Watch. A different set of +// events (see actions below) are received on the channel passed to Watch. +// For more information refer to +// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365465(v=vs.85).aspx +const ( + FileNotifyChangeFileName = Event(syscall.FILE_NOTIFY_CHANGE_FILE_NAME) + FileNotifyChangeDirName = Event(syscall.FILE_NOTIFY_CHANGE_DIR_NAME) + FileNotifyChangeAttributes = Event(syscall.FILE_NOTIFY_CHANGE_ATTRIBUTES) + FileNotifyChangeSize = Event(syscall.FILE_NOTIFY_CHANGE_SIZE) + FileNotifyChangeLastWrite = Event(syscall.FILE_NOTIFY_CHANGE_LAST_WRITE) + FileNotifyChangeLastAccess = Event(syscall.FILE_NOTIFY_CHANGE_LAST_ACCESS) + FileNotifyChangeCreation = Event(syscall.FILE_NOTIFY_CHANGE_CREATION) + FileNotifyChangeSecurity = Event(syscallFileNotifyChangeSecurity) +) + +const ( + fileNotifyChangeAll = 0x17f // logical sum of all FileNotifyChange* events. + fileNotifyChangeModified = fileNotifyChangeAll &^ (FileNotifyChangeFileName | FileNotifyChangeDirName) +) + +// according to: http://msdn.microsoft.com/en-us/library/windows/desktop/aa365465(v=vs.85).aspx +// this flag should be declared in: http://golang.org/src/pkg/syscall/ztypes_windows.go +const syscallFileNotifyChangeSecurity = 0x00000100 + +// ReadDirectoryChangesW actions +// The following events are returned on the channel passed to Watch, but cannot +// be passed to Watch itself (see filters above). You can find a table showing +// the relation between actions and filteres at +// https://github.com/rjeczalik/notify/issues/10#issuecomment-66179535 +// The msdn documentation on actions is part of +// https://msdn.microsoft.com/en-us/library/windows/desktop/aa364391(v=vs.85).aspx +const ( + FileActionAdded = Event(syscall.FILE_ACTION_ADDED) << 12 + FileActionRemoved = Event(syscall.FILE_ACTION_REMOVED) << 12 + FileActionModified = Event(syscall.FILE_ACTION_MODIFIED) << 14 + FileActionRenamedOldName = Event(syscall.FILE_ACTION_RENAMED_OLD_NAME) << 15 + FileActionRenamedNewName = Event(syscall.FILE_ACTION_RENAMED_NEW_NAME) << 16 +) + +const fileActionAll = 0x7f000 // logical sum of all FileAction* events. + +var osestr = map[Event]string{ + FileNotifyChangeFileName: "notify.FileNotifyChangeFileName", + FileNotifyChangeDirName: "notify.FileNotifyChangeDirName", + FileNotifyChangeAttributes: "notify.FileNotifyChangeAttributes", + FileNotifyChangeSize: "notify.FileNotifyChangeSize", + FileNotifyChangeLastWrite: "notify.FileNotifyChangeLastWrite", + FileNotifyChangeLastAccess: "notify.FileNotifyChangeLastAccess", + FileNotifyChangeCreation: "notify.FileNotifyChangeCreation", + FileNotifyChangeSecurity: "notify.FileNotifyChangeSecurity", + + FileActionAdded: "notify.FileActionAdded", + FileActionRemoved: "notify.FileActionRemoved", + FileActionModified: "notify.FileActionModified", + FileActionRenamedOldName: "notify.FileActionRenamedOldName", + FileActionRenamedNewName: "notify.FileActionRenamedNewName", +} + +const ( + fTypeUnknown uint8 = iota + fTypeFile + fTypeDirectory +) + +// TODO(ppknap) : doc. +type event struct { + pathw []uint16 + name string + ftype uint8 + action uint32 + filter uint32 + e Event +} + +func (e *event) Event() Event { return e.e } +func (e *event) Path() string { return filepath.Join(syscall.UTF16ToString(e.pathw), e.name) } +func (e *event) Sys() interface{} { return e.ftype } + +func (e *event) isDir() (bool, error) { + if e.ftype != fTypeUnknown { + return e.ftype == fTypeDirectory, nil + } + fi, err := os.Stat(e.Path()) + if err != nil { + return false, err + } + return fi.IsDir(), nil +} diff --git a/vendor/github.com/zillode/notify/event_stub.go b/vendor/github.com/zillode/notify/event_stub.go new file mode 100644 index 000000000..faac7c7cb --- /dev/null +++ b/vendor/github.com/zillode/notify/event_stub.go @@ -0,0 +1,31 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build !darwin,!linux,!freebsd,!dragonfly,!netbsd,!openbsd,!windows +// +build !kqueue,!solaris + +package notify + +// Platform independent event values. +const ( + osSpecificCreate Event = 1 << iota + osSpecificRemove + osSpecificWrite + osSpecificRename + // internal + // recursive is used to distinguish recursive eventsets from non-recursive ones + recursive + // omit is used for dispatching internal events; only those events are sent + // for which both the event and the watchpoint has omit in theirs event sets. + omit +) + +var osestr = map[Event]string{} + +type event struct{} + +func (e *event) Event() (_ Event) { return } +func (e *event) Path() (_ string) { return } +func (e *event) Sys() (_ interface{}) { return } +func (e *event) isDir() (_ bool, _ error) { return } diff --git a/vendor/github.com/zillode/notify/event_trigger.go b/vendor/github.com/zillode/notify/event_trigger.go new file mode 100644 index 000000000..94470fd37 --- /dev/null +++ b/vendor/github.com/zillode/notify/event_trigger.go @@ -0,0 +1,22 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build darwin,kqueue dragonfly freebsd netbsd openbsd solaris + +package notify + +type event struct { + p string + e Event + d bool + pe interface{} +} + +func (e *event) Event() Event { return e.e } + +func (e *event) Path() string { return e.p } + +func (e *event) Sys() interface{} { return e.pe } + +func (e *event) isDir() (bool, error) { return e.d, nil } diff --git a/vendor/github.com/zillode/notify/node.go b/vendor/github.com/zillode/notify/node.go new file mode 100644 index 000000000..29c1bb20a --- /dev/null +++ b/vendor/github.com/zillode/notify/node.go @@ -0,0 +1,272 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +package notify + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +var errSkip = errors.New("notify: skip") + +type walkPathFunc func(nd node, isbase bool) error + +type walkFunc func(node) error + +func errnotexist(name string) error { + return &os.PathError{ + Op: "Node", + Path: name, + Err: os.ErrNotExist, + } +} + +type node struct { + Name string + Watch watchpoint + Child map[string]node +} + +func newnode(name string) node { + return node{ + Name: name, + Watch: make(watchpoint), + Child: make(map[string]node), + } +} + +func (nd node) addchild(name, base string) node { + child, ok := nd.Child[base] + if !ok { + child = newnode(name) + nd.Child[base] = child + } + return child +} + +func (nd node) Add(name string) node { + i := indexbase(nd.Name, name) + if i == -1 { + return node{} + } + for j := indexSep(name[i:]); j != -1; j = indexSep(name[i:]) { + nd = nd.addchild(name[:i+j], name[i:i+j]) + i += j + 1 + } + return nd.addchild(name, name[i:]) +} + +func (nd node) AddDir(fn walkFunc) error { + stack := []node{nd} +Traverse: + for n := len(stack); n != 0; n = len(stack) { + nd, stack = stack[n-1], stack[:n-1] + switch err := fn(nd); err { + case nil: + case errSkip: + continue Traverse + default: + return fmt.Errorf("error while traversing %q: %v", nd.Name, err) + } + // TODO(rjeczalik): tolerate open failures - add failed names to + // AddDirError and notify users which names are not added to the tree. + fi, err := ioutil.ReadDir(nd.Name) + if err != nil { + return err + } + for _, fi := range fi { + if fi.Mode()&(os.ModeSymlink|os.ModeDir) == os.ModeDir { + name := filepath.Join(nd.Name, fi.Name()) + stack = append(stack, nd.addchild(name, name[len(nd.Name)+1:])) + } + } + } + return nil +} + +func (nd node) Get(name string) (node, error) { + i := indexbase(nd.Name, name) + if i == -1 { + return node{}, errnotexist(name) + } + ok := false + for j := indexSep(name[i:]); j != -1; j = indexSep(name[i:]) { + if nd, ok = nd.Child[name[i:i+j]]; !ok { + return node{}, errnotexist(name) + } + i += j + 1 + } + if nd, ok = nd.Child[name[i:]]; !ok { + return node{}, errnotexist(name) + } + return nd, nil +} + +func (nd node) Del(name string) error { + i := indexbase(nd.Name, name) + if i == -1 { + return errnotexist(name) + } + stack := []node{nd} + ok := false + for j := indexSep(name[i:]); j != -1; j = indexSep(name[i:]) { + if nd, ok = nd.Child[name[i:i+j]]; !ok { + return errnotexist(name[:i+j]) + } + stack = append(stack, nd) + } + if nd, ok = nd.Child[name[i:]]; !ok { + return errnotexist(name) + } + nd.Child = nil + nd.Watch = nil + for name, i = base(nd.Name), len(stack); i != 0; name, i = base(nd.Name), i-1 { + nd = stack[i-1] + if nd := nd.Child[name]; len(nd.Watch) > 1 || len(nd.Child) != 0 { + break + } else { + nd.Child = nil + nd.Watch = nil + } + delete(nd.Child, name) + } + return nil +} + +func (nd node) Walk(fn walkFunc) error { + stack := []node{nd} +Traverse: + for n := len(stack); n != 0; n = len(stack) { + nd, stack = stack[n-1], stack[:n-1] + switch err := fn(nd); err { + case nil: + case errSkip: + continue Traverse + default: + return err + } + for name, nd := range nd.Child { + if name == "" { + // Node storing inactive watchpoints has empty name, skip it + // form traversing. Root node has also an empty name, but it + // never has a parent node. + continue + } + stack = append(stack, nd) + } + } + return nil +} + +func (nd node) WalkPath(name string, fn walkPathFunc) error { + i := indexbase(nd.Name, name) + if i == -1 { + return errnotexist(name) + } + ok := false + for j := indexSep(name[i:]); j != -1; j = indexSep(name[i:]) { + switch err := fn(nd, false); err { + case nil: + case errSkip: + return nil + default: + return err + } + if nd, ok = nd.Child[name[i:i+j]]; !ok { + return errnotexist(name[:i+j]) + } + i += j + 1 + } + switch err := fn(nd, false); err { + case nil: + case errSkip: + return nil + default: + return err + } + if nd, ok = nd.Child[name[i:]]; !ok { + return errnotexist(name) + } + switch err := fn(nd, true); err { + case nil, errSkip: + return nil + default: + return err + } +} + +type root struct { + nd node +} + +func (r root) addroot(name string) node { + if vol := filepath.VolumeName(name); vol != "" { + root, ok := r.nd.Child[vol] + if !ok { + root = r.nd.addchild(vol, vol) + } + return root + } + return r.nd +} + +func (r root) root(name string) (node, error) { + if vol := filepath.VolumeName(name); vol != "" { + nd, ok := r.nd.Child[vol] + if !ok { + return node{}, errnotexist(name) + } + return nd, nil + } + return r.nd, nil +} + +func (r root) Add(name string) node { + return r.addroot(name).Add(name) +} + +func (r root) AddDir(dir string, fn walkFunc) error { + return r.Add(dir).AddDir(fn) +} + +func (r root) Del(name string) error { + nd, err := r.root(name) + if err != nil { + return err + } + return nd.Del(name) +} + +func (r root) Get(name string) (node, error) { + nd, err := r.root(name) + if err != nil { + return node{}, err + } + if nd.Name != name { + if nd, err = nd.Get(name); err != nil { + return node{}, err + } + } + return nd, nil +} + +func (r root) Walk(name string, fn walkFunc) error { + nd, err := r.Get(name) + if err != nil { + return err + } + return nd.Walk(fn) +} + +func (r root) WalkPath(name string, fn walkPathFunc) error { + nd, err := r.root(name) + if err != nil { + return err + } + return nd.WalkPath(name, fn) +} diff --git a/vendor/github.com/zillode/notify/notify.go b/vendor/github.com/zillode/notify/notify.go new file mode 100644 index 000000000..293843ca1 --- /dev/null +++ b/vendor/github.com/zillode/notify/notify.go @@ -0,0 +1,83 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// BUG(rjeczalik): Notify does not collect watchpoints, when underlying watches +// were removed by their os-specific watcher implementations. Instead users are +// advised to listen on persistent paths to have guarantee they receive events +// for the whole lifetime of their applications (to discuss see #69). + +// BUG(ppknap): Linux (inotify) does not support watcher behavior masks like +// InOneshot, InOnlydir etc. Instead users are advised to perform the filtering +// themselves (to discuss see #71). + +// BUG(ppknap): Notify was not tested for short path name support under Windows +// (ReadDirectoryChangesW). + +// BUG(ppknap): Windows (ReadDirectoryChangesW) cannot recognize which notification +// triggers FileActionModified event. (to discuss see #75). + +package notify + +var defaultTree = newTree() + +// Watch sets up a watchpoint on path listening for events given by the events +// argument. +// +// File or directory given by the path must exist, otherwise Watch will fail +// with non-nil error. Notify resolves, for its internal purpose, any symlinks +// the provided path may contain, so it may fail if the symlinks form a cycle. +// It does so, since not all watcher implementations treat passed paths as-is. +// E.g. FSEvents reports a real path for every event, setting a watchpoint +// on /tmp will report events with paths rooted at /private/tmp etc. +// +// The c almost always is a buffered channel. Watch will not block sending to c +// - the caller must ensure that c has sufficient buffer space to keep up with +// the expected event rate. +// +// It is allowed to pass the same channel multiple times with different event +// list or different paths. Calling Watch with different event lists for a single +// watchpoint expands its event set. The only way to shrink it, is to call +// Stop on its channel. +// +// Calling Watch with empty event list does expand nor shrink watchpoint's event +// set. If c is the first channel to listen for events on the given path, Watch +// will seamlessly create a watch on the filesystem. +// +// Notify dispatches copies of single filesystem event to all channels registered +// for each path. If a single filesystem event contains multiple coalesced events, +// each of them is dispatched separately. E.g. the following filesystem change: +// +// ~ $ echo Hello > Notify.txt +// +// dispatches two events - notify.Create and notify.Write. However, it may depend +// on the underlying watcher implementation whether OS reports both of them. +// +// Windows and recursive watches +// +// If a directory which path was used to create recursive watch under Windows +// gets deleted, the OS will not report such event. It is advised to keep in +// mind this limitation while setting recursive watchpoints for your application, +// e.g. use persistent paths like %userprofile% or watch additionally parent +// directory of a recursive watchpoint in order to receive delete events for it. +func Watch(path string, c chan<- EventInfo, events ...Event) error { + return defaultTree.Watch(path, c, nil, events...) +} + +// This function works the same way as Watch. In addition it does not watch +// files or directories based on the return value of the argument function +// doNotWatch. Given a path as argument doNotWatch should return true if the +// file or directory should not be watched. +func WatchWithFilter(path string, c chan<- EventInfo, + doNotWatch func(string) bool, events ...Event) error { + return defaultTree.Watch(path, c, doNotWatch, events...) +} + +// Stop removes all watchpoints registered for c. All underlying watches are +// also removed, for which c was the last channel listening for events. +// +// Stop does not close c. When Stop returns, it is guaranteed that c will +// receive no more signals. +func Stop(c chan<- EventInfo) { + defaultTree.Stop(c) +} diff --git a/vendor/github.com/zillode/notify/tree.go b/vendor/github.com/zillode/notify/tree.go new file mode 100644 index 000000000..e5f86ae7d --- /dev/null +++ b/vendor/github.com/zillode/notify/tree.go @@ -0,0 +1,22 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +package notify + +const buffer = 128 + +type tree interface { + Watch(string, chan<- EventInfo, func(string) bool, ...Event) error + Stop(chan<- EventInfo) + Close() error +} + +func newTree() tree { + c := make(chan EventInfo, buffer) + w := newWatcher(c) + if rw, ok := w.(recursiveWatcher); ok { + return newRecursiveTree(rw, c) + } + return newNonrecursiveTree(w, c, make(chan EventInfo, buffer)) +} diff --git a/vendor/github.com/zillode/notify/tree_nonrecursive.go b/vendor/github.com/zillode/notify/tree_nonrecursive.go new file mode 100644 index 000000000..b492a8a5e --- /dev/null +++ b/vendor/github.com/zillode/notify/tree_nonrecursive.go @@ -0,0 +1,303 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +package notify + +import "sync" + +// nonrecursiveTree TODO(rjeczalik) +type nonrecursiveTree struct { + rw sync.RWMutex // protects root + root root + w watcher + c chan EventInfo + rec chan EventInfo +} + +// newNonrecursiveTree TODO(rjeczalik) +func newNonrecursiveTree(w watcher, c, rec chan EventInfo) *nonrecursiveTree { + if rec == nil { + rec = make(chan EventInfo, buffer) + } + t := &nonrecursiveTree{ + root: root{nd: newnode("")}, + w: w, + c: c, + rec: rec, + } + go t.dispatch(c) + go t.internal(rec) + return t +} + +// dispatch TODO(rjeczalik) +func (t *nonrecursiveTree) dispatch(c <-chan EventInfo) { + for ei := range c { + dbgprintf("dispatching %v on %q", ei.Event(), ei.Path()) + go func(ei EventInfo) { + var nd node + var isrec bool + dir, base := split(ei.Path()) + fn := func(it node, isbase bool) error { + isrec = isrec || it.Watch.IsRecursive() + if isbase { + nd = it + } else { + it.Watch.Dispatch(ei, recursive) + } + return nil + } + t.rw.RLock() + // Notify recursive watchpoints found on the path. + if err := t.root.WalkPath(dir, fn); err != nil { + dbgprint("dispatch did not reach leaf:", err) + t.rw.RUnlock() + return + } + // Notify parent watchpoint. + nd.Watch.Dispatch(ei, 0) + isrec = isrec || nd.Watch.IsRecursive() + // If leaf watchpoint exists, notify it. + if nd, ok := nd.Child[base]; ok { + isrec = isrec || nd.Watch.IsRecursive() + nd.Watch.Dispatch(ei, 0) + } + t.rw.RUnlock() + // If the event describes newly leaf directory created within + if !isrec || ei.Event() != Create { + return + } + if ok, err := ei.(isDirer).isDir(); !ok || err != nil { + return + } + t.rec <- ei + }(ei) + } +} + +// internal TODO(rjeczalik) +func (t *nonrecursiveTree) internal(rec <-chan EventInfo) { + for ei := range rec { + var nd node + var eset = internal + t.rw.Lock() + t.root.WalkPath(ei.Path(), func(it node, _ bool) error { + if e := it.Watch[t.rec]; e != 0 && e > eset { + eset = e + } + nd = it + return nil + }) + if eset == internal { + t.rw.Unlock() + continue + } + err := nd.Add(ei.Path()).AddDir(t.recFunc(eset, nil)) + t.rw.Unlock() + if err != nil { + dbgprintf("internal(%p) error: %v", rec, err) + } + } +} + +// watchAdd TODO(rjeczalik) +func (t *nonrecursiveTree) watchAdd(nd node, c chan<- EventInfo, e Event) eventDiff { + if e&recursive != 0 { + diff := nd.Watch.Add(t.rec, e|Create|omit) + nd.Watch.Add(c, e) + return diff + } + return nd.Watch.Add(c, e) +} + +// watchDelMin TODO(rjeczalik) +func (t *nonrecursiveTree) watchDelMin(min Event, nd node, c chan<- EventInfo, e Event) eventDiff { + old, ok := nd.Watch[t.rec] + if ok { + nd.Watch[t.rec] = min + } + diff := nd.Watch.Del(c, e) + if ok { + switch old &^= diff[0] &^ diff[1]; { + case old|internal == internal: + delete(nd.Watch, t.rec) + if set, ok := nd.Watch[nil]; ok && len(nd.Watch) == 1 && set == 0 { + delete(nd.Watch, nil) + } + default: + nd.Watch.Add(t.rec, old|Create) + switch { + case diff == none: + case diff[1]|Create == diff[0]: + diff = none + default: + diff[1] |= Create + } + } + } + return diff +} + +// watchDel TODO(rjeczalik) +func (t *nonrecursiveTree) watchDel(nd node, c chan<- EventInfo, e Event) eventDiff { + return t.watchDelMin(0, nd, c, e) +} + +// Watch TODO(rjeczalik) +func (t *nonrecursiveTree) Watch(path string, c chan<- EventInfo, + doNotWatch func(string) bool, events ...Event) error { + if c == nil { + panic("notify: Watch using nil channel") + } + // Expanding with empty event set is a nop. + if len(events) == 0 { + return nil + } + path, isrec, err := cleanpath(path) + if err != nil { + return err + } + eset := joinevents(events) + t.rw.Lock() + defer t.rw.Unlock() + nd := t.root.Add(path) + if isrec { + return t.watchrec(nd, c, eset|recursive, doNotWatch) + } + return t.watch(nd, c, eset) +} + +func (t *nonrecursiveTree) watch(nd node, c chan<- EventInfo, e Event) (err error) { + diff := nd.Watch.Add(c, e) + switch { + case diff == none: + return nil + case diff[1] == 0: + // TODO(rjeczalik): cleanup this panic after implementation is stable + panic("eset is empty: " + nd.Name) + case diff[0] == 0: + err = t.w.Watch(nd.Name, diff[1]) + default: + err = t.w.Rewatch(nd.Name, diff[0], diff[1]) + } + if err != nil { + nd.Watch.Del(c, diff.Event()) + return err + } + return nil +} + +func (t *nonrecursiveTree) recFunc(e Event, doNotWatch func(string) bool) walkFunc { + addWatch := func(nd node) (err error) { + switch diff := nd.Watch.Add(t.rec, e|omit|Create); { + case diff == none: + case diff[1] == 0: + // TODO(rjeczalik): cleanup this panic after implementation is stable + panic("eset is empty: " + nd.Name) + case diff[0] == 0: + err = t.w.Watch(nd.Name, diff[1]) + default: + err = t.w.Rewatch(nd.Name, diff[0], diff[1]) + } + return + } + if doNotWatch != nil { + return func(nd node) (err error) { + if doNotWatch(nd.Name) { + return errSkip + } + return addWatch(nd) + } + } + return addWatch +} + +func (t *nonrecursiveTree) watchrec(nd node, c chan<- EventInfo, e Event, + doNotWatch func(string) bool) error { + var traverse func(walkFunc) error + // Non-recursive tree listens on Create event for every recursive + // watchpoint in order to automagically set a watch for every + // created directory. + switch diff := nd.Watch.dryAdd(t.rec, e|Create); { + case diff == none: + t.watchAdd(nd, c, e) + nd.Watch.Add(t.rec, e|omit|Create) + return nil + case diff[1] == 0: + // TODO(rjeczalik): cleanup this panic after implementation is stable + panic("eset is empty: " + nd.Name) + case diff[0] == 0: + // TODO(rjeczalik): BFS into directories and skip subtree as soon as first + // recursive watchpoint is encountered. + traverse = nd.AddDir + default: + traverse = nd.Walk + } + // TODO(rjeczalik): account every path that failed to be (re)watched + // and retry. + if err := traverse(t.recFunc(e, doNotWatch)); err != nil { + return err + } + t.watchAdd(nd, c, e) + return nil +} + +type walkWatchpointFunc func(Event, node) error + +func (t *nonrecursiveTree) walkWatchpoint(nd node, fn walkWatchpointFunc) error { + type minode struct { + min Event + nd node + } + mnd := minode{nd: nd} + stack := []minode{mnd} +Traverse: + for n := len(stack); n != 0; n = len(stack) { + mnd, stack = stack[n-1], stack[:n-1] + // There must be no recursive watchpoints if the node has no watchpoints + // itself (every node in subtree rooted at recursive watchpoints must + // have at least nil (total) and t.rec watchpoints). + if len(mnd.nd.Watch) != 0 { + switch err := fn(mnd.min, mnd.nd); err { + case nil: + case errSkip: + continue Traverse + default: + return err + } + } + for _, nd := range mnd.nd.Child { + stack = append(stack, minode{mnd.nd.Watch[t.rec], nd}) + } + } + return nil +} + +// Stop TODO(rjeczalik) +func (t *nonrecursiveTree) Stop(c chan<- EventInfo) { + fn := func(min Event, nd node) error { + // TODO(rjeczalik): aggregate watcher errors and retry; in worst case + // forward to the user. + switch diff := t.watchDelMin(min, nd, c, all); { + case diff == none: + return nil + case diff[1] == 0: + t.w.Unwatch(nd.Name) + default: + t.w.Rewatch(nd.Name, diff[0], diff[1]) + } + return nil + } + t.rw.Lock() + err := t.walkWatchpoint(t.root.nd, fn) // TODO(rjeczalik): store max root per c + t.rw.Unlock() + dbgprintf("Stop(%p) error: %v\n", c, err) +} + +// Close TODO(rjeczalik) +func (t *nonrecursiveTree) Close() error { + err := t.w.Close() + close(t.c) + return err +} diff --git a/vendor/github.com/zillode/notify/tree_recursive.go b/vendor/github.com/zillode/notify/tree_recursive.go new file mode 100644 index 000000000..4d0aaba14 --- /dev/null +++ b/vendor/github.com/zillode/notify/tree_recursive.go @@ -0,0 +1,355 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +package notify + +import "sync" + +// watchAdd TODO(rjeczalik) +func watchAdd(nd node, c chan<- EventInfo, e Event) eventDiff { + diff := nd.Watch.Add(c, e) + if wp := nd.Child[""].Watch; len(wp) != 0 { + e = wp.Total() + diff[0] |= e + diff[1] |= e + if diff[0] == diff[1] { + return none + } + } + return diff +} + +// watchAddInactive TODO(rjeczalik) +func watchAddInactive(nd node, c chan<- EventInfo, e Event) eventDiff { + wp := nd.Child[""].Watch + if wp == nil { + wp = make(watchpoint) + nd.Child[""] = node{Watch: wp} + } + diff := wp.Add(c, e) + e = nd.Watch.Total() + diff[0] |= e + diff[1] |= e + if diff[0] == diff[1] { + return none + } + return diff +} + +// watchCopy TODO(rjeczalik) +func watchCopy(src, dst node) { + for c, e := range src.Watch { + if c == nil { + continue + } + watchAddInactive(dst, c, e) + } + if wpsrc := src.Child[""].Watch; len(wpsrc) != 0 { + wpdst := dst.Child[""].Watch + for c, e := range wpsrc { + if c == nil { + continue + } + wpdst.Add(c, e) + } + } +} + +// watchDel TODO(rjeczalik) +func watchDel(nd node, c chan<- EventInfo, e Event) eventDiff { + diff := nd.Watch.Del(c, e) + if wp := nd.Child[""].Watch; len(wp) != 0 { + diffInactive := wp.Del(c, e) + e = wp.Total() + // TODO(rjeczalik): add e if e != all? + diff[0] |= diffInactive[0] | e + diff[1] |= diffInactive[1] | e + if diff[0] == diff[1] { + return none + } + } + return diff +} + +// watchTotal TODO(rjeczalik) +func watchTotal(nd node) Event { + e := nd.Watch.Total() + if wp := nd.Child[""].Watch; len(wp) != 0 { + e |= wp.Total() + } + return e +} + +// watchIsRecursive TODO(rjeczalik) +func watchIsRecursive(nd node) bool { + ok := nd.Watch.IsRecursive() + // TODO(rjeczalik): add a test for len(wp) != 0 change the condition. + if wp := nd.Child[""].Watch; len(wp) != 0 { + // If a watchpoint holds inactive watchpoints, it means it's a parent + // one, which is recursive by nature even though it may be not recursive + // itself. + ok = true + } + return ok +} + +// recursiveTree TODO(rjeczalik) +type recursiveTree struct { + rw sync.RWMutex // protects root + root root + // TODO(rjeczalik): merge watcher + recursiveWatcher after #5 and #6 + w interface { + watcher + recursiveWatcher + } + c chan EventInfo +} + +// newRecursiveTree TODO(rjeczalik) +func newRecursiveTree(w recursiveWatcher, c chan EventInfo) *recursiveTree { + t := &recursiveTree{ + root: root{nd: newnode("")}, + w: struct { + watcher + recursiveWatcher + }{w.(watcher), w}, + c: c, + } + go t.dispatch() + return t +} + +// dispatch TODO(rjeczalik) +func (t *recursiveTree) dispatch() { + for ei := range t.c { + dbgprintf("dispatching %v on %q", ei.Event(), ei.Path()) + go func(ei EventInfo) { + nd, ok := node{}, false + dir, base := split(ei.Path()) + fn := func(it node, isbase bool) error { + if isbase { + nd = it + } else { + it.Watch.Dispatch(ei, recursive) + } + return nil + } + t.rw.RLock() + defer t.rw.RUnlock() + // Notify recursive watchpoints found on the path. + if err := t.root.WalkPath(dir, fn); err != nil { + dbgprint("dispatch did not reach leaf:", err) + return + } + // Notify parent watchpoint. + nd.Watch.Dispatch(ei, 0) + // If leaf watchpoint exists, notify it. + if nd, ok = nd.Child[base]; ok { + nd.Watch.Dispatch(ei, 0) + } + }(ei) + } +} + +// Watch TODO(rjeczalik) +func (t *recursiveTree) Watch(path string, c chan<- EventInfo, + doNotWatch func(string) bool, events ...Event) error { + if c == nil { + panic("notify: Watch using nil channel") + } + // Expanding with empty event set is a nop. + if len(events) == 0 { + return nil + } + path, isrec, err := cleanpath(path) + if err != nil { + return err + } + eventset := joinevents(events) + if isrec { + eventset |= recursive + } + t.rw.Lock() + defer t.rw.Unlock() + // case 1: cur is a child + // + // Look for parent watch which already covers the given path. + parent := node{} + self := false + err = t.root.WalkPath(path, func(nd node, isbase bool) error { + if watchTotal(nd) != 0 { + parent = nd + self = isbase + return errSkip + } + return nil + }) + cur := t.root.Add(path) // add after the walk, so it's less to traverse + if err == nil && parent.Watch != nil { + // Parent watch found. Register inactive watchpoint, so we have enough + // information to shrink the eventset on eventual Stop. + // return t.resetwatchpoint(parent, parent, c, eventset|inactive) + var diff eventDiff + if self { + diff = watchAdd(cur, c, eventset) + } else { + diff = watchAddInactive(parent, c, eventset) + } + switch { + case diff == none: + // the parent watchpoint already covers requested subtree with its + // eventset + case diff[0] == 0: + // TODO(rjeczalik): cleanup this panic after implementation is stable + panic("dangling watchpoint: " + parent.Name) + default: + if isrec || watchIsRecursive(parent) { + err = t.w.RecursiveRewatch(parent.Name, parent.Name, diff[0], diff[1]) + } else { + err = t.w.Rewatch(parent.Name, diff[0], diff[1]) + } + if err != nil { + watchDel(parent, c, diff.Event()) + return err + } + watchAdd(cur, c, eventset) + // TODO(rjeczalik): account top-most path for c + return nil + } + if !self { + watchAdd(cur, c, eventset) + } + return nil + } + // case 2: cur is new parent + // + // Look for children nodes, unwatch n-1 of them and rewatch the last one. + var children []node + fn := func(nd node) error { + if len(nd.Watch) == 0 { + return nil + } + children = append(children, nd) + return errSkip + } + switch must(cur.Walk(fn)); len(children) { + case 0: + // no child watches, cur holds a new watch + case 1: + watchAdd(cur, c, eventset) // TODO(rjeczalik): update cache c subtree root? + watchCopy(children[0], cur) + err = t.w.RecursiveRewatch(children[0].Name, cur.Name, watchTotal(children[0]), + watchTotal(cur)) + if err != nil { + // Clean inactive watchpoint. The c chan did not exist before. + cur.Child[""] = node{} + delete(cur.Watch, c) + return err + } + return nil + default: + watchAdd(cur, c, eventset) + // Copy children inactive watchpoints to the new parent. + for _, nd := range children { + watchCopy(nd, cur) + } + // Watch parent subtree. + if err = t.w.RecursiveWatch(cur.Name, watchTotal(cur)); err != nil { + // Clean inactive watchpoint. The c chan did not exist before. + cur.Child[""] = node{} + delete(cur.Watch, c) + return err + } + // Unwatch children subtrees. + var e error + for _, nd := range children { + if watchIsRecursive(nd) { + e = t.w.RecursiveUnwatch(nd.Name) + } else { + e = t.w.Unwatch(nd.Name) + } + if e != nil { + err = nonil(err, e) + // TODO(rjeczalik): child is still watched, warn all its watchpoints + // about possible duplicate events via Error event + } + } + return err + } + // case 3: cur is new, alone node + switch diff := watchAdd(cur, c, eventset); { + case diff == none: + // TODO(rjeczalik): cleanup this panic after implementation is stable + panic("watch requested but no parent watchpoint found: " + cur.Name) + case diff[0] == 0: + if isrec { + err = t.w.RecursiveWatch(cur.Name, diff[1]) + } else { + err = t.w.Watch(cur.Name, diff[1]) + } + if err != nil { + watchDel(cur, c, diff.Event()) + return err + } + default: + // TODO(rjeczalik): cleanup this panic after implementation is stable + panic("watch requested but no parent watchpoint found: " + cur.Name) + } + return nil +} + +// Stop TODO(rjeczalik) +// +// TODO(rjeczalik): Split parent watchpoint - transfer watches to children +// if parent is no longer needed. This carries a risk that underlying +// watcher calls could fail - reconsider if it's worth the effort. +func (t *recursiveTree) Stop(c chan<- EventInfo) { + var err error + fn := func(nd node) (e error) { + diff := watchDel(nd, c, all) + switch { + case diff == none && watchTotal(nd) == 0: + // TODO(rjeczalik): There's no watchpoints deeper in the tree, + // probably we should remove the nodes as well. + return nil + case diff == none: + // Removing c from nd does not require shrinking its eventset. + case diff[1] == 0: + if watchIsRecursive(nd) { + e = t.w.RecursiveUnwatch(nd.Name) + } else { + e = t.w.Unwatch(nd.Name) + } + default: + if watchIsRecursive(nd) { + e = t.w.RecursiveRewatch(nd.Name, nd.Name, diff[0], diff[1]) + } else { + e = t.w.Rewatch(nd.Name, diff[0], diff[1]) + } + } + fn := func(nd node) error { + watchDel(nd, c, all) + return nil + } + err = nonil(err, e, nd.Walk(fn)) + // TODO(rjeczalik): if e != nil store dummy chan in nd.Watch just to + // retry un/rewatching next time and/or let the user handle the failure + // vie Error event? + return errSkip + } + t.rw.Lock() + e := t.root.Walk("", fn) // TODO(rjeczalik): use max root per c + t.rw.Unlock() + if e != nil { + err = nonil(err, e) + } + dbgprintf("Stop(%p) error: %v\n", c, err) +} + +// Close TODO(rjeczalik) +func (t *recursiveTree) Close() error { + err := t.w.Close() + close(t.c) + return err +} diff --git a/vendor/github.com/zillode/notify/util.go b/vendor/github.com/zillode/notify/util.go new file mode 100644 index 000000000..67e01fbbd --- /dev/null +++ b/vendor/github.com/zillode/notify/util.go @@ -0,0 +1,150 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +package notify + +import ( + "errors" + "os" + "path/filepath" + "strings" +) + +const all = ^Event(0) +const sep = string(os.PathSeparator) + +var errDepth = errors.New("exceeded allowed iteration count (circular symlink?)") + +func min(i, j int) int { + if i > j { + return j + } + return i +} + +func max(i, j int) int { + if i < j { + return j + } + return i +} + +// must panics if err is non-nil. +func must(err error) { + if err != nil { + panic(err) + } +} + +// nonil gives first non-nil error from the given arguments. +func nonil(err ...error) error { + for _, err := range err { + if err != nil { + return err + } + } + return nil +} + +func cleanpath(path string) (realpath string, isrec bool, err error) { + if strings.HasSuffix(path, "...") { + isrec = true + path = path[:len(path)-3] + } + if path, err = filepath.Abs(path); err != nil { + return "", false, err + } + if path, err = canonical(path); err != nil { + return "", false, err + } + return path, isrec, nil +} + +// canonical resolves any symlink in the given path and returns it in a clean form. +// It expects the path to be absolute. It fails to resolve circular symlinks by +// maintaining a simple iteration limit. +func canonical(p string) (string, error) { + p, err := filepath.Abs(p) + if err != nil { + return "", err + } + for i, j, depth := 1, 0, 1; i < len(p); i, depth = i+1, depth+1 { + if depth > 128 { + return "", &os.PathError{Op: "canonical", Path: p, Err: errDepth} + } + if j = strings.IndexRune(p[i:], '/'); j == -1 { + j, i = i, len(p) + } else { + j, i = i, i+j + } + fi, err := os.Lstat(p[:i]) + if err != nil { + return "", err + } + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + s, err := os.Readlink(p[:i]) + if err != nil { + return "", err + } + if filepath.IsAbs(s) { + p = "/" + s + p[i:] + } else { + p = p[:j] + s + p[i:] + } + i = 1 // no guarantee s is canonical, start all over + } + } + return filepath.Clean(p), nil +} + +func joinevents(events []Event) (e Event) { + if len(events) == 0 { + e = All + } else { + for _, event := range events { + e |= event + } + } + return +} + +func split(s string) (string, string) { + if i := lastIndexSep(s); i != -1 { + return s[:i], s[i+1:] + } + return "", s +} + +func base(s string) string { + if i := lastIndexSep(s); i != -1 { + return s[i+1:] + } + return s +} + +func indexbase(root, name string) int { + if n, m := len(root), len(name); m >= n && name[:n] == root && + (n == m || name[n] == os.PathSeparator) { + return min(n+1, m) + } + return -1 +} + +func indexSep(s string) int { + for i := 0; i < len(s); i++ { + if s[i] == os.PathSeparator { + return i + } + } + return -1 +} + +func lastIndexSep(s string) int { + for i := len(s) - 1; i >= 0; i-- { + if s[i] == os.PathSeparator { + return i + } + } + return -1 +} diff --git a/vendor/github.com/zillode/notify/watcher.go b/vendor/github.com/zillode/notify/watcher.go new file mode 100644 index 000000000..34148eff3 --- /dev/null +++ b/vendor/github.com/zillode/notify/watcher.go @@ -0,0 +1,85 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +package notify + +import "errors" + +var ( + errAlreadyWatched = errors.New("path is already watched") + errNotWatched = errors.New("path is not being watched") + errInvalidEventSet = errors.New("invalid event set provided") +) + +// Watcher is a intermediate interface for wrapping inotify, ReadDirChangesW, +// FSEvents, kqueue and poller implementations. +// +// The watcher implementation is expected to do its own mapping between paths and +// create watchers if underlying event notification does not support it. For +// the ease of implementation it is guaranteed that paths provided via Watch and +// Unwatch methods are absolute and clean. +type watcher interface { + // Watch requests a watcher creation for the given path and given event set. + Watch(path string, event Event) error + + // Unwatch requests a watcher deletion for the given path and given event set. + Unwatch(path string) error + + // Rewatch provides a functionality for modifying existing watch-points, like + // expanding its event set. + // + // Rewatch modifies existing watch-point under for the given path. It passes + // the existing event set currently registered for the given path, and the + // new, requested event set. + // + // It is guaranteed that Tree will not pass to Rewatch zero value for any + // of its arguments. If old == new and watcher can be upgraded to + // recursiveWatcher interface, a watch for the corresponding path is expected + // to be changed from recursive to the non-recursive one. + Rewatch(path string, old, new Event) error + + // Close unwatches all paths that are registered. When Close returns, it + // is expected it will report no more events. + Close() error +} + +// RecursiveWatcher is an interface for a Watcher for those OS, which do support +// recursive watching over directories. +type recursiveWatcher interface { + RecursiveWatch(path string, event Event) error + + // RecursiveUnwatch removes a recursive watch-point given by the path. For + // native recursive implementation there is no difference in functionality + // between Unwatch and RecursiveUnwatch, however for those platforms, that + // requires emulation for recursive watch-points, the implementation differs. + RecursiveUnwatch(path string) error + + // RecursiveRewatcher provides a functionality for modifying and/or relocating + // existing recursive watch-points. + // + // To relocate a watch-point means to unwatch oldpath and set a watch-point on + // newpath. + // + // To modify a watch-point means either to expand or shrink its event set. + // + // Tree can want to either relocate, modify or relocate and modify a watch-point + // via single RecursiveRewatch call. + // + // If oldpath == newpath, the watch-point is expected to change its event set value + // from oldevent to newevent. + // + // If oldevent == newevent, the watch-point is expected to relocate from oldpath + // to the newpath. + // + // If oldpath != newpath and oldevent != newevent, the watch-point is expected + // to relocate from oldpath to the newpath first and then change its event set + // value from oldevent to the newevent. In other words the end result must be + // a watch-point set on newpath with newevent value of its event set. + // + // It is guaranteed that Tree will not pass to RecurisveRewatcha zero value + // for any of its arguments. If oldpath == newpath and oldevent == newevent, + // a watch for the corresponding path is expected to be changed for + // non-recursive to the recursive one. + RecursiveRewatch(oldpath, newpath string, oldevent, newevent Event) error +} diff --git a/vendor/github.com/zillode/notify/watcher_fen.go b/vendor/github.com/zillode/notify/watcher_fen.go new file mode 100644 index 000000000..dfe77f2f1 --- /dev/null +++ b/vendor/github.com/zillode/notify/watcher_fen.go @@ -0,0 +1,161 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build solaris + +package notify + +import ( + "fmt" + "os" + "syscall" +) + +// newTrigger returns implementation of trigger. +func newTrigger(pthLkp map[string]*watched) trigger { + return &fen{ + pthLkp: pthLkp, + cf: newCfen(), + } +} + +// fen is a structure implementing trigger for FEN. +type fen struct { + // p is a FEN port identifier + p int + // pthLkp is a structure mapping monitored files/dir with data about them, + // shared with parent trg structure + pthLkp map[string]*watched + // cf wraps C operations for FEN + cf cfen +} + +// watched is a data structure representing watched file/directory. +type watched struct { + trgWatched +} + +// Stop implements trigger. +func (f *fen) Stop() error { + return f.cf.portAlert(f.p) +} + +// Close implements trigger. +func (f *fen) Close() (err error) { + return syscall.Close(f.p) +} + +// NewWatched implements trigger. +func (*fen) NewWatched(p string, fi os.FileInfo) (*watched, error) { + return &watched{trgWatched{p: p, fi: fi}}, nil +} + +// Record implements trigger. +func (f *fen) Record(w *watched) { + f.pthLkp[w.p] = w +} + +// Del implements trigger. +func (f *fen) Del(w *watched) { + delete(f.pthLkp, w.p) +} + +func inter2pe(n interface{}) PortEvent { + pe, ok := n.(PortEvent) + if !ok { + panic(fmt.Sprintf("fen: type should be PortEvent, %T instead", n)) + } + return pe +} + +// Watched implements trigger. +func (f *fen) Watched(n interface{}) (*watched, int64, error) { + pe := inter2pe(n) + fo, ok := pe.PortevObject.(*FileObj) + if !ok || fo == nil { + panic(fmt.Sprintf("fen: type should be *FileObj, %T instead", fo)) + } + w, ok := f.pthLkp[fo.Name] + if !ok { + return nil, 0, errNotWatched + } + return w, int64(pe.PortevEvents), nil +} + +// init initializes FEN. +func (f *fen) Init() (err error) { + f.p, err = f.cf.portCreate() + return +} + +func fi2fo(fi os.FileInfo, p string) FileObj { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + panic(fmt.Sprintf("fen: type should be *syscall.Stat_t, %T instead", st)) + } + return FileObj{Name: p, Atim: st.Atim, Mtim: st.Mtim, Ctim: st.Ctim} +} + +// Unwatch implements trigger. +func (f *fen) Unwatch(w *watched) error { + return f.cf.portDissociate(f.p, FileObj{Name: w.p}) +} + +// Watch implements trigger. +func (f *fen) Watch(fi os.FileInfo, w *watched, e int64) error { + return f.cf.portAssociate(f.p, fi2fo(fi, w.p), int(e)) +} + +// Wait implements trigger. +func (f *fen) Wait() (interface{}, error) { + var ( + pe PortEvent + err error + ) + err = f.cf.portGet(f.p, &pe) + return pe, err +} + +// IsStop implements trigger. +func (f *fen) IsStop(n interface{}, err error) bool { + return err == syscall.EBADF || inter2pe(n).PortevSource == srcAlert +} + +func init() { + encode = func(e Event, dir bool) (o int64) { + // Create event is not supported by FEN. Instead FileModified event will + // be registered. If this event will be reported on dir which is to be + // monitored for Create, dir will be rescanned and Create events will + // be generated and returned for new files. In case of files, + // if not requested FileModified event is reported, it will be ignored. + o = int64(e &^ Create) + if (e&Create != 0 && dir) || e&Write != 0 { + o = (o &^ int64(Write)) | int64(FileModified) + } + // Following events are 'exception events' and as such cannot be requested + // explicitly for monitoring or filtered out. If the will be reported + // by FEN and not subscribed with by user, they will be filtered out by + // watcher's logic. + o &= int64(^Rename & ^Remove &^ FileDelete &^ FileRenameTo &^ + FileRenameFrom &^ Unmounted &^ MountedOver) + return + } + nat2not = map[Event]Event{ + FileModified: Write, + FileRenameFrom: Rename, + FileDelete: Remove, + FileAccess: Event(0), + FileAttrib: Event(0), + FileRenameTo: Event(0), + FileTrunc: Event(0), + FileNoFollow: Event(0), + Unmounted: Event(0), + MountedOver: Event(0), + } + not2nat = map[Event]Event{ + Write: FileModified, + Rename: FileRenameFrom, + Remove: FileDelete, + } +} diff --git a/vendor/github.com/zillode/notify/watcher_fen_cgo.go b/vendor/github.com/zillode/notify/watcher_fen_cgo.go new file mode 100644 index 000000000..8ec8ead34 --- /dev/null +++ b/vendor/github.com/zillode/notify/watcher_fen_cgo.go @@ -0,0 +1,141 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build solaris + +package notify + +// #include +// #include +// #include +// struct file_obj* newFo() { return (struct file_obj*) malloc(sizeof(struct file_obj)); } +// port_event_t* newPe() { return (port_event_t*) malloc(sizeof(port_event_t)); } +// uintptr_t conv(struct file_obj* fo) { return (uintptr_t) fo; } +// struct file_obj* dconv(uintptr_t fo) { return (struct file_obj*) fo; } +import "C" + +import ( + "syscall" + "unsafe" +) + +const ( + fileAccess = Event(C.FILE_ACCESS) + fileModified = Event(C.FILE_MODIFIED) + fileAttrib = Event(C.FILE_ATTRIB) + fileDelete = Event(C.FILE_DELETE) + fileRenameTo = Event(C.FILE_RENAME_TO) + fileRenameFrom = Event(C.FILE_RENAME_FROM) + fileTrunc = Event(C.FILE_TRUNC) + fileNoFollow = Event(C.FILE_NOFOLLOW) + unmounted = Event(C.UNMOUNTED) + mountedOver = Event(C.MOUNTEDOVER) +) + +// PortEvent is a notify's equivalent of port_event_t. +type PortEvent struct { + PortevEvents int // PortevEvents is an equivalent of portev_events. + PortevSource uint8 // PortevSource is an equivalent of portev_source. + PortevPad uint8 // Portevpad is an equivalent of portev_pad. + PortevObject interface{} // PortevObject is an equivalent of portev_object. + PortevUser uintptr // PortevUser is an equivalent of portev_user. +} + +// FileObj is a notify's equivalent of file_obj. +type FileObj struct { + Atim syscall.Timespec // Atim is an equivalent of fo_atime. + Mtim syscall.Timespec // Mtim is an equivalent of fo_mtime. + Ctim syscall.Timespec // Ctim is an equivalent of fo_ctime. + Pad [3]uintptr // Pad is an equivalent of fo_pad. + Name string // Name is an equivalent of fo_name. +} + +type cfen struct { + p2pe map[string]*C.port_event_t + p2fo map[string]*C.struct_file_obj +} + +func newCfen() cfen { + return cfen{ + p2pe: make(map[string]*C.port_event_t), + p2fo: make(map[string]*C.struct_file_obj), + } +} + +func unix2C(sec int64, nsec int64) (C.time_t, C.long) { + return C.time_t(sec), C.long(nsec) +} + +func (c *cfen) portAssociate(p int, fo FileObj, e int) (err error) { + cfo := C.newFo() + cfo.fo_atime.tv_sec, cfo.fo_atime.tv_nsec = unix2C(fo.Atim.Unix()) + cfo.fo_mtime.tv_sec, cfo.fo_mtime.tv_nsec = unix2C(fo.Mtim.Unix()) + cfo.fo_ctime.tv_sec, cfo.fo_ctime.tv_nsec = unix2C(fo.Ctim.Unix()) + cfo.fo_name = C.CString(fo.Name) + c.p2fo[fo.Name] = cfo + _, err = C.port_associate(C.int(p), srcFile, C.conv(cfo), C.int(e), nil) + return +} + +func (c *cfen) portDissociate(port int, fo FileObj) (err error) { + cfo, ok := c.p2fo[fo.Name] + if !ok { + return errNotWatched + } + _, err = C.port_dissociate(C.int(port), srcFile, C.conv(cfo)) + C.free(unsafe.Pointer(cfo.fo_name)) + C.free(unsafe.Pointer(cfo)) + delete(c.p2fo, fo.Name) + return +} + +const srcAlert = C.PORT_SOURCE_ALERT +const srcFile = C.PORT_SOURCE_FILE +const alertSet = C.PORT_ALERT_SET + +func cfo2fo(cfo *C.struct_file_obj) *FileObj { + // Currently remaining attributes are not used. + if cfo == nil { + return nil + } + var fo FileObj + fo.Name = C.GoString(cfo.fo_name) + return &fo +} + +func (c *cfen) portGet(port int, pe *PortEvent) (err error) { + cpe := C.newPe() + if _, err = C.port_get(C.int(port), cpe, nil); err != nil { + C.free(unsafe.Pointer(cpe)) + return + } + pe.PortevEvents, pe.PortevSource, pe.PortevPad = + int(cpe.portev_events), uint8(cpe.portev_source), uint8(cpe.portev_pad) + pe.PortevObject = cfo2fo(C.dconv(cpe.portev_object)) + pe.PortevUser = uintptr(cpe.portev_user) + C.free(unsafe.Pointer(cpe)) + return +} + +func (c *cfen) portCreate() (int, error) { + p, err := C.port_create() + return int(p), err +} + +func (c *cfen) portAlert(p int) (err error) { + _, err = C.port_alert(C.int(p), alertSet, C.int(666), nil) + return +} + +func (c *cfen) free() { + for i := range c.p2fo { + C.free(unsafe.Pointer(c.p2fo[i].fo_name)) + C.free(unsafe.Pointer(c.p2fo[i])) + } + for i := range c.p2pe { + C.free(unsafe.Pointer(c.p2pe[i])) + } + c.p2fo = make(map[string]*C.struct_file_obj) + c.p2pe = make(map[string]*C.port_event_t) +} diff --git a/vendor/github.com/zillode/notify/watcher_fsevents.go b/vendor/github.com/zillode/notify/watcher_fsevents.go new file mode 100644 index 000000000..7d9b97b65 --- /dev/null +++ b/vendor/github.com/zillode/notify/watcher_fsevents.go @@ -0,0 +1,311 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build darwin,!kqueue + +package notify + +import ( + "errors" + "strings" + "sync/atomic" +) + +const ( + failure = uint32(FSEventsMustScanSubDirs | FSEventsUserDropped | FSEventsKernelDropped) + filter = uint32(FSEventsCreated | FSEventsRemoved | FSEventsRenamed | + FSEventsModified | FSEventsInodeMetaMod) +) + +// FSEvent represents single file event. It is created out of values passed by +// FSEvents to FSEventStreamCallback function. +type FSEvent struct { + Path string // real path of the file or directory + ID uint64 // ID of the event (FSEventStreamEventId) + Flags uint32 // joint FSEvents* flags (FSEventStreamEventFlags) +} + +// splitflags separates event flags from single set into slice of flags. +func splitflags(set uint32) (e []uint32) { + for i := uint32(1); set != 0; i, set = i<<1, set>>1 { + if (set & 1) != 0 { + e = append(e, i) + } + } + return +} + +// watch represents a filesystem watchpoint. It is a higher level abstraction +// over FSEvents' stream, which implements filtering of file events based +// on path and event set. It emulates non-recursive watch-point by filtering out +// events which paths are more than 1 level deeper than the watched path. +type watch struct { + // prev stores last event set per path in order to filter out old flags + // for new events, which appratenly FSEvents likes to retain. It's a disgusting + // hack, it should be researched how to get rid of it. + prev map[string]uint32 + c chan<- EventInfo + stream *stream + path string + events uint32 + isrec int32 + flushed bool +} + +// Example format: +// +// ~ $ (trigger command) # (event set) -> (effective event set) +// +// Heuristics: +// +// 1. Create event is removed when it was present in previous event set. +// Example: +// +// ~ $ echo > file # Create|Write -> Create|Write +// ~ $ echo > file # Create|Write|InodeMetaMod -> Write|InodeMetaMod +// +// 2. Remove event is removed if it was present in previouse event set. +// Example: +// +// ~ $ touch file # Create -> Create +// ~ $ rm file # Create|Remove -> Remove +// ~ $ touch file # Create|Remove -> Create +// +// 3. Write event is removed if not followed by InodeMetaMod on existing +// file. Example: +// +// ~ $ echo > file # Create|Write -> Create|Write +// ~ $ chmod +x file # Create|Write|ChangeOwner -> ChangeOwner +// +// 4. Write&InodeMetaMod is removed when effective event set contain Remove event. +// Example: +// +// ~ $ echo > file # Write|InodeMetaMod -> Write|InodeMetaMod +// ~ $ rm file # Remove|Write|InodeMetaMod -> Remove +// +func (w *watch) strip(base string, set uint32) uint32 { + const ( + write = FSEventsModified | FSEventsInodeMetaMod + both = FSEventsCreated | FSEventsRemoved + ) + switch w.prev[base] { + case FSEventsCreated: + set &^= FSEventsCreated + if set&FSEventsRemoved != 0 { + w.prev[base] = FSEventsRemoved + set &^= write + } + case FSEventsRemoved: + set &^= FSEventsRemoved + if set&FSEventsCreated != 0 { + w.prev[base] = FSEventsCreated + } + default: + switch set & both { + case FSEventsCreated: + w.prev[base] = FSEventsCreated + case FSEventsRemoved: + w.prev[base] = FSEventsRemoved + set &^= write + } + } + dbgprintf("split()=%v\n", Event(set)) + return set +} + +// Dispatch is a stream function which forwards given file events for the watched +// path to underlying FileInfo channel. +func (w *watch) Dispatch(ev []FSEvent) { + events := atomic.LoadUint32(&w.events) + isrec := (atomic.LoadInt32(&w.isrec) == 1) + for i := range ev { + if ev[i].Flags&FSEventsHistoryDone != 0 { + w.flushed = true + continue + } + if !w.flushed { + continue + } + dbgprintf("%v (0x%x) (%s, i=%d, ID=%d, len=%d)\n", Event(ev[i].Flags), + ev[i].Flags, ev[i].Path, i, ev[i].ID, len(ev)) + if ev[i].Flags&failure != 0 { + // TODO(rjeczalik): missing error handling + continue + } + if !strings.HasPrefix(ev[i].Path, w.path) { + continue + } + n := len(w.path) + base := "" + if len(ev[i].Path) > n { + if ev[i].Path[n] != '/' { + continue + } + base = ev[i].Path[n+1:] + if !isrec && strings.IndexByte(base, '/') != -1 { + continue + } + } + // TODO(rjeczalik): get diff only from filtered events? + e := w.strip(string(base), ev[i].Flags) & events + if e == 0 { + continue + } + for _, e := range splitflags(e) { + dbgprintf("%d: single event: %v", ev[i].ID, Event(e)) + w.c <- &event{ + fse: ev[i], + event: Event(e), + } + } + } +} + +// Stop closes underlying FSEvents stream and stops dispatching events. +func (w *watch) Stop() { + w.stream.Stop() + // TODO(rjeczalik): make (*stream).Stop flush synchronously undelivered events, + // so the following hack can be removed. It should flush all the streams + // concurrently as we care not to block too much here. + atomic.StoreUint32(&w.events, 0) + atomic.StoreInt32(&w.isrec, 0) +} + +// fsevents implements Watcher and RecursiveWatcher interfaces backed by FSEvents +// framework. +type fsevents struct { + watches map[string]*watch + c chan<- EventInfo +} + +func newWatcher(c chan<- EventInfo) watcher { + return &fsevents{ + watches: make(map[string]*watch), + c: c, + } +} + +func (fse *fsevents) watch(path string, event Event, isrec int32) (err error) { + if _, ok := fse.watches[path]; ok { + return errAlreadyWatched + } + w := &watch{ + prev: make(map[string]uint32), + c: fse.c, + path: path, + events: uint32(event), + isrec: isrec, + } + w.stream = newStream(path, w.Dispatch) + if err = w.stream.Start(); err != nil { + return err + } + fse.watches[path] = w + return nil +} + +func (fse *fsevents) unwatch(path string) (err error) { + w, ok := fse.watches[path] + if !ok { + return errNotWatched + } + w.stream.Stop() + delete(fse.watches, path) + return nil +} + +// Watch implements Watcher interface. It fails with non-nil error when setting +// the watch-point by FSEvents fails or with errAlreadyWatched error when +// the given path is already watched. +func (fse *fsevents) Watch(path string, event Event) error { + return fse.watch(path, event, 0) +} + +// Unwatch implements Watcher interface. It fails with errNotWatched when +// the given path is not being watched. +func (fse *fsevents) Unwatch(path string) error { + return fse.unwatch(path) +} + +// Rewatch implements Watcher interface. It fails with errNotWatched when +// the given path is not being watched or with errInvalidEventSet when oldevent +// does not match event set the watch-point currently holds. +func (fse *fsevents) Rewatch(path string, oldevent, newevent Event) error { + w, ok := fse.watches[path] + if !ok { + return errNotWatched + } + if !atomic.CompareAndSwapUint32(&w.events, uint32(oldevent), uint32(newevent)) { + return errInvalidEventSet + } + atomic.StoreInt32(&w.isrec, 0) + return nil +} + +// RecursiveWatch implements RecursiveWatcher interface. It fails with non-nil +// error when setting the watch-point by FSEvents fails or with errAlreadyWatched +// error when the given path is already watched. +func (fse *fsevents) RecursiveWatch(path string, event Event) error { + return fse.watch(path, event, 1) +} + +// RecursiveUnwatch implements RecursiveWatcher interface. It fails with +// errNotWatched when the given path is not being watched. +// +// TODO(rjeczalik): fail if w.isrec == 0? +func (fse *fsevents) RecursiveUnwatch(path string) error { + return fse.unwatch(path) +} + +// RecrusiveRewatch implements RecursiveWatcher interface. It fails: +// +// * with errNotWatched when the given path is not being watched +// * with errInvalidEventSet when oldevent does not match the current event set +// * with errAlreadyWatched when watch-point given by the oldpath was meant to +// be relocated to newpath, but the newpath is already watched +// * a non-nil error when setting the watch-point with FSEvents fails +// +// TODO(rjeczalik): Improve handling of watch-point relocation? See two TODOs +// that follows. +func (fse *fsevents) RecursiveRewatch(oldpath, newpath string, oldevent, newevent Event) error { + switch [2]bool{oldpath == newpath, oldevent == newevent} { + case [2]bool{true, true}: + w, ok := fse.watches[oldpath] + if !ok { + return errNotWatched + } + atomic.StoreInt32(&w.isrec, 1) + return nil + case [2]bool{true, false}: + w, ok := fse.watches[oldpath] + if !ok { + return errNotWatched + } + if !atomic.CompareAndSwapUint32(&w.events, uint32(oldevent), uint32(newevent)) { + return errors.New("invalid event state diff") + } + atomic.StoreInt32(&w.isrec, 1) + return nil + default: + // TODO(rjeczalik): rewatch newpath only if exists? + // TODO(rjeczalik): migrate w.prev to new watch? + if _, ok := fse.watches[newpath]; ok { + return errAlreadyWatched + } + if err := fse.Unwatch(oldpath); err != nil { + return err + } + // TODO(rjeczalik): revert unwatch if watch fails? + return fse.watch(newpath, newevent, 1) + } +} + +// Close unwatches all watch-points. +func (fse *fsevents) Close() error { + for _, w := range fse.watches { + w.Stop() + } + fse.watches = nil + return nil +} diff --git a/vendor/github.com/zillode/notify/watcher_fsevents_cgo.go b/vendor/github.com/zillode/notify/watcher_fsevents_cgo.go new file mode 100644 index 000000000..5be64632e --- /dev/null +++ b/vendor/github.com/zillode/notify/watcher_fsevents_cgo.go @@ -0,0 +1,190 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build darwin,!kqueue + +package notify + +/* +#include + +typedef void (*CFRunLoopPerformCallBack)(void*); + +void gosource(void *); +void gostream(uintptr_t, uintptr_t, size_t, uintptr_t, uintptr_t, uintptr_t); + +static FSEventStreamRef EventStreamCreate(FSEventStreamContext * context, uintptr_t info, CFArrayRef paths, FSEventStreamEventId since, CFTimeInterval latency, FSEventStreamCreateFlags flags) { + context->info = (void*) info; + return FSEventStreamCreate(NULL, (FSEventStreamCallback) gostream, context, paths, since, latency, flags); +} + +#cgo LDFLAGS: -framework CoreServices +*/ +import "C" + +import ( + "errors" + "os" + "sync" + "sync/atomic" + "time" + "unsafe" +) + +var nilstream C.FSEventStreamRef + +// Default arguments for FSEventStreamCreate function. +var ( + latency C.CFTimeInterval + flags = C.FSEventStreamCreateFlags(C.kFSEventStreamCreateFlagFileEvents | C.kFSEventStreamCreateFlagNoDefer) + since = uint64(C.FSEventsGetCurrentEventId()) +) + +var runloop C.CFRunLoopRef // global runloop which all streams are registered with +var wg sync.WaitGroup // used to wait until the runloop starts + +// source is used for synchronization purposes - it signals when runloop has +// started and is ready via the wg. It also serves purpose of a dummy source, +// thanks to it the runloop does not return as it also has at least one source +// registered. +var source = C.CFRunLoopSourceCreate(nil, 0, &C.CFRunLoopSourceContext{ + perform: (C.CFRunLoopPerformCallBack)(C.gosource), +}) + +// Errors returned when FSEvents functions fail. +var ( + errCreate = os.NewSyscallError("FSEventStreamCreate", errors.New("NULL")) + errStart = os.NewSyscallError("FSEventStreamStart", errors.New("false")) +) + +// initializes the global runloop and ensures any created stream awaits its +// readiness. +func init() { + wg.Add(1) + go func() { + runloop = C.CFRunLoopGetCurrent() + C.CFRunLoopAddSource(runloop, source, C.kCFRunLoopDefaultMode) + C.CFRunLoopRun() + panic("runloop has just unexpectedly stopped") + }() + C.CFRunLoopSourceSignal(source) +} + +//export gosource +func gosource(unsafe.Pointer) { + time.Sleep(time.Second) + wg.Done() +} + +//export gostream +func gostream(_, info uintptr, n C.size_t, paths, flags, ids uintptr) { + const ( + offchar = unsafe.Sizeof((*C.char)(nil)) + offflag = unsafe.Sizeof(C.FSEventStreamEventFlags(0)) + offid = unsafe.Sizeof(C.FSEventStreamEventId(0)) + ) + if n == 0 { + return + } + ev := make([]FSEvent, 0, int(n)) + for i := uintptr(0); i < uintptr(n); i++ { + switch flags := *(*uint32)(unsafe.Pointer((flags + i*offflag))); { + case flags&uint32(FSEventsEventIdsWrapped) != 0: + atomic.StoreUint64(&since, uint64(C.FSEventsGetCurrentEventId())) + default: + ev = append(ev, FSEvent{ + Path: C.GoString(*(**C.char)(unsafe.Pointer(paths + i*offchar))), + Flags: flags, + ID: *(*uint64)(unsafe.Pointer(ids + i*offid)), + }) + } + + } + streamFuncs.get(info)(ev) +} + +// StreamFunc is a callback called when stream receives file events. +type streamFunc func([]FSEvent) + +var streamFuncs = streamFuncRegistry{m: map[uintptr]streamFunc{}} + +type streamFuncRegistry struct { + mu sync.Mutex + m map[uintptr]streamFunc + i uintptr +} + +func (r *streamFuncRegistry) get(id uintptr) streamFunc { + r.mu.Lock() + defer r.mu.Unlock() + return r.m[id] +} + +func (r *streamFuncRegistry) add(fn streamFunc) uintptr { + r.mu.Lock() + defer r.mu.Unlock() + r.i++ + r.m[r.i] = fn + return r.i +} + +func (r *streamFuncRegistry) delete(id uintptr) { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.m, id) +} + +// Stream represents single watch-point which listens for events scheduled by +// the global runloop. +type stream struct { + path string + ref C.FSEventStreamRef + info uintptr +} + +// NewStream creates a stream for given path, listening for file events and +// calling fn upon receiving any. +func newStream(path string, fn streamFunc) *stream { + return &stream{ + path: path, + info: streamFuncs.add(fn), + } +} + +// Start creates a FSEventStream for the given path and schedules it with +// global runloop. It's a nop if the stream was already started. +func (s *stream) Start() error { + if s.ref != nilstream { + return nil + } + wg.Wait() + p := C.CFStringCreateWithCStringNoCopy(nil, C.CString(s.path), C.kCFStringEncodingUTF8, nil) + path := C.CFArrayCreate(nil, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) + ctx := C.FSEventStreamContext{} + ref := C.EventStreamCreate(&ctx, C.uintptr_t(s.info), path, C.FSEventStreamEventId(atomic.LoadUint64(&since)), latency, flags) + if ref == nilstream { + return errCreate + } + C.FSEventStreamScheduleWithRunLoop(ref, runloop, C.kCFRunLoopDefaultMode) + if C.FSEventStreamStart(ref) == C.Boolean(0) { + C.FSEventStreamInvalidate(ref) + return errStart + } + C.CFRunLoopWakeUp(runloop) + s.ref = ref + return nil +} + +// Stop stops underlying FSEventStream and unregisters it from global runloop. +func (s *stream) Stop() { + if s.ref == nilstream { + return + } + wg.Wait() + C.FSEventStreamStop(s.ref) + C.FSEventStreamInvalidate(s.ref) + C.CFRunLoopWakeUp(runloop) + s.ref = nilstream + streamFuncs.delete(s.info) +} diff --git a/vendor/github.com/zillode/notify/watcher_inotify.go b/vendor/github.com/zillode/notify/watcher_inotify.go new file mode 100644 index 000000000..d082baca0 --- /dev/null +++ b/vendor/github.com/zillode/notify/watcher_inotify.go @@ -0,0 +1,405 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build linux + +package notify + +import ( + "bytes" + "errors" + "path/filepath" + "runtime" + "sync" + "sync/atomic" + "unsafe" + + "golang.org/x/sys/unix" +) + +// eventBufferSize defines the size of the buffer given to read(2) function. One +// should not depend on this value, since it was arbitrary chosen and may be +// changed in the future. +const eventBufferSize = 64 * (unix.SizeofInotifyEvent + unix.PathMax + 1) + +// consumersCount defines the number of consumers in producer-consumer based +// implementation. Each consumer is run in a separate goroutine and has read +// access to watched files map. +const consumersCount = 2 + +const invalidDescriptor = -1 + +// watched is a pair of file path and inotify mask used as a value in +// watched files map. +type watched struct { + path string + mask uint32 +} + +// inotify implements Watcher interface. +type inotify struct { + sync.RWMutex // protects inotify.m map + m map[int32]*watched // watch descriptor to watched object + fd int32 // inotify file descriptor + pipefd []int // pipe's read and write descriptors + epfd int // epoll descriptor + epes []unix.EpollEvent // epoll events + buffer [eventBufferSize]byte // inotify event buffer + wg sync.WaitGroup // wait group used to close main loop + c chan<- EventInfo // event dispatcher channel +} + +// NewWatcher creates new non-recursive inotify backed by inotify. +func newWatcher(c chan<- EventInfo) watcher { + i := &inotify{ + m: make(map[int32]*watched), + fd: invalidDescriptor, + pipefd: []int{invalidDescriptor, invalidDescriptor}, + epfd: invalidDescriptor, + epes: make([]unix.EpollEvent, 0), + c: c, + } + runtime.SetFinalizer(i, func(i *inotify) { + i.epollclose() + if i.fd != invalidDescriptor { + unix.Close(int(i.fd)) + } + }) + return i +} + +// Watch implements notify.watcher interface. +func (i *inotify) Watch(path string, e Event) error { + return i.watch(path, e) +} + +// Rewatch implements notify.watcher interface. +func (i *inotify) Rewatch(path string, _, newevent Event) error { + return i.watch(path, newevent) +} + +// watch adds a new watcher to the set of watched objects or modifies the existing +// one. If called for the first time, this function initializes inotify filesystem +// monitor and starts producer-consumers goroutines. +func (i *inotify) watch(path string, e Event) (err error) { + if e&^(All|Event(unix.IN_ALL_EVENTS)) != 0 { + return errors.New("notify: unknown event") + } + if err = i.lazyinit(); err != nil { + return + } + iwd, err := unix.InotifyAddWatch(int(i.fd), path, encode(e)) + if err != nil { + return + } + i.RLock() + wd := i.m[int32(iwd)] + i.RUnlock() + if wd == nil { + i.Lock() + if i.m[int32(iwd)] == nil { + i.m[int32(iwd)] = &watched{path: path, mask: uint32(e)} + } + i.Unlock() + } else { + i.Lock() + wd.mask = uint32(e) + i.Unlock() + } + return nil +} + +// lazyinit sets up all required file descriptors and starts 1+consumersCount +// goroutines. The producer goroutine blocks until file-system notifications +// occur. Then, all events are read from system buffer and sent to consumer +// goroutines which construct valid notify events. This method uses +// Double-Checked Locking optimization. +func (i *inotify) lazyinit() error { + if atomic.LoadInt32(&i.fd) == invalidDescriptor { + i.Lock() + defer i.Unlock() + if atomic.LoadInt32(&i.fd) == invalidDescriptor { + fd, err := unix.InotifyInit1(unix.IN_CLOEXEC) + if err != nil { + return err + } + i.fd = int32(fd) + if err = i.epollinit(); err != nil { + _, _ = i.epollclose(), unix.Close(int(fd)) // Ignore errors. + i.fd = invalidDescriptor + return err + } + esch := make(chan []*event) + go i.loop(esch) + i.wg.Add(consumersCount) + for n := 0; n < consumersCount; n++ { + go i.send(esch) + } + } + } + return nil +} + +// epollinit opens an epoll file descriptor and creates a pipe which will be +// used to wake up the epoll_wait(2) function. Then, file descriptor associated +// with inotify event queue and the read end of the pipe are added to epoll set. +// Note that `fd` member must be set before this function is called. +func (i *inotify) epollinit() (err error) { + if i.epfd, err = unix.EpollCreate1(0); err != nil { + return + } + if err = unix.Pipe(i.pipefd); err != nil { + return + } + i.epes = []unix.EpollEvent{ + {Events: unix.EPOLLIN, Fd: i.fd}, + {Events: unix.EPOLLIN, Fd: int32(i.pipefd[0])}, + } + if err = unix.EpollCtl(i.epfd, unix.EPOLL_CTL_ADD, int(i.fd), &i.epes[0]); err != nil { + return + } + return unix.EpollCtl(i.epfd, unix.EPOLL_CTL_ADD, i.pipefd[0], &i.epes[1]) +} + +// epollclose closes the file descriptor created by the call to epoll_create(2) +// and two file descriptors opened by pipe(2) function. +func (i *inotify) epollclose() (err error) { + if i.epfd != invalidDescriptor { + if err = unix.Close(i.epfd); err == nil { + i.epfd = invalidDescriptor + } + } + for n, fd := range i.pipefd { + if fd != invalidDescriptor { + switch e := unix.Close(fd); { + case e != nil && err == nil: + err = e + case e == nil: + i.pipefd[n] = invalidDescriptor + } + } + } + return +} + +// loop blocks until either inotify or pipe file descriptor is ready for I/O. +// All read operations triggered by filesystem notifications are forwarded to +// one of the event's consumers. If pipe fd became ready, loop function closes +// all file descriptors opened by lazyinit method and returns afterwards. +func (i *inotify) loop(esch chan<- []*event) { + epes := make([]unix.EpollEvent, 1) + fd := atomic.LoadInt32(&i.fd) + for { + switch _, err := unix.EpollWait(i.epfd, epes, -1); err { + case nil: + switch epes[0].Fd { + case fd: + esch <- i.read() + epes[0].Fd = 0 + case int32(i.pipefd[0]): + i.Lock() + defer i.Unlock() + if err = unix.Close(int(fd)); err != nil && err != unix.EINTR { + panic("notify: close(2) error " + err.Error()) + } + atomic.StoreInt32(&i.fd, invalidDescriptor) + if err = i.epollclose(); err != nil && err != unix.EINTR { + panic("notify: epollclose error " + err.Error()) + } + close(esch) + return + } + case unix.EINTR: + continue + default: // We should never reach this line. + panic("notify: epoll_wait(2) error " + err.Error()) + } + } +} + +// read reads events from an inotify file descriptor. It does not handle errors +// returned from read(2) function since they are not critical to watcher logic. +func (i *inotify) read() (es []*event) { + n, err := unix.Read(int(i.fd), i.buffer[:]) + if err != nil || n < unix.SizeofInotifyEvent { + return + } + var sys *unix.InotifyEvent + nmin := n - unix.SizeofInotifyEvent + for pos, path := 0, ""; pos <= nmin; { + sys = (*unix.InotifyEvent)(unsafe.Pointer(&i.buffer[pos])) + pos += unix.SizeofInotifyEvent + if path = ""; sys.Len > 0 { + endpos := pos + int(sys.Len) + path = string(bytes.TrimRight(i.buffer[pos:endpos], "\x00")) + pos = endpos + } + es = append(es, &event{ + sys: unix.InotifyEvent{ + Wd: sys.Wd, + Mask: sys.Mask, + Cookie: sys.Cookie, + }, + path: path, + }) + } + return +} + +// send is a consumer function which sends events to event dispatcher channel. +// It is run in a separate goroutine in order to not block loop method when +// possibly expensive write operations are performed on inotify map. +func (i *inotify) send(esch <-chan []*event) { + for es := range esch { + for _, e := range i.transform(es) { + if e != nil { + i.c <- e + } + } + } + i.wg.Done() +} + +// transform prepares events read from inotify file descriptor for sending to +// user. It removes invalid events and these which are no longer present in +// inotify map. This method may also split one raw event into two different ones +// when system-dependent result is required. +func (i *inotify) transform(es []*event) []*event { + var multi []*event + i.RLock() + for idx, e := range es { + if e.sys.Mask&(unix.IN_IGNORED|unix.IN_Q_OVERFLOW) != 0 { + es[idx] = nil + continue + } + wd, ok := i.m[e.sys.Wd] + if !ok || e.sys.Mask&encode(Event(wd.mask)) == 0 { + es[idx] = nil + continue + } + if e.path == "" { + e.path = wd.path + } else { + e.path = filepath.Join(wd.path, e.path) + } + multi = append(multi, decode(Event(wd.mask), e)) + if e.event == 0 { + es[idx] = nil + } + } + i.RUnlock() + es = append(es, multi...) + return es +} + +// encode converts notify system-independent events to valid inotify mask +// which can be passed to inotify_add_watch(2) function. +func encode(e Event) uint32 { + if e&Create != 0 { + e = (e ^ Create) | InCreate | InMovedTo + } + if e&Remove != 0 { + e = (e ^ Remove) | InDelete | InDeleteSelf + } + if e&Write != 0 { + e = (e ^ Write) | InModify + } + if e&Rename != 0 { + e = (e ^ Rename) | InMovedFrom | InMoveSelf + } + return uint32(e) +} + +// decode uses internally stored mask to distinguish whether system-independent +// or system-dependent event is requested. The first one is created by modifying +// `e` argument. decode method sets e.event value to 0 when an event should be +// skipped. System-dependent event is set as the function's return value which +// can be nil when the event should not be passed on. +func decode(mask Event, e *event) (syse *event) { + if sysmask := uint32(mask) & e.sys.Mask; sysmask != 0 { + syse = &event{sys: unix.InotifyEvent{ + Wd: e.sys.Wd, + Mask: e.sys.Mask, + Cookie: e.sys.Cookie, + }, event: Event(sysmask), path: e.path} + } + imask := encode(mask) + switch { + case mask&Create != 0 && imask&uint32(InCreate|InMovedTo)&e.sys.Mask != 0: + e.event = Create + case mask&Remove != 0 && imask&uint32(InDelete|InDeleteSelf)&e.sys.Mask != 0: + e.event = Remove + case mask&Write != 0 && imask&uint32(InModify)&e.sys.Mask != 0: + e.event = Write + case mask&Rename != 0 && imask&uint32(InMovedFrom|InMoveSelf)&e.sys.Mask != 0: + e.event = Rename + default: + e.event = 0 + } + return +} + +// Unwatch implements notify.watcher interface. It looks for watch descriptor +// related to registered path and if found, calls inotify_rm_watch(2) function. +// This method is allowed to return EINVAL error when concurrently requested to +// delete identical path. +func (i *inotify) Unwatch(path string) (err error) { + iwd := int32(invalidDescriptor) + i.RLock() + for iwdkey, wd := range i.m { + if wd.path == path { + iwd = iwdkey + break + } + } + i.RUnlock() + if iwd == invalidDescriptor { + return errors.New("notify: path " + path + " is already watched") + } + fd := atomic.LoadInt32(&i.fd) + if err = removeInotifyWatch(fd, iwd); err != nil { + return + } + i.Lock() + delete(i.m, iwd) + i.Unlock() + return nil +} + +// Close implements notify.watcher interface. It removes all existing watch +// descriptors and wakes up producer goroutine by sending data to the write end +// of the pipe. The function waits for a signal from producer which means that +// all operations on current monitoring instance are done. +func (i *inotify) Close() (err error) { + i.Lock() + if fd := atomic.LoadInt32(&i.fd); fd == invalidDescriptor { + i.Unlock() + return nil + } + for iwd := range i.m { + if e := removeInotifyWatch(i.fd, iwd); e != nil && err == nil { + err = e + } + delete(i.m, iwd) + } + switch _, errwrite := unix.Write(i.pipefd[1], []byte{0x00}); { + case errwrite != nil && err == nil: + err = errwrite + fallthrough + case errwrite != nil: + i.Unlock() + default: + i.Unlock() + i.wg.Wait() + } + return +} + +// if path was removed, notify already removed the watch and returns EINVAL error +func removeInotifyWatch(fd int32, iwd int32) (err error) { + if _, err = unix.InotifyRmWatch(int(fd), uint32(iwd)); err != nil && err != unix.EINVAL { + return + } + return nil +} diff --git a/vendor/github.com/zillode/notify/watcher_kqueue.go b/vendor/github.com/zillode/notify/watcher_kqueue.go new file mode 100644 index 000000000..22e3c2c4a --- /dev/null +++ b/vendor/github.com/zillode/notify/watcher_kqueue.go @@ -0,0 +1,189 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build darwin,kqueue dragonfly freebsd netbsd openbsd + +package notify + +import ( + "fmt" + "os" + "syscall" +) + +// newTrigger returns implementation of trigger. +func newTrigger(pthLkp map[string]*watched) trigger { + return &kq{ + pthLkp: pthLkp, + idLkp: make(map[int]*watched), + } +} + +// kq is a structure implementing trigger for kqueue. +type kq struct { + // fd is a kqueue file descriptor + fd int + // pipefds are file descriptors used to stop `Kevent` call. + pipefds [2]int + // idLkp is a data structure mapping file descriptors with data about watching + // represented by them files/directories. + idLkp map[int]*watched + // pthLkp is a structure mapping monitored files/dir with data about them, + // shared with parent trg structure + pthLkp map[string]*watched +} + +// watched is a data structure representing watched file/directory. +type watched struct { + trgWatched + // fd is a file descriptor for watched file/directory. + fd int +} + +// Stop implements trigger. +func (k *kq) Stop() (err error) { + // trigger event used to interrupt Kevent call. + _, err = syscall.Write(k.pipefds[1], []byte{0x00}) + return +} + +// Close implements trigger. +func (k *kq) Close() error { + return syscall.Close(k.fd) +} + +// NewWatched implements trigger. +func (*kq) NewWatched(p string, fi os.FileInfo) (*watched, error) { + fd, err := syscall.Open(p, syscall.O_NONBLOCK|syscall.O_RDONLY, 0) + if err != nil { + return nil, err + } + return &watched{ + trgWatched: trgWatched{p: p, fi: fi}, + fd: fd, + }, nil +} + +// Record implements trigger. +func (k *kq) Record(w *watched) { + k.idLkp[w.fd], k.pthLkp[w.p] = w, w +} + +// Del implements trigger. +func (k *kq) Del(w *watched) { + syscall.Close(w.fd) + delete(k.idLkp, w.fd) + delete(k.pthLkp, w.p) +} + +func inter2kq(n interface{}) syscall.Kevent_t { + kq, ok := n.(syscall.Kevent_t) + if !ok { + panic(fmt.Sprintf("kqueue: type should be Kevent_t, %T instead", n)) + } + return kq +} + +// Init implements trigger. +func (k *kq) Init() (err error) { + if k.fd, err = syscall.Kqueue(); err != nil { + return + } + // Creates pipe used to stop `Kevent` call by registering it, + // watching read end and writing to other end of it. + if err = syscall.Pipe(k.pipefds[:]); err != nil { + return nonil(err, k.Close()) + } + var kevn [1]syscall.Kevent_t + syscall.SetKevent(&kevn[0], k.pipefds[0], syscall.EVFILT_READ, syscall.EV_ADD) + if _, err = syscall.Kevent(k.fd, kevn[:], nil, nil); err != nil { + return nonil(err, k.Close()) + } + return +} + +// Unwatch implements trigger. +func (k *kq) Unwatch(w *watched) (err error) { + var kevn [1]syscall.Kevent_t + syscall.SetKevent(&kevn[0], w.fd, syscall.EVFILT_VNODE, syscall.EV_DELETE) + + _, err = syscall.Kevent(k.fd, kevn[:], nil, nil) + return +} + +// Watch implements trigger. +func (k *kq) Watch(fi os.FileInfo, w *watched, e int64) (err error) { + var kevn [1]syscall.Kevent_t + syscall.SetKevent(&kevn[0], w.fd, syscall.EVFILT_VNODE, + syscall.EV_ADD|syscall.EV_CLEAR) + kevn[0].Fflags = uint32(e) + + _, err = syscall.Kevent(k.fd, kevn[:], nil, nil) + return +} + +// Wait implements trigger. +func (k *kq) Wait() (interface{}, error) { + var ( + kevn [1]syscall.Kevent_t + err error + ) + kevn[0] = syscall.Kevent_t{} + _, err = syscall.Kevent(k.fd, nil, kevn[:], nil) + + return kevn[0], err +} + +// Watched implements trigger. +func (k *kq) Watched(n interface{}) (*watched, int64, error) { + kevn, ok := n.(syscall.Kevent_t) + if !ok { + panic(fmt.Sprintf("kq: type should be syscall.Kevent_t, %T instead", kevn)) + } + if _, ok = k.idLkp[int(kevn.Ident)]; !ok { + return nil, 0, errNotWatched + } + return k.idLkp[int(kevn.Ident)], int64(kevn.Fflags), nil +} + +// IsStop implements trigger. +func (k *kq) IsStop(n interface{}, err error) bool { + return int(inter2kq(n).Ident) == k.pipefds[0] +} + +func init() { + encode = func(e Event, dir bool) (o int64) { + // Create event is not supported by kqueue. Instead NoteWrite event will + // be registered for a directory. If this event will be reported on dir + // which is to be monitored for Create, dir will be rescanned + // and Create events will be generated and returned for new files. + // In case of files, if not requested NoteRename event is reported, + // it will be ignored. + o = int64(e &^ Create) + if (e&Create != 0 && dir) || e&Write != 0 { + o = (o &^ int64(Write)) | int64(NoteWrite) + } + if e&Rename != 0 { + o = (o &^ int64(Rename)) | int64(NoteRename) + } + if e&Remove != 0 { + o = (o &^ int64(Remove)) | int64(NoteDelete) + } + return + } + nat2not = map[Event]Event{ + NoteWrite: Write, + NoteRename: Rename, + NoteDelete: Remove, + NoteExtend: Event(0), + NoteAttrib: Event(0), + NoteRevoke: Event(0), + NoteLink: Event(0), + } + not2nat = map[Event]Event{ + Write: NoteWrite, + Rename: NoteRename, + Remove: NoteDelete, + } +} diff --git a/vendor/github.com/zillode/notify/watcher_readdcw.go b/vendor/github.com/zillode/notify/watcher_readdcw.go new file mode 100644 index 000000000..1494fcd79 --- /dev/null +++ b/vendor/github.com/zillode/notify/watcher_readdcw.go @@ -0,0 +1,582 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build windows + +package notify + +import ( + "errors" + "runtime" + "sync" + "sync/atomic" + "syscall" + "unsafe" +) + +// readBufferSize defines the size of an array in which read statuses are stored. +// The buffer have to be DWORD-aligned and, if notify is used in monitoring a +// directory over the network, its size must not be greater than 64KB. Each of +// watched directories uses its own buffer for storing events. +const readBufferSize = 4096 + +// Since all operations which go through the Windows completion routine are done +// asynchronously, filter may set one of the constants belor. They were defined +// in order to distinguish whether current folder should be re-registered in +// ReadDirectoryChangesW function or some control operations need to be executed. +const ( + stateRewatch uint32 = 1 << (28 + iota) + stateUnwatch + stateCPClose +) + +// Filter used in current implementation was split into four segments: +// - bits 0-11 store ReadDirectoryChangesW filters, +// - bits 12-19 store File notify actions, +// - bits 20-27 store notify specific events and flags, +// - bits 28-31 store states which are used in loop's FSM. +// Constants below are used as masks to retrieve only specific filter parts. +const ( + onlyNotifyChanges uint32 = 0x00000FFF + onlyNGlobalEvents uint32 = 0x0FF00000 + onlyMachineStates uint32 = 0xF0000000 +) + +// grip represents a single watched directory. It stores the data required by +// ReadDirectoryChangesW function. Only the filter, recursive, and handle members +// may by modified by watcher implementation. Rest of the them have to remain +// constant since they are used by Windows completion routine. This indicates that +// grip can be removed only when all operations on the file handle are finished. +type grip struct { + handle syscall.Handle + filter uint32 + recursive bool + pathw []uint16 + buffer [readBufferSize]byte + parent *watched + ovlapped *overlappedEx +} + +// overlappedEx stores information used in asynchronous input and output. +// Additionally, overlappedEx contains a pointer to 'grip' item which is used in +// order to gather the structure in which the overlappedEx object was created. +type overlappedEx struct { + syscall.Overlapped + parent *grip +} + +// newGrip creates a new file handle that can be used in overlapped operations. +// Then, the handle is associated with I/O completion port 'cph' and its value +// is stored in newly created 'grip' object. +func newGrip(cph syscall.Handle, parent *watched, filter uint32) (*grip, error) { + g := &grip{ + handle: syscall.InvalidHandle, + filter: filter, + recursive: parent.recursive, + pathw: parent.pathw, + parent: parent, + ovlapped: &overlappedEx{}, + } + if err := g.register(cph); err != nil { + return nil, err + } + g.ovlapped.parent = g + return g, nil +} + +// NOTE : Thread safe +func (g *grip) register(cph syscall.Handle) (err error) { + if g.handle, err = syscall.CreateFile( + &g.pathw[0], + syscall.FILE_LIST_DIRECTORY, + syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, + nil, + syscall.OPEN_EXISTING, + syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OVERLAPPED, + 0, + ); err != nil { + return + } + if _, err = syscall.CreateIoCompletionPort(g.handle, cph, 0, 0); err != nil { + syscall.CloseHandle(g.handle) + return + } + return g.readDirChanges() +} + +// readDirChanges tells the system to store file change information in grip's +// buffer. Directory changes that occur between calls to this function are added +// to the buffer and then, returned with the next call. +func (g *grip) readDirChanges() error { + return syscall.ReadDirectoryChanges( + g.handle, + &g.buffer[0], + uint32(unsafe.Sizeof(g.buffer)), + g.recursive, + encode(g.filter), + nil, + (*syscall.Overlapped)(unsafe.Pointer(g.ovlapped)), + 0, + ) +} + +// encode transforms a generic filter, which contains platform independent and +// implementation specific bit fields, to value that can be used as NotifyFilter +// parameter in ReadDirectoryChangesW function. +func encode(filter uint32) uint32 { + e := Event(filter & (onlyNGlobalEvents | onlyNotifyChanges)) + if e&dirmarker != 0 { + return uint32(FileNotifyChangeDirName) + } + if e&Create != 0 { + e = (e ^ Create) | FileNotifyChangeFileName + } + if e&Remove != 0 { + e = (e ^ Remove) | FileNotifyChangeFileName + } + if e&Write != 0 { + e = (e ^ Write) | FileNotifyChangeAttributes | FileNotifyChangeSize | + FileNotifyChangeCreation | FileNotifyChangeSecurity + } + if e&Rename != 0 { + e = (e ^ Rename) | FileNotifyChangeFileName + } + return uint32(e) +} + +// watched is made in order to check whether an action comes from a directory or +// file. This approach requires two file handlers per single monitored folder. The +// second grip handles actions which include creating or deleting a directory. If +// these processes are not monitored, only the first grip is created. +type watched struct { + filter uint32 + recursive bool + count uint8 + pathw []uint16 + digrip [2]*grip +} + +// newWatched creates a new watched instance. It splits the filter variable into +// two parts. The first part is responsible for watching all events which can be +// created for a file in watched directory structure and the second one watches +// only directory Create/Remove actions. If all operations succeed, the Create +// message is sent to I/O completion port queue for further processing. +func newWatched(cph syscall.Handle, filter uint32, recursive bool, + path string) (wd *watched, err error) { + wd = &watched{ + filter: filter, + recursive: recursive, + } + if wd.pathw, err = syscall.UTF16FromString(path); err != nil { + return + } + if err = wd.recreate(cph); err != nil { + return + } + return wd, nil +} + +// TODO : doc +func (wd *watched) recreate(cph syscall.Handle) (err error) { + filefilter := wd.filter &^ uint32(FileNotifyChangeDirName) + if err = wd.updateGrip(0, cph, filefilter == 0, filefilter); err != nil { + return + } + dirfilter := wd.filter & uint32(FileNotifyChangeDirName|Create|Remove) + if err = wd.updateGrip(1, cph, dirfilter == 0, wd.filter|uint32(dirmarker)); err != nil { + return + } + wd.filter &^= onlyMachineStates + return +} + +// TODO : doc +func (wd *watched) updateGrip(idx int, cph syscall.Handle, reset bool, + newflag uint32) (err error) { + if reset { + wd.digrip[idx] = nil + } else { + if wd.digrip[idx] == nil { + if wd.digrip[idx], err = newGrip(cph, wd, newflag); err != nil { + wd.closeHandle() + return + } + } else { + wd.digrip[idx].filter = newflag + wd.digrip[idx].recursive = wd.recursive + if err = wd.digrip[idx].register(cph); err != nil { + wd.closeHandle() + return + } + } + wd.count++ + } + return +} + +// closeHandle closes handles that are stored in digrip array. Function always +// tries to close all of the handlers before it exits, even when there are errors +// returned from the operating system kernel. +func (wd *watched) closeHandle() (err error) { + for _, g := range wd.digrip { + if g != nil && g.handle != syscall.InvalidHandle { + switch suberr := syscall.CloseHandle(g.handle); { + case suberr == nil: + g.handle = syscall.InvalidHandle + case err == nil: + err = suberr + } + } + } + return +} + +// watcher implements Watcher interface. It stores a set of watched directories. +// All operations which remove watched objects from map `m` must be performed in +// loop goroutine since these structures are used internally by operating system. +type readdcw struct { + sync.Mutex + m map[string]*watched + cph syscall.Handle + start bool + wg sync.WaitGroup + c chan<- EventInfo +} + +// NewWatcher creates new non-recursive watcher backed by ReadDirectoryChangesW. +func newWatcher(c chan<- EventInfo) watcher { + r := &readdcw{ + m: make(map[string]*watched), + cph: syscall.InvalidHandle, + c: c, + } + runtime.SetFinalizer(r, func(r *readdcw) { + if r.cph != syscall.InvalidHandle { + syscall.CloseHandle(r.cph) + } + }) + return r +} + +// Watch implements notify.Watcher interface. +func (r *readdcw) Watch(path string, event Event) error { + return r.watch(path, event, false) +} + +// RecursiveWatch implements notify.RecursiveWatcher interface. +func (r *readdcw) RecursiveWatch(path string, event Event) error { + return r.watch(path, event, true) +} + +// watch inserts a directory to the group of watched folders. If watched folder +// already exists, function tries to rewatch it with new filters(NOT VALID). Moreover, +// watch starts the main event loop goroutine when called for the first time. +func (r *readdcw) watch(path string, event Event, recursive bool) (err error) { + if event&^(All|fileNotifyChangeAll) != 0 { + return errors.New("notify: unknown event") + } + r.Lock() + wd, ok := r.m[path] + r.Unlock() + if !ok { + if err = r.lazyinit(); err != nil { + return + } + r.Lock() + defer r.Unlock() + if wd, ok = r.m[path]; ok { + dbgprint("watch: exists already") + return + } + if wd, err = newWatched(r.cph, uint32(event), recursive, path); err != nil { + return + } + r.m[path] = wd + dbgprint("watch: new watch added") + } else { + dbgprint("watch: exists already") + } + return nil +} + +// lazyinit creates an I/O completion port and starts the main event processing +// loop. This method uses Double-Checked Locking optimization. +func (r *readdcw) lazyinit() (err error) { + invalid := uintptr(syscall.InvalidHandle) + if atomic.LoadUintptr((*uintptr)(&r.cph)) == invalid { + r.Lock() + defer r.Unlock() + if atomic.LoadUintptr((*uintptr)(&r.cph)) == invalid { + cph := syscall.InvalidHandle + if cph, err = syscall.CreateIoCompletionPort(cph, 0, 0, 0); err != nil { + return + } + r.cph, r.start = cph, true + go r.loop() + } + } + return +} + +// TODO(pknap) : doc +func (r *readdcw) loop() { + var n, key uint32 + var overlapped *syscall.Overlapped + for { + err := syscall.GetQueuedCompletionStatus(r.cph, &n, &key, &overlapped, syscall.INFINITE) + if key == stateCPClose { + r.Lock() + handle := r.cph + r.cph = syscall.InvalidHandle + r.Unlock() + syscall.CloseHandle(handle) + r.wg.Done() + return + } + if overlapped == nil { + // TODO: check key == rewatch delete or 0(panic) + continue + } + overEx := (*overlappedEx)(unsafe.Pointer(overlapped)) + if n != 0 { + r.loopevent(n, overEx) + if err = overEx.parent.readDirChanges(); err != nil { + // TODO: error handling + } + } + r.loopstate(overEx) + } +} + +// TODO(pknap) : doc +func (r *readdcw) loopstate(overEx *overlappedEx) { + r.Lock() + defer r.Unlock() + filter := overEx.parent.parent.filter + if filter&onlyMachineStates == 0 { + return + } + if overEx.parent.parent.count--; overEx.parent.parent.count == 0 { + switch filter & onlyMachineStates { + case stateRewatch: + dbgprint("loopstate rewatch") + overEx.parent.parent.recreate(r.cph) + case stateUnwatch: + dbgprint("loopstate unwatch") + delete(r.m, syscall.UTF16ToString(overEx.parent.pathw)) + case stateCPClose: + default: + panic(`notify: windows loopstate logic error`) + } + } +} + +// TODO(pknap) : doc +func (r *readdcw) loopevent(n uint32, overEx *overlappedEx) { + events := []*event{} + var currOffset uint32 + for { + raw := (*syscall.FileNotifyInformation)(unsafe.Pointer(&overEx.parent.buffer[currOffset])) + name := syscall.UTF16ToString((*[syscall.MAX_LONG_PATH]uint16)(unsafe.Pointer(&raw.FileName))[:raw.FileNameLength>>1]) + events = append(events, &event{ + pathw: overEx.parent.pathw, + filter: overEx.parent.filter, + action: raw.Action, + name: name, + }) + if raw.NextEntryOffset == 0 { + break + } + if currOffset += raw.NextEntryOffset; currOffset >= n { + break + } + } + r.send(events) +} + +// TODO(pknap) : doc +func (r *readdcw) send(es []*event) { + for _, e := range es { + var syse Event + if e.e, syse = decode(e.filter, e.action); e.e == 0 && syse == 0 { + continue + } + switch { + case e.action == syscall.FILE_ACTION_MODIFIED: + e.ftype = fTypeUnknown + case e.filter&uint32(dirmarker) != 0: + e.ftype = fTypeDirectory + default: + e.ftype = fTypeFile + } + switch { + case e.e == 0: + e.e = syse + case syse != 0: + r.c <- &event{ + pathw: e.pathw, + name: e.name, + ftype: e.ftype, + action: e.action, + filter: e.filter, + e: syse, + } + } + r.c <- e + } +} + +// Rewatch implements notify.Rewatcher interface. +func (r *readdcw) Rewatch(path string, oldevent, newevent Event) error { + return r.rewatch(path, uint32(oldevent), uint32(newevent), false) +} + +// RecursiveRewatch implements notify.RecursiveRewatcher interface. +func (r *readdcw) RecursiveRewatch(oldpath, newpath string, oldevent, + newevent Event) error { + if oldpath != newpath { + if err := r.unwatch(oldpath); err != nil { + return err + } + return r.watch(newpath, newevent, true) + } + return r.rewatch(newpath, uint32(oldevent), uint32(newevent), true) +} + +// TODO : (pknap) doc. +func (r *readdcw) rewatch(path string, oldevent, newevent uint32, recursive bool) (err error) { + if Event(newevent)&^(All|fileNotifyChangeAll) != 0 { + return errors.New("notify: unknown event") + } + var wd *watched + r.Lock() + defer r.Unlock() + if wd, err = r.nonStateWatchedLocked(path); err != nil { + return + } + if wd.filter&(onlyNotifyChanges|onlyNGlobalEvents) != oldevent { + panic(`notify: windows re-watcher logic error`) + } + wd.filter = stateRewatch | newevent + wd.recursive, recursive = recursive, wd.recursive + if err = wd.closeHandle(); err != nil { + wd.filter = oldevent + wd.recursive = recursive + return + } + return +} + +// TODO : pknap +func (r *readdcw) nonStateWatchedLocked(path string) (wd *watched, err error) { + wd, ok := r.m[path] + if !ok || wd == nil { + err = errors.New(`notify: ` + path + ` path is unwatched`) + return + } + if wd.filter&onlyMachineStates != 0 { + err = errors.New(`notify: another re/unwatching operation in progress`) + return + } + return +} + +// Unwatch implements notify.Watcher interface. +func (r *readdcw) Unwatch(path string) error { + return r.unwatch(path) +} + +// RecursiveUnwatch implements notify.RecursiveWatcher interface. +func (r *readdcw) RecursiveUnwatch(path string) error { + return r.unwatch(path) +} + +// TODO : pknap +func (r *readdcw) unwatch(path string) (err error) { + var wd *watched + r.Lock() + defer r.Unlock() + if wd, err = r.nonStateWatchedLocked(path); err != nil { + return + } + wd.filter |= stateUnwatch + if err = wd.closeHandle(); err != nil { + wd.filter &^= stateUnwatch + return + } + if _, attrErr := syscall.GetFileAttributes(&wd.pathw[0]); attrErr != nil { + for _, g := range wd.digrip { + if g != nil { + dbgprint("unwatch: posting") + if err = syscall.PostQueuedCompletionStatus(r.cph, 0, 0, (*syscall.Overlapped)(unsafe.Pointer(g.ovlapped))); err != nil { + wd.filter &^= stateUnwatch + return + } + } + } + } + return +} + +// Close resets the whole watcher object, closes all existing file descriptors, +// and sends stateCPClose state as completion key to the main watcher's loop. +func (r *readdcw) Close() (err error) { + r.Lock() + if !r.start { + r.Unlock() + return nil + } + for _, wd := range r.m { + wd.filter &^= onlyMachineStates + wd.filter |= stateCPClose + if e := wd.closeHandle(); e != nil && err == nil { + err = e + } + } + r.start = false + r.Unlock() + r.wg.Add(1) + if e := syscall.PostQueuedCompletionStatus(r.cph, 0, stateCPClose, nil); e != nil && err == nil { + return e + } + r.wg.Wait() + return +} + +// decode creates a notify event from both non-raw filter and action which was +// returned from completion routine. Function may return Event(0) in case when +// filter was replaced by a new value which does not contain fields that are +// valid with passed action. +func decode(filter, action uint32) (Event, Event) { + switch action { + case syscall.FILE_ACTION_ADDED: + return gensys(filter, Create, FileActionAdded) + case syscall.FILE_ACTION_REMOVED: + return gensys(filter, Remove, FileActionRemoved) + case syscall.FILE_ACTION_MODIFIED: + return gensys(filter, Write, FileActionModified) + case syscall.FILE_ACTION_RENAMED_OLD_NAME: + return gensys(filter, Rename, FileActionRenamedOldName) + case syscall.FILE_ACTION_RENAMED_NEW_NAME: + return gensys(filter, Rename, FileActionRenamedNewName) + } + panic(`notify: cannot decode internal mask`) +} + +// gensys decides whether the Windows action, system-independent event or both +// of them should be returned. Since the grip's filter may be atomically changed +// during watcher lifetime, it is possible that neither Windows nor notify masks +// are watched by the user when this function is called. +func gensys(filter uint32, ge, se Event) (gene, syse Event) { + isdir := filter&uint32(dirmarker) != 0 + if isdir && filter&uint32(FileNotifyChangeDirName) != 0 || + !isdir && filter&uint32(FileNotifyChangeFileName) != 0 || + filter&uint32(fileNotifyChangeModified) != 0 { + syse = se + } + if filter&uint32(ge) != 0 { + gene = ge + } + return +} diff --git a/vendor/github.com/zillode/notify/watcher_stub.go b/vendor/github.com/zillode/notify/watcher_stub.go new file mode 100644 index 000000000..68b9c135b --- /dev/null +++ b/vendor/github.com/zillode/notify/watcher_stub.go @@ -0,0 +1,23 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build !darwin,!linux,!freebsd,!dragonfly,!netbsd,!openbsd,!windows +// +build !kqueue,!solaris + +package notify + +import "errors" + +type stub struct{ error } + +// newWatcher stub. +func newWatcher(chan<- EventInfo) watcher { + return stub{errors.New("notify: not implemented")} +} + +// Following methods implement notify.watcher interface. +func (s stub) Watch(string, Event) error { return s } +func (s stub) Rewatch(string, Event, Event) error { return s } +func (s stub) Unwatch(string) (err error) { return s } +func (s stub) Close() error { return s } diff --git a/vendor/github.com/zillode/notify/watcher_trigger.go b/vendor/github.com/zillode/notify/watcher_trigger.go new file mode 100644 index 000000000..78151f909 --- /dev/null +++ b/vendor/github.com/zillode/notify/watcher_trigger.go @@ -0,0 +1,449 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build darwin,kqueue dragonfly freebsd netbsd openbsd solaris + +// watcher_trigger is used for FEN and kqueue which behave similarly: +// only files and dirs can be watched directly, but not files inside dirs. +// As a result Create events have to be generated by implementation when +// after Write event is returned for watched dir, it is rescanned and Create +// event is returned for new files and these are automatically added +// to watchlist. In case of removal of watched directory, native system returns +// events for all files, but for Rename, they also need to be generated. +// As a result native system works as something like trigger for rescan, +// but contains additional data about dir in which changes occurred. For files +// detailed data is returned. +// Usage of watcher_trigger requires: +// - trigger implementation, +// - encode func, +// - not2nat, nat2not maps. +// Required manual operations on filesystem can lead to loss of precision. + +package notify + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "syscall" +) + +// trigger is to be implemented by platform implementation like FEN or kqueue. +type trigger interface { + // Close closes watcher's main native file descriptor. + Close() error + // Stop waiting for new events. + Stop() error + // Create new instance of watched. + NewWatched(string, os.FileInfo) (*watched, error) + // Record internally new *watched instance. + Record(*watched) + // Del removes internal copy of *watched instance. + Del(*watched) + // Watched returns *watched instance and native events for native type. + Watched(interface{}) (*watched, int64, error) + // Init initializes native watcher call. + Init() error + // Watch starts watching provided file/dir. + Watch(os.FileInfo, *watched, int64) error + // Unwatch stops watching provided file/dir. + Unwatch(*watched) error + // Wait for new events. + Wait() (interface{}, error) + // IsStop checks if Wait finished because of request watcher's stop. + IsStop(n interface{}, err error) bool +} + +// trgWatched is a the base data structure representing watched file/directory. +// The platform specific full data structure (watched) must embed this type. +type trgWatched struct { + // p is a path to watched file/directory. + p string + // fi provides information about watched file/dir. + fi os.FileInfo + // eDir represents events watched directly. + eDir Event + // eNonDir represents events watched indirectly. + eNonDir Event +} + +// encode Event to native representation. Implementation is to be provided by +// platform specific implementation. +var encode func(Event, bool) int64 + +var ( + // nat2not matches native events to notify's ones. To be initialized by + // platform dependent implementation. + nat2not map[Event]Event + // not2nat matches notify's events to native ones. To be initialized by + // platform dependent implementation. + not2nat map[Event]Event +) + +// trg is a main structure implementing watcher. +type trg struct { + sync.Mutex + // s is a channel used to stop monitoring. + s chan struct{} + // c is a channel used to pass events further. + c chan<- EventInfo + // pthLkp is a data structure mapping file names with data about watching + // represented by them files/directories. + pthLkp map[string]*watched + // t is a platform dependent implementation of trigger. + t trigger +} + +// newWatcher returns new watcher's implementation. +func newWatcher(c chan<- EventInfo) watcher { + t := &trg{ + s: make(chan struct{}, 1), + pthLkp: make(map[string]*watched, 0), + c: c, + } + t.t = newTrigger(t.pthLkp) + if err := t.t.Init(); err != nil { + panic(err) + } + go t.monitor() + return t +} + +// Close implements watcher. +func (t *trg) Close() (err error) { + t.Lock() + if err = t.t.Stop(); err != nil { + t.Unlock() + return + } + <-t.s + var e error + for _, w := range t.pthLkp { + if e = t.unwatch(w.p, w.fi); e != nil { + dbgprintf("trg: unwatch %q failed: %q\n", w.p, e) + err = nonil(err, e) + } + } + if e = t.t.Close(); e != nil { + dbgprintf("trg: closing native watch failed: %q\n", e) + err = nonil(err, e) + } + if remaining := len(t.pthLkp); remaining != 0 { + err = nonil(err, fmt.Errorf("Not all watches were removed: len(t.pthLkp) == %v", len(t.pthLkp))) + } + t.Unlock() + return +} + +// send reported events one by one through chan. +func (t *trg) send(evn []event) { + for i := range evn { + t.c <- &evn[i] + } +} + +// singlewatch starts to watch given p file/directory. +func (t *trg) singlewatch(p string, e Event, direct mode, fi os.FileInfo) (err error) { + w, ok := t.pthLkp[p] + if !ok { + if w, err = t.t.NewWatched(p, fi); err != nil { + return + } + } + switch direct { + case dir: + w.eDir |= e + case ndir: + w.eNonDir |= e + case both: + w.eDir |= e + w.eNonDir |= e + } + if err = t.t.Watch(fi, w, encode(w.eDir|w.eNonDir, fi.IsDir())); err != nil { + return + } + if !ok { + t.t.Record(w) + return nil + } + return errAlreadyWatched +} + +// decode converts event received from native to notify.Event +// representation taking into account requested events (w). +func decode(o int64, w Event) (e Event) { + for f, n := range nat2not { + if o&int64(f) != 0 { + if w&f != 0 { + e |= f + } + if w&n != 0 { + e |= n + } + } + } + + return +} + +func (t *trg) watch(p string, e Event, fi os.FileInfo) error { + if err := t.singlewatch(p, e, dir, fi); err != nil { + if err != errAlreadyWatched { + return err + } + } + if fi.IsDir() { + err := t.walk(p, func(fi os.FileInfo) (err error) { + if err = t.singlewatch(filepath.Join(p, fi.Name()), e, ndir, + fi); err != nil { + if err != errAlreadyWatched { + return + } + } + return nil + }) + if err != nil { + return err + } + } + return nil +} + +// walk runs f func on each file/dir from p directory. +func (t *trg) walk(p string, fn func(os.FileInfo) error) error { + fp, err := os.Open(p) + if err != nil { + return err + } + ls, err := fp.Readdir(0) + fp.Close() + if err != nil { + return err + } + for i := range ls { + if err := fn(ls[i]); err != nil { + return err + } + } + return nil +} + +func (t *trg) unwatch(p string, fi os.FileInfo) error { + if fi.IsDir() { + err := t.walk(p, func(fi os.FileInfo) error { + err := t.singleunwatch(filepath.Join(p, fi.Name()), ndir) + if err != errNotWatched { + return err + } + return nil + }) + if err != nil { + return err + } + } + return t.singleunwatch(p, dir) +} + +// Watch implements Watcher interface. +func (t *trg) Watch(p string, e Event) error { + fi, err := os.Stat(p) + if err != nil { + return err + } + t.Lock() + err = t.watch(p, e, fi) + t.Unlock() + return err +} + +// Unwatch implements Watcher interface. +func (t *trg) Unwatch(p string) error { + fi, err := os.Stat(p) + if err != nil { + return err + } + t.Lock() + err = t.unwatch(p, fi) + t.Unlock() + return err +} + +// Rewatch implements Watcher interface. +// +// TODO(rjeczalik): This is a naive hack. Rewrite might help. +func (t *trg) Rewatch(p string, _, e Event) error { + fi, err := os.Stat(p) + if err != nil { + return err + } + t.Lock() + if err = t.unwatch(p, fi); err == nil { + // TODO(rjeczalik): If watch fails then we leave trigger in inconsistent + // state. Handle? Panic? Native version of rewatch? + err = t.watch(p, e, fi) + } + t.Unlock() + return nil +} + +func (*trg) file(w *watched, n interface{}, e Event) (evn []event) { + evn = append(evn, event{w.p, e, w.fi.IsDir(), n}) + return +} + +func (t *trg) dir(w *watched, n interface{}, e, ge Event) (evn []event) { + // If it's dir and delete we have to send it and continue, because + // other processing relies on opening (in this case not existing) dir. + // Events for contents of this dir are reported by native impl. + // However events for rename must be generated for all monitored files + // inside of moved directory, because native impl does not report it independently + // for each file descriptor being moved in result of move action on + // parent directory. + if (ge & (not2nat[Rename] | not2nat[Remove])) != 0 { + // Write is reported also for Remove on directory. Because of that + // we have to filter it out explicitly. + evn = append(evn, event{w.p, e & ^Write & ^not2nat[Write], true, n}) + if ge¬2nat[Rename] != 0 { + for p := range t.pthLkp { + if strings.HasPrefix(p, w.p+string(os.PathSeparator)) { + if err := t.singleunwatch(p, both); err != nil && err != errNotWatched && + !os.IsNotExist(err) { + dbgprintf("trg: failed stop watching moved file (%q): %q\n", + p, err) + } + if (w.eDir|w.eNonDir)&(not2nat[Rename]|Rename) != 0 { + evn = append(evn, event{ + p, (w.eDir | w.eNonDir) & e &^ Write &^ not2nat[Write], + w.fi.IsDir(), nil, + }) + } + } + } + } + t.t.Del(w) + return + } + if (ge & not2nat[Write]) != 0 { + switch err := t.walk(w.p, func(fi os.FileInfo) error { + p := filepath.Join(w.p, fi.Name()) + switch err := t.singlewatch(p, w.eDir, ndir, fi); { + case os.IsNotExist(err) && ((w.eDir & Remove) != 0): + evn = append(evn, event{p, Remove, fi.IsDir(), n}) + case err == errAlreadyWatched: + case err != nil: + dbgprintf("trg: watching %q failed: %q", p, err) + case (w.eDir & Create) != 0: + evn = append(evn, event{p, Create, fi.IsDir(), n}) + default: + } + return nil + }); { + case os.IsNotExist(err): + return + case err != nil: + dbgprintf("trg: dir processing failed: %q", err) + default: + } + } + return +} + +type mode uint + +const ( + dir mode = iota + ndir + both +) + +// unwatch stops watching p file/directory. +func (t *trg) singleunwatch(p string, direct mode) error { + w, ok := t.pthLkp[p] + if !ok { + return errNotWatched + } + switch direct { + case dir: + w.eDir = 0 + case ndir: + w.eNonDir = 0 + case both: + w.eDir, w.eNonDir = 0, 0 + } + if err := t.t.Unwatch(w); err != nil { + return err + } + if w.eNonDir|w.eDir != 0 { + mod := dir + if w.eNonDir != 0 { + mod = ndir + } + if err := t.singlewatch(p, w.eNonDir|w.eDir, mod, + w.fi); err != nil && err != errAlreadyWatched { + return err + } + } else { + t.t.Del(w) + } + return nil +} + +func (t *trg) monitor() { + var ( + n interface{} + err error + ) + for { + switch n, err = t.t.Wait(); { + case err == syscall.EINTR: + case t.t.IsStop(n, err): + t.s <- struct{}{} + return + case err != nil: + dbgprintf("trg: failed to read events: %q\n", err) + default: + t.send(t.process(n)) + } + } +} + +// process event returned by native call. +func (t *trg) process(n interface{}) (evn []event) { + t.Lock() + w, ge, err := t.t.Watched(n) + if err != nil { + t.Unlock() + dbgprintf("trg: %v event lookup failed: %q", Event(ge), err) + return + } + + e := decode(ge, w.eDir|w.eNonDir) + if ge&int64(not2nat[Remove]|not2nat[Rename]) == 0 { + switch fi, err := os.Stat(w.p); { + case err != nil: + default: + if err = t.t.Watch(fi, w, encode(w.eDir|w.eNonDir, fi.IsDir())); err != nil { + dbgprintf("trg: %q is no longer watched: %q", w.p, err) + t.t.Del(w) + } + } + } + if e == Event(0) && (!w.fi.IsDir() || (ge&int64(not2nat[Write])) == 0) { + t.Unlock() + return + } + + if w.fi.IsDir() { + evn = append(evn, t.dir(w, n, e, Event(ge))...) + } else { + evn = append(evn, t.file(w, n, e)...) + } + if Event(ge)&(not2nat[Remove]|not2nat[Rename]) != 0 { + t.t.Del(w) + } + t.Unlock() + return +} diff --git a/vendor/github.com/zillode/notify/watchpoint.go b/vendor/github.com/zillode/notify/watchpoint.go new file mode 100644 index 000000000..5afc914f4 --- /dev/null +++ b/vendor/github.com/zillode/notify/watchpoint.go @@ -0,0 +1,103 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +package notify + +// EventDiff describes a change to an event set - EventDiff[0] is an old state, +// while EventDiff[1] is a new state. If event set has not changed (old == new), +// functions typically return the None value. +type eventDiff [2]Event + +func (diff eventDiff) Event() Event { + return diff[1] &^ diff[0] +} + +// Watchpoint +// +// The nil key holds total event set - logical sum for all registered events. +// It speeds up computing EventDiff for Add method. +// +// The rec key holds an event set for a watchpoints created by RecursiveWatch +// for a Watcher implementation which is not natively recursive. +type watchpoint map[chan<- EventInfo]Event + +// None is an empty event diff, think null object. +var none eventDiff + +// rec is just a placeholder +var rec = func() (ch chan<- EventInfo) { + ch = make(chan<- EventInfo) + close(ch) + return +}() + +func (wp watchpoint) dryAdd(ch chan<- EventInfo, e Event) eventDiff { + if e &^= internal; wp[ch]&e == e { + return none + } + total := wp[ch] &^ internal + return eventDiff{total, total | e} +} + +// Add assumes neither c nor e are nil or zero values. +func (wp watchpoint) Add(c chan<- EventInfo, e Event) (diff eventDiff) { + wp[c] |= e + diff[0] = wp[nil] + diff[1] = diff[0] | e + wp[nil] = diff[1] &^ omit + // Strip diff from internal events. + diff[0] &^= internal + diff[1] &^= internal + if diff[0] == diff[1] { + return none + } + return +} + +func (wp watchpoint) Del(c chan<- EventInfo, e Event) (diff eventDiff) { + wp[c] &^= e + if wp[c] == 0 { + delete(wp, c) + } + diff[0] = wp[nil] + delete(wp, nil) + if len(wp) != 0 { + // Recalculate total event set. + for _, e := range wp { + diff[1] |= e + } + wp[nil] = diff[1] &^ omit + } + // Strip diff from internal events. + diff[0] &^= internal + diff[1] &^= internal + if diff[0] == diff[1] { + return none + } + return +} + +func (wp watchpoint) Dispatch(ei EventInfo, extra Event) { + e := eventmask(ei, extra) + if !matches(wp[nil], e) { + return + } + for ch, eset := range wp { + if ch != nil && matches(eset, e) { + select { + case ch <- ei: + default: // Drop event if receiver is too slow + dbgprintf("dropped %s on %q: receiver too slow", ei.Event(), ei.Path()) + } + } + } +} + +func (wp watchpoint) Total() Event { + return wp[nil] &^ internal +} + +func (wp watchpoint) IsRecursive() bool { + return wp[nil]&recursive != 0 +} diff --git a/vendor/github.com/zillode/notify/watchpoint_other.go b/vendor/github.com/zillode/notify/watchpoint_other.go new file mode 100644 index 000000000..9bb381db7 --- /dev/null +++ b/vendor/github.com/zillode/notify/watchpoint_other.go @@ -0,0 +1,23 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build !windows + +package notify + +// eventmask uses ei to create a new event which contains internal flags used by +// notify package logic. +func eventmask(ei EventInfo, extra Event) Event { + return ei.Event() | extra +} + +// matches reports a match only when: +// +// - for user events, when event is present in the given set +// - for internal events, when additionally both event and set have omit bit set +// +// Internal events must not be sent to user channels and vice versa. +func matches(set, event Event) bool { + return (set&omit)^(event&omit) == 0 && set&event == event +} diff --git a/vendor/github.com/zillode/notify/watchpoint_readdcw.go b/vendor/github.com/zillode/notify/watchpoint_readdcw.go new file mode 100644 index 000000000..9fd1e1df3 --- /dev/null +++ b/vendor/github.com/zillode/notify/watchpoint_readdcw.go @@ -0,0 +1,38 @@ +// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build windows + +package notify + +// eventmask uses ei to create a new event which contains internal flags used by +// notify package logic. If one of FileAction* masks is detected, this function +// adds corresponding FileNotifyChange* values. This allows non registered +// FileAction* events to be passed on. +func eventmask(ei EventInfo, extra Event) (e Event) { + if e = ei.Event() | extra; e&fileActionAll != 0 { + if ev, ok := ei.(*event); ok { + switch ev.ftype { + case fTypeFile: + e |= FileNotifyChangeFileName + case fTypeDirectory: + e |= FileNotifyChangeDirName + case fTypeUnknown: + e |= fileNotifyChangeModified + } + return e &^ fileActionAll + } + } + return +} + +// matches reports a match only when: +// +// - for user events, when event is present in the given set +// - for internal events, when additionally both event and set have omit bit set +// +// Internal events must not be sent to user channels and vice versa. +func matches(set, event Event) bool { + return (set&omit)^(event&omit) == 0 && (set&event == event || set&fileNotifyChangeModified&event != 0) +} diff --git a/vendor/manifest b/vendor/manifest index 33e66ba0e..3961946b6 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -446,6 +446,14 @@ "branch": "master", "notests": true }, + { + "importpath": "github.com/zillode/notify", + "repository": "https://github.com/zillode/notify", + "vcs": "git", + "revision": "54e3093eb7377fd139c4605f475cc78e83610b9d", + "branch": "master", + "notests": true + }, { "importpath": "golang.org/x/crypto/bcrypt", "repository": "https://go.googlesource.com/crypto",