2016-04-26 14:01:46 +00:00
|
|
|
// Copyright (C) 2014 The Syncthing Authors.
|
|
|
|
//
|
|
|
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
|
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
2017-02-09 06:52:18 +00:00
|
|
|
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
2016-04-26 14:01:46 +00:00
|
|
|
|
|
|
|
package model
|
|
|
|
|
2017-04-26 00:15:23 +00:00
|
|
|
import (
|
|
|
|
"context"
|
2018-02-04 21:46:24 +00:00
|
|
|
"errors"
|
2018-05-23 07:23:21 +00:00
|
|
|
"fmt"
|
2018-05-21 06:45:05 +00:00
|
|
|
"math/rand"
|
2018-11-07 10:04:41 +00:00
|
|
|
"path/filepath"
|
|
|
|
"sort"
|
2018-10-05 08:26:25 +00:00
|
|
|
"sync/atomic"
|
2017-04-26 00:15:23 +00:00
|
|
|
"time"
|
2017-08-25 19:47:01 +00:00
|
|
|
|
|
|
|
"github.com/syncthing/syncthing/lib/config"
|
2018-05-21 06:56:24 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/db"
|
2018-06-11 13:47:54 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/events"
|
2018-11-07 10:04:41 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/fs"
|
2018-02-04 21:46:24 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/ignore"
|
2018-11-07 10:04:41 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/osutil"
|
2018-02-25 08:39:00 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/protocol"
|
2018-11-07 10:04:41 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/scanner"
|
2018-02-04 21:46:24 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/sync"
|
2017-10-20 14:52:55 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/watchaggregator"
|
2017-04-26 00:15:23 +00:00
|
|
|
)
|
2016-04-26 14:01:46 +00:00
|
|
|
|
2018-12-05 07:40:05 +00:00
|
|
|
// scanLimiter limits the number of concurrent scans. A limit of zero means no limit.
|
|
|
|
var scanLimiter = newByteSemaphore(0)
|
|
|
|
|
2018-05-21 06:45:05 +00:00
|
|
|
var errWatchNotStarted = errors.New("not started")
|
2018-02-04 21:46:24 +00:00
|
|
|
|
2016-04-26 14:01:46 +00:00
|
|
|
type folder struct {
|
|
|
|
stateTracker
|
2017-08-25 19:47:01 +00:00
|
|
|
config.FolderConfiguration
|
2018-07-12 08:15:57 +00:00
|
|
|
localFlags uint32
|
2017-04-01 09:58:06 +00:00
|
|
|
|
2018-02-25 08:39:00 +00:00
|
|
|
model *Model
|
|
|
|
shortID protocol.ShortID
|
|
|
|
ctx context.Context
|
|
|
|
cancel context.CancelFunc
|
|
|
|
|
2018-05-21 06:45:05 +00:00
|
|
|
scanInterval time.Duration
|
|
|
|
scanTimer *time.Timer
|
|
|
|
scanNow chan rescanRequest
|
|
|
|
scanDelay chan time.Duration
|
2017-04-20 00:20:34 +00:00
|
|
|
initialScanFinished chan struct{}
|
2018-06-10 11:24:59 +00:00
|
|
|
stopped chan struct{}
|
2018-11-07 10:04:41 +00:00
|
|
|
scanErrors []FileError
|
|
|
|
scanErrorsMut sync.Mutex
|
2018-02-25 08:39:00 +00:00
|
|
|
|
|
|
|
pullScheduled chan struct{}
|
|
|
|
|
|
|
|
watchCancel context.CancelFunc
|
|
|
|
watchChan chan []string
|
|
|
|
restartWatchChan chan struct{}
|
|
|
|
watchErr error
|
2018-12-21 11:06:21 +00:00
|
|
|
watchMut sync.Mutex
|
2018-05-11 08:45:13 +00:00
|
|
|
|
|
|
|
puller puller
|
|
|
|
}
|
|
|
|
|
2018-05-21 06:45:05 +00:00
|
|
|
type rescanRequest struct {
|
|
|
|
subdirs []string
|
|
|
|
err chan error
|
|
|
|
}
|
|
|
|
|
2018-05-11 08:45:13 +00:00
|
|
|
type puller interface {
|
|
|
|
pull() bool // true when successfull and should not be retried
|
2016-04-26 14:01:46 +00:00
|
|
|
}
|
|
|
|
|
2017-08-25 19:47:01 +00:00
|
|
|
func newFolder(model *Model, cfg config.FolderConfiguration) folder {
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
|
|
|
|
return folder{
|
|
|
|
stateTracker: newStateTracker(cfg.ID),
|
|
|
|
FolderConfiguration: cfg,
|
|
|
|
|
2018-02-25 08:39:00 +00:00
|
|
|
model: model,
|
|
|
|
shortID: model.shortID,
|
|
|
|
ctx: ctx,
|
|
|
|
cancel: cancel,
|
|
|
|
|
2018-05-21 06:45:05 +00:00
|
|
|
scanInterval: time.Duration(cfg.RescanIntervalS) * time.Second,
|
|
|
|
scanTimer: time.NewTimer(time.Millisecond), // The first scan should be done immediately.
|
|
|
|
scanNow: make(chan rescanRequest),
|
|
|
|
scanDelay: make(chan time.Duration),
|
2017-08-25 19:47:01 +00:00
|
|
|
initialScanFinished: make(chan struct{}),
|
2018-06-11 13:47:54 +00:00
|
|
|
stopped: make(chan struct{}),
|
2018-11-07 10:04:41 +00:00
|
|
|
scanErrorsMut: sync.NewMutex(),
|
2018-02-25 08:39:00 +00:00
|
|
|
|
|
|
|
pullScheduled: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a pull if we're busy when it comes.
|
|
|
|
|
2018-06-11 13:47:54 +00:00
|
|
|
watchCancel: func() {},
|
|
|
|
restartWatchChan: make(chan struct{}, 1),
|
2018-12-21 11:06:21 +00:00
|
|
|
watchMut: sync.NewMutex(),
|
2017-08-25 19:47:01 +00:00
|
|
|
}
|
2016-04-26 14:01:46 +00:00
|
|
|
}
|
|
|
|
|
2018-05-11 08:45:13 +00:00
|
|
|
func (f *folder) Serve() {
|
2018-10-05 08:26:25 +00:00
|
|
|
atomic.AddInt32(&f.model.foldersRunning, 1)
|
|
|
|
defer atomic.AddInt32(&f.model.foldersRunning, -1)
|
|
|
|
|
2018-05-11 08:45:13 +00:00
|
|
|
l.Debugln(f, "starting")
|
|
|
|
defer l.Debugln(f, "exiting")
|
|
|
|
|
|
|
|
defer func() {
|
2018-05-21 06:45:05 +00:00
|
|
|
f.scanTimer.Stop()
|
2018-05-11 08:45:13 +00:00
|
|
|
f.setState(FolderIdle)
|
2018-06-10 11:24:59 +00:00
|
|
|
close(f.stopped)
|
2018-05-11 08:45:13 +00:00
|
|
|
}()
|
|
|
|
|
|
|
|
pause := f.basePause()
|
|
|
|
pullFailTimer := time.NewTimer(0)
|
|
|
|
<-pullFailTimer.C
|
|
|
|
|
|
|
|
if f.FSWatcherEnabled && f.CheckHealth() == nil {
|
|
|
|
f.startWatch()
|
|
|
|
}
|
|
|
|
|
|
|
|
initialCompleted := f.initialScanFinished
|
|
|
|
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-f.ctx.Done():
|
|
|
|
return
|
|
|
|
|
|
|
|
case <-f.pullScheduled:
|
|
|
|
pullFailTimer.Stop()
|
|
|
|
select {
|
|
|
|
case <-pullFailTimer.C:
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
|
|
|
if !f.puller.pull() {
|
|
|
|
// Pulling failed, try again later.
|
|
|
|
pullFailTimer.Reset(pause)
|
|
|
|
}
|
|
|
|
|
|
|
|
case <-pullFailTimer.C:
|
|
|
|
if f.puller.pull() {
|
|
|
|
// We're good. Don't schedule another fail pull and reset
|
|
|
|
// the pause interval.
|
|
|
|
pause = f.basePause()
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
// Pulling failed, try again later.
|
|
|
|
l.Infof("Folder %v isn't making sync progress - retrying in %v.", f.Description(), pause)
|
|
|
|
pullFailTimer.Reset(pause)
|
|
|
|
// Back off from retrying to pull with an upper limit.
|
|
|
|
if pause < 60*f.basePause() {
|
|
|
|
pause *= 2
|
|
|
|
}
|
|
|
|
|
|
|
|
case <-initialCompleted:
|
|
|
|
// Initial scan has completed, we should do a pull
|
|
|
|
initialCompleted = nil // never hit this case again
|
|
|
|
if !f.puller.pull() {
|
|
|
|
// Pulling failed, try again later.
|
|
|
|
pullFailTimer.Reset(pause)
|
|
|
|
}
|
|
|
|
|
|
|
|
// The reason for running the scanner from within the puller is that
|
|
|
|
// this is the easiest way to make sure we are not doing both at the
|
|
|
|
// same time.
|
2018-05-21 06:45:05 +00:00
|
|
|
case <-f.scanTimer.C:
|
2018-05-11 08:45:13 +00:00
|
|
|
l.Debugln(f, "Scanning subdirectories")
|
|
|
|
f.scanTimerFired()
|
|
|
|
|
2018-05-21 06:45:05 +00:00
|
|
|
case req := <-f.scanNow:
|
2018-05-11 08:45:13 +00:00
|
|
|
req.err <- f.scanSubdirs(req.subdirs)
|
|
|
|
|
2018-05-21 06:45:05 +00:00
|
|
|
case next := <-f.scanDelay:
|
|
|
|
f.scanTimer.Reset(next)
|
2018-05-11 08:45:13 +00:00
|
|
|
|
|
|
|
case fsEvents := <-f.watchChan:
|
|
|
|
l.Debugln(f, "filesystem notification rescan")
|
|
|
|
f.scanSubdirs(fsEvents)
|
|
|
|
|
|
|
|
case <-f.restartWatchChan:
|
|
|
|
f.restartWatch()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-10-24 07:58:55 +00:00
|
|
|
func (f *folder) BringToFront(string) {}
|
|
|
|
|
2018-05-21 06:56:24 +00:00
|
|
|
func (f *folder) Override(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {}
|
|
|
|
|
2018-07-12 08:15:57 +00:00
|
|
|
func (f *folder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {}
|
|
|
|
|
2016-04-26 14:01:46 +00:00
|
|
|
func (f *folder) DelayScan(next time.Duration) {
|
2018-05-21 06:45:05 +00:00
|
|
|
f.Delay(next)
|
2016-04-26 14:01:46 +00:00
|
|
|
}
|
|
|
|
|
2019-01-01 09:17:14 +00:00
|
|
|
func (f *folder) ignoresUpdated() {
|
2017-10-24 07:58:55 +00:00
|
|
|
if f.FSWatcherEnabled {
|
|
|
|
f.scheduleWatchRestart()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-02-25 08:39:00 +00:00
|
|
|
func (f *folder) SchedulePull() {
|
|
|
|
select {
|
|
|
|
case f.pullScheduled <- struct{}{}:
|
|
|
|
default:
|
|
|
|
// We might be busy doing a pull and thus not reading from this
|
|
|
|
// channel. The channel is 1-buffered, so one notification will be
|
|
|
|
// queued to ensure we recheck after the pull, but beyond that we must
|
|
|
|
// make sure to not block index receiving.
|
|
|
|
}
|
|
|
|
}
|
2017-11-07 06:59:35 +00:00
|
|
|
|
2017-10-24 07:58:55 +00:00
|
|
|
func (f *folder) Jobs() ([]string, []string) {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
2016-04-26 14:01:46 +00:00
|
|
|
func (f *folder) Scan(subdirs []string) error {
|
2017-04-20 00:20:34 +00:00
|
|
|
<-f.initialScanFinished
|
2018-05-21 06:45:05 +00:00
|
|
|
req := rescanRequest{
|
|
|
|
subdirs: subdirs,
|
|
|
|
err: make(chan error),
|
|
|
|
}
|
2018-07-04 07:07:33 +00:00
|
|
|
|
|
|
|
select {
|
|
|
|
case f.scanNow <- req:
|
|
|
|
return <-req.err
|
|
|
|
case <-f.ctx.Done():
|
|
|
|
return f.ctx.Err()
|
|
|
|
}
|
2018-05-21 06:45:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (f *folder) Reschedule() {
|
|
|
|
if f.scanInterval == 0 {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
// Sleep a random time between 3/4 and 5/4 of the configured interval.
|
|
|
|
sleepNanos := (f.scanInterval.Nanoseconds()*3 + rand.Int63n(2*f.scanInterval.Nanoseconds())) / 4
|
|
|
|
interval := time.Duration(sleepNanos) * time.Nanosecond
|
|
|
|
l.Debugln(f, "next rescan in", interval)
|
|
|
|
f.scanTimer.Reset(interval)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *folder) Delay(next time.Duration) {
|
|
|
|
f.scanDelay <- next
|
2016-04-26 14:01:46 +00:00
|
|
|
}
|
2017-04-26 00:15:23 +00:00
|
|
|
|
2016-04-26 14:01:46 +00:00
|
|
|
func (f *folder) Stop() {
|
2017-04-26 00:15:23 +00:00
|
|
|
f.cancel()
|
2018-06-10 11:24:59 +00:00
|
|
|
<-f.stopped
|
2016-04-26 14:01:46 +00:00
|
|
|
}
|
|
|
|
|
2017-10-24 07:58:55 +00:00
|
|
|
// CheckHealth checks the folder for common errors, updates the folder state
|
|
|
|
// and returns the current folder error, or nil if the folder is healthy.
|
|
|
|
func (f *folder) CheckHealth() error {
|
|
|
|
err := f.getHealthError()
|
|
|
|
f.setError(err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *folder) getHealthError() error {
|
|
|
|
// Check for folder errors, with the most serious and specific first and
|
|
|
|
// generic ones like out of space on the home disk later.
|
|
|
|
|
|
|
|
if err := f.CheckPath(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := f.model.cfg.CheckHomeFreeSpace(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2016-04-26 14:01:46 +00:00
|
|
|
|
2017-10-12 06:16:46 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-04-20 00:20:34 +00:00
|
|
|
func (f *folder) scanSubdirs(subDirs []string) error {
|
2018-11-07 10:04:41 +00:00
|
|
|
if err := f.CheckHealth(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
f.model.fmut.RLock()
|
|
|
|
fset := f.model.folderFiles[f.ID]
|
|
|
|
ignores := f.model.folderIgnores[f.ID]
|
|
|
|
f.model.fmut.RUnlock()
|
|
|
|
mtimefs := fset.MtimeFS()
|
|
|
|
|
2018-12-05 07:40:05 +00:00
|
|
|
f.setState(FolderScanWaiting)
|
|
|
|
scanLimiter.take(1)
|
|
|
|
defer scanLimiter.give(1)
|
|
|
|
|
2018-11-07 10:04:41 +00:00
|
|
|
for i := range subDirs {
|
|
|
|
sub := osutil.NativeFilename(subDirs[i])
|
|
|
|
|
|
|
|
if sub == "" {
|
|
|
|
// A blank subdirs means to scan the entire folder. We can trim
|
|
|
|
// the subDirs list and go on our way.
|
|
|
|
subDirs = nil
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
subDirs[i] = sub
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check if the ignore patterns changed as part of scanning this folder.
|
|
|
|
// If they did we should schedule a pull of the folder so that we
|
|
|
|
// request things we might have suddenly become unignored and so on.
|
|
|
|
oldHash := ignores.Hash()
|
|
|
|
defer func() {
|
|
|
|
if ignores.Hash() != oldHash {
|
2019-01-01 09:17:14 +00:00
|
|
|
l.Debugln("Folder", f.Description(), "ignore patterns change detected while scanning; triggering puller")
|
|
|
|
f.ignoresUpdated()
|
|
|
|
f.SchedulePull()
|
2018-11-07 10:04:41 +00:00
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
if err := ignores.Load(".stignore"); err != nil && !fs.IsNotExist(err) {
|
|
|
|
err = fmt.Errorf("loading ignores: %v", err)
|
2016-04-26 14:01:46 +00:00
|
|
|
f.setError(err)
|
|
|
|
return err
|
|
|
|
}
|
2018-11-07 10:04:41 +00:00
|
|
|
|
|
|
|
// Clean the list of subitems to ensure that we start at a known
|
|
|
|
// directory, and don't scan subdirectories of things we've already
|
|
|
|
// scanned.
|
|
|
|
subDirs = unifySubs(subDirs, func(f string) bool {
|
|
|
|
_, ok := fset.Get(protocol.LocalDeviceID, f)
|
|
|
|
return ok
|
|
|
|
})
|
|
|
|
|
|
|
|
f.setState(FolderScanning)
|
|
|
|
|
|
|
|
fchan := scanner.Walk(f.ctx, scanner.Config{
|
|
|
|
Folder: f.ID,
|
|
|
|
Subs: subDirs,
|
|
|
|
Matcher: ignores,
|
|
|
|
TempLifetime: time.Duration(f.model.cfg.Options().KeepTemporariesH) * time.Hour,
|
|
|
|
CurrentFiler: cFiler{f.model, f.ID},
|
|
|
|
Filesystem: mtimefs,
|
|
|
|
IgnorePerms: f.IgnorePerms,
|
|
|
|
AutoNormalize: f.AutoNormalize,
|
|
|
|
Hashers: f.model.numHashers(f.ID),
|
|
|
|
ShortID: f.model.shortID,
|
|
|
|
ProgressTickIntervalS: f.ScanProgressIntervalS,
|
|
|
|
UseLargeBlocks: f.UseLargeBlocks,
|
|
|
|
LocalFlags: f.localFlags,
|
|
|
|
})
|
|
|
|
|
|
|
|
batchFn := func(fs []protocol.FileInfo) error {
|
|
|
|
if err := f.CheckHealth(); err != nil {
|
|
|
|
l.Debugf("Stopping scan of folder %s due to: %s", f.Description(), err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
f.model.updateLocalsFromScanning(f.ID, fs)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
// Resolve items which are identical with the global state.
|
|
|
|
if f.localFlags&protocol.FlagLocalReceiveOnly != 0 {
|
|
|
|
oldBatchFn := batchFn // can't reference batchFn directly (recursion)
|
|
|
|
batchFn = func(fs []protocol.FileInfo) error {
|
|
|
|
for i := range fs {
|
|
|
|
switch gf, ok := fset.GetGlobal(fs[i].Name); {
|
|
|
|
case !ok:
|
|
|
|
continue
|
|
|
|
case gf.IsEquivalentOptional(fs[i], false, false, protocol.FlagLocalReceiveOnly):
|
|
|
|
// What we have locally is equivalent to the global file.
|
|
|
|
fs[i].Version = fs[i].Version.Merge(gf.Version)
|
|
|
|
fallthrough
|
|
|
|
case fs[i].IsDeleted() && gf.IsReceiveOnlyChanged():
|
|
|
|
// Our item is deleted and the global item is our own
|
|
|
|
// receive only file. We can't delete file infos, so
|
|
|
|
// we just pretend it is a normal deleted file (nobody
|
|
|
|
// cares about that).
|
|
|
|
fs[i].LocalFlags &^= protocol.FlagLocalReceiveOnly
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return oldBatchFn(fs)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
batch := newFileInfoBatch(batchFn)
|
|
|
|
|
|
|
|
// Schedule a pull after scanning, but only if we actually detected any
|
|
|
|
// changes.
|
|
|
|
changes := 0
|
|
|
|
defer func() {
|
|
|
|
if changes > 0 {
|
|
|
|
f.SchedulePull()
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
f.clearScanErrors(subDirs)
|
|
|
|
for res := range fchan {
|
|
|
|
if res.Err != nil {
|
|
|
|
f.newScanError(res.Path, res.Err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if err := batch.flushIfFull(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
batch.append(res.File)
|
|
|
|
changes++
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := batch.flush(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(subDirs) == 0 {
|
|
|
|
// If we have no specific subdirectories to traverse, set it to one
|
|
|
|
// empty prefix so we traverse the entire folder contents once.
|
|
|
|
subDirs = []string{""}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Do a scan of the database for each prefix, to check for deleted and
|
|
|
|
// ignored files.
|
|
|
|
var toIgnore []db.FileInfoTruncated
|
|
|
|
ignoredParent := ""
|
|
|
|
for _, sub := range subDirs {
|
|
|
|
var iterError error
|
|
|
|
|
|
|
|
fset.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool {
|
|
|
|
file := fi.(db.FileInfoTruncated)
|
|
|
|
|
|
|
|
if err := batch.flushIfFull(); err != nil {
|
|
|
|
iterError = err
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2018-11-22 10:16:45 +00:00
|
|
|
if ignoredParent != "" && !fs.IsParent(file.Name, ignoredParent) {
|
2018-11-07 10:04:41 +00:00
|
|
|
for _, file := range toIgnore {
|
|
|
|
l.Debugln("marking file as ignored", file)
|
|
|
|
nf := file.ConvertToIgnoredFileInfo(f.model.id.Short())
|
|
|
|
batch.append(nf)
|
|
|
|
changes++
|
|
|
|
if err := batch.flushIfFull(); err != nil {
|
|
|
|
iterError = err
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
toIgnore = toIgnore[:0]
|
|
|
|
ignoredParent = ""
|
|
|
|
}
|
|
|
|
|
|
|
|
switch ignored := ignores.Match(file.Name).IsIgnored(); {
|
|
|
|
case !file.IsIgnored() && ignored:
|
|
|
|
// File was not ignored at last pass but has been ignored.
|
|
|
|
if file.IsDirectory() {
|
|
|
|
// Delay ignoring as a child might be unignored.
|
|
|
|
toIgnore = append(toIgnore, file)
|
|
|
|
if ignoredParent == "" {
|
|
|
|
// If the parent wasn't ignored already, set
|
|
|
|
// this path as the "highest" ignored parent
|
|
|
|
ignoredParent = file.Name
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
l.Debugln("marking file as ignored", f)
|
|
|
|
nf := file.ConvertToIgnoredFileInfo(f.model.id.Short())
|
|
|
|
batch.append(nf)
|
|
|
|
changes++
|
|
|
|
|
|
|
|
case file.IsIgnored() && !ignored:
|
|
|
|
// Successfully scanned items are already un-ignored during
|
|
|
|
// the scan, so check whether it is deleted.
|
|
|
|
fallthrough
|
|
|
|
case !file.IsIgnored() && !file.IsDeleted() && !file.IsUnsupported():
|
|
|
|
// The file is not ignored, deleted or unsupported. Lets check if
|
|
|
|
// it's still here. Simply stat:ing it wont do as there are
|
|
|
|
// tons of corner cases (e.g. parent dir->symlink, missing
|
|
|
|
// permissions)
|
|
|
|
if !osutil.IsDeleted(mtimefs, file.Name) {
|
|
|
|
if ignoredParent != "" {
|
|
|
|
// Don't ignore parents of this not ignored item
|
|
|
|
toIgnore = toIgnore[:0]
|
|
|
|
ignoredParent = ""
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
nf := protocol.FileInfo{
|
|
|
|
Name: file.Name,
|
|
|
|
Type: file.Type,
|
|
|
|
Size: 0,
|
|
|
|
ModifiedS: file.ModifiedS,
|
|
|
|
ModifiedNs: file.ModifiedNs,
|
|
|
|
ModifiedBy: f.model.id.Short(),
|
|
|
|
Deleted: true,
|
|
|
|
Version: file.Version.Update(f.model.shortID),
|
|
|
|
LocalFlags: f.localFlags,
|
|
|
|
}
|
|
|
|
// We do not want to override the global version
|
|
|
|
// with the deleted file. Keeping only our local
|
|
|
|
// counter makes sure we are in conflict with any
|
|
|
|
// other existing versions, which will be resolved
|
|
|
|
// by the normal pulling mechanisms.
|
|
|
|
if file.ShouldConflict() {
|
|
|
|
nf.Version = nf.Version.DropOthers(f.model.shortID)
|
|
|
|
}
|
|
|
|
|
|
|
|
batch.append(nf)
|
|
|
|
changes++
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
})
|
|
|
|
|
|
|
|
if iterError == nil && len(toIgnore) > 0 {
|
|
|
|
for _, file := range toIgnore {
|
|
|
|
l.Debugln("marking file as ignored", f)
|
|
|
|
nf := file.ConvertToIgnoredFileInfo(f.model.id.Short())
|
|
|
|
batch.append(nf)
|
|
|
|
changes++
|
|
|
|
if iterError = batch.flushIfFull(); iterError != nil {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
toIgnore = toIgnore[:0]
|
|
|
|
}
|
|
|
|
|
|
|
|
if iterError != nil {
|
|
|
|
return iterError
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := batch.flush(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
f.model.folderStatRef(f.ID).ScanCompleted()
|
|
|
|
f.setState(FolderIdle)
|
2016-04-26 14:01:46 +00:00
|
|
|
return nil
|
|
|
|
}
|
2017-09-07 06:17:47 +00:00
|
|
|
|
|
|
|
func (f *folder) scanTimerFired() {
|
|
|
|
err := f.scanSubdirs(nil)
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-f.initialScanFinished:
|
|
|
|
default:
|
|
|
|
status := "Completed"
|
|
|
|
if err != nil {
|
|
|
|
status = "Failed"
|
|
|
|
}
|
|
|
|
l.Infoln(status, "initial scan of", f.Type.String(), "folder", f.Description())
|
|
|
|
close(f.initialScanFinished)
|
|
|
|
}
|
|
|
|
|
2018-05-21 06:45:05 +00:00
|
|
|
f.Reschedule()
|
2017-09-07 06:17:47 +00:00
|
|
|
}
|
2017-10-20 14:52:55 +00:00
|
|
|
|
2018-02-04 21:46:24 +00:00
|
|
|
func (f *folder) WatchError() error {
|
2018-12-21 11:06:21 +00:00
|
|
|
f.watchMut.Lock()
|
|
|
|
defer f.watchMut.Unlock()
|
2018-02-04 21:46:24 +00:00
|
|
|
return f.watchErr
|
2017-10-20 14:52:55 +00:00
|
|
|
}
|
|
|
|
|
2018-02-04 21:46:24 +00:00
|
|
|
// stopWatch immediately aborts watching and may be called asynchronously
|
|
|
|
func (f *folder) stopWatch() {
|
2018-12-21 11:06:21 +00:00
|
|
|
f.watchMut.Lock()
|
2017-10-20 14:52:55 +00:00
|
|
|
f.watchCancel()
|
2018-06-11 13:47:54 +00:00
|
|
|
prevErr := f.watchErr
|
2018-02-04 21:46:24 +00:00
|
|
|
f.watchErr = errWatchNotStarted
|
2018-12-21 11:06:21 +00:00
|
|
|
f.watchMut.Unlock()
|
2018-06-11 13:47:54 +00:00
|
|
|
if prevErr != errWatchNotStarted {
|
|
|
|
data := map[string]interface{}{
|
|
|
|
"folder": f.ID,
|
|
|
|
"to": errWatchNotStarted.Error(),
|
|
|
|
}
|
|
|
|
if prevErr != nil {
|
|
|
|
data["from"] = prevErr.Error()
|
|
|
|
}
|
|
|
|
events.Default.Log(events.FolderWatchStateChanged, data)
|
|
|
|
}
|
2017-10-20 14:52:55 +00:00
|
|
|
}
|
|
|
|
|
2018-02-04 21:46:24 +00:00
|
|
|
// scheduleWatchRestart makes sure watching is restarted from the main for loop
|
|
|
|
// in a folder's Serve and thus may be called asynchronously (e.g. when ignores change).
|
2017-10-24 07:58:55 +00:00
|
|
|
func (f *folder) scheduleWatchRestart() {
|
2017-10-20 14:52:55 +00:00
|
|
|
select {
|
2017-10-24 07:58:55 +00:00
|
|
|
case f.restartWatchChan <- struct{}{}:
|
2017-10-20 14:52:55 +00:00
|
|
|
default:
|
|
|
|
// We might be busy doing a pull and thus not reading from this
|
|
|
|
// channel. The channel is 1-buffered, so one notification will be
|
|
|
|
// queued to ensure we recheck after the pull.
|
|
|
|
}
|
|
|
|
}
|
2017-10-24 07:58:55 +00:00
|
|
|
|
2018-02-04 21:46:24 +00:00
|
|
|
// restartWatch should only ever be called synchronously. If you want to use
|
|
|
|
// this asynchronously, you should probably use scheduleWatchRestart instead.
|
|
|
|
func (f *folder) restartWatch() {
|
|
|
|
f.stopWatch()
|
|
|
|
f.startWatch()
|
2018-06-11 13:47:54 +00:00
|
|
|
f.scanSubdirs(nil)
|
2018-02-04 21:46:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// startWatch should only ever be called synchronously. If you want to use
|
|
|
|
// this asynchronously, you should probably use scheduleWatchRestart instead.
|
|
|
|
func (f *folder) startWatch() {
|
|
|
|
ctx, cancel := context.WithCancel(f.ctx)
|
|
|
|
f.model.fmut.RLock()
|
|
|
|
ignores := f.model.folderIgnores[f.folderID]
|
|
|
|
f.model.fmut.RUnlock()
|
2018-12-21 11:06:21 +00:00
|
|
|
f.watchMut.Lock()
|
2018-02-04 21:46:24 +00:00
|
|
|
f.watchChan = make(chan []string)
|
|
|
|
f.watchCancel = cancel
|
2018-12-21 11:06:21 +00:00
|
|
|
f.watchMut.Unlock()
|
2018-02-04 21:46:24 +00:00
|
|
|
go f.startWatchAsync(ctx, ignores)
|
|
|
|
}
|
|
|
|
|
|
|
|
// startWatchAsync tries to start the filesystem watching and retries every minute on failure.
|
|
|
|
// It is a convenience function that should not be used except in startWatch.
|
|
|
|
func (f *folder) startWatchAsync(ctx context.Context, ignores *ignore.Matcher) {
|
|
|
|
timer := time.NewTimer(0)
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-timer.C:
|
|
|
|
eventChan, err := f.Filesystem().Watch(".", ignores, ctx, f.IgnorePerms)
|
2018-12-21 11:06:21 +00:00
|
|
|
f.watchMut.Lock()
|
2018-02-04 21:46:24 +00:00
|
|
|
prevErr := f.watchErr
|
|
|
|
f.watchErr = err
|
2018-12-21 11:06:21 +00:00
|
|
|
f.watchMut.Unlock()
|
2018-06-11 13:47:54 +00:00
|
|
|
if err != prevErr {
|
|
|
|
data := map[string]interface{}{
|
|
|
|
"folder": f.ID,
|
|
|
|
}
|
|
|
|
if prevErr != nil {
|
|
|
|
data["from"] = prevErr.Error()
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
data["to"] = err.Error()
|
|
|
|
}
|
|
|
|
events.Default.Log(events.FolderWatchStateChanged, data)
|
|
|
|
}
|
2018-02-04 21:46:24 +00:00
|
|
|
if err != nil {
|
|
|
|
if prevErr == errWatchNotStarted {
|
2018-06-11 13:47:54 +00:00
|
|
|
l.Infof("Error while trying to start filesystem watcher for folder %s, trying again in 1min: %v", f.Description(), err)
|
2018-02-04 21:46:24 +00:00
|
|
|
} else {
|
2018-06-11 13:47:54 +00:00
|
|
|
l.Debugf("Repeat error while trying to start filesystem watcher for folder %s, trying again in 1min: %v", f.Description(), err)
|
2018-02-04 21:46:24 +00:00
|
|
|
}
|
|
|
|
timer.Reset(time.Minute)
|
|
|
|
continue
|
|
|
|
}
|
2018-12-21 11:06:21 +00:00
|
|
|
f.watchMut.Lock()
|
|
|
|
defer f.watchMut.Unlock()
|
2018-02-04 21:46:24 +00:00
|
|
|
watchaggregator.Aggregate(eventChan, f.watchChan, f.FolderConfiguration, f.model.cfg, ctx)
|
|
|
|
l.Debugln("Started filesystem watcher for folder", f.Description())
|
|
|
|
return
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-10-24 07:58:55 +00:00
|
|
|
func (f *folder) setError(err error) {
|
2018-12-30 20:56:16 +00:00
|
|
|
select {
|
|
|
|
case <-f.ctx.Done():
|
|
|
|
return
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
2017-10-24 07:58:55 +00:00
|
|
|
_, _, oldErr := f.getState()
|
|
|
|
if (err != nil && oldErr != nil && oldErr.Error() == err.Error()) || (err == nil && oldErr == nil) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
if oldErr == nil {
|
|
|
|
l.Warnf("Error on folder %s: %v", f.Description(), err)
|
|
|
|
} else {
|
|
|
|
l.Infof("Error on folder %s changed: %q -> %q", f.Description(), oldErr, err)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
l.Infoln("Cleared error on folder", f.Description())
|
|
|
|
}
|
|
|
|
|
|
|
|
if f.FSWatcherEnabled {
|
|
|
|
if err != nil {
|
2018-02-04 21:46:24 +00:00
|
|
|
f.stopWatch()
|
2017-10-24 07:58:55 +00:00
|
|
|
} else {
|
|
|
|
f.scheduleWatchRestart()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
f.stateTracker.setError(err)
|
|
|
|
}
|
2018-05-11 08:45:13 +00:00
|
|
|
|
|
|
|
func (f *folder) basePause() time.Duration {
|
|
|
|
if f.PullerPauseS == 0 {
|
|
|
|
return defaultPullerPause
|
|
|
|
}
|
|
|
|
return time.Duration(f.PullerPauseS) * time.Second
|
|
|
|
}
|
2018-05-23 07:23:21 +00:00
|
|
|
|
|
|
|
func (f *folder) String() string {
|
|
|
|
return fmt.Sprintf("%s/%s@%p", f.Type, f.folderID, f)
|
|
|
|
}
|
2018-11-07 10:04:41 +00:00
|
|
|
|
|
|
|
func (f *folder) newScanError(path string, err error) {
|
|
|
|
f.scanErrorsMut.Lock()
|
|
|
|
f.scanErrors = append(f.scanErrors, FileError{
|
|
|
|
Err: err.Error(),
|
|
|
|
Path: path,
|
|
|
|
})
|
|
|
|
f.scanErrorsMut.Unlock()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *folder) clearScanErrors(subDirs []string) {
|
|
|
|
f.scanErrorsMut.Lock()
|
|
|
|
defer f.scanErrorsMut.Unlock()
|
|
|
|
if len(subDirs) == 0 {
|
|
|
|
f.scanErrors = nil
|
|
|
|
return
|
|
|
|
}
|
|
|
|
filtered := f.scanErrors[:0]
|
|
|
|
outer:
|
|
|
|
for _, fe := range f.scanErrors {
|
|
|
|
for _, sub := range subDirs {
|
2018-11-22 10:16:45 +00:00
|
|
|
if fe.Path == sub || fs.IsParent(fe.Path, sub) {
|
2018-11-07 10:04:41 +00:00
|
|
|
continue outer
|
|
|
|
}
|
|
|
|
}
|
|
|
|
filtered = append(filtered, fe)
|
|
|
|
}
|
|
|
|
f.scanErrors = filtered
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *folder) Errors() []FileError {
|
|
|
|
f.scanErrorsMut.Lock()
|
|
|
|
defer f.scanErrorsMut.Unlock()
|
|
|
|
return append([]FileError{}, f.scanErrors...)
|
|
|
|
}
|
|
|
|
|
|
|
|
// The exists function is expected to return true for all known paths
|
|
|
|
// (excluding "" and ".")
|
|
|
|
func unifySubs(dirs []string, exists func(dir string) bool) []string {
|
|
|
|
if len(dirs) == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
sort.Strings(dirs)
|
|
|
|
if dirs[0] == "" || dirs[0] == "." || dirs[0] == string(fs.PathSeparator) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
prev := "./" // Anything that can't be parent of a clean path
|
|
|
|
for i := 0; i < len(dirs); {
|
|
|
|
dir, err := fs.Canonicalize(dirs[i])
|
|
|
|
if err != nil {
|
|
|
|
l.Debugf("Skipping %v for scan: %s", dirs[i], err)
|
|
|
|
dirs = append(dirs[:i], dirs[i+1:]...)
|
|
|
|
continue
|
|
|
|
}
|
2018-11-22 10:16:45 +00:00
|
|
|
if dir == prev || fs.IsParent(dir, prev) {
|
2018-11-07 10:04:41 +00:00
|
|
|
dirs = append(dirs[:i], dirs[i+1:]...)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
parent := filepath.Dir(dir)
|
|
|
|
for parent != "." && parent != string(fs.PathSeparator) && !exists(parent) {
|
|
|
|
dir = parent
|
|
|
|
parent = filepath.Dir(dir)
|
|
|
|
}
|
|
|
|
dirs[i] = dir
|
|
|
|
prev = dir
|
|
|
|
i++
|
|
|
|
}
|
|
|
|
return dirs
|
|
|
|
}
|