2015-09-27 08:50:54 +00:00
|
|
|
// Copyright (C) 2014 The Syncthing Authors.
|
2014-09-29 19:43:32 +00:00
|
|
|
//
|
2015-03-07 20:36:35 +00:00
|
|
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
|
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
2017-02-09 06:52:18 +00:00
|
|
|
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
2014-06-01 20:50:14 +00:00
|
|
|
|
2014-05-15 03:26:55 +00:00
|
|
|
package model
|
2014-03-28 13:36:57 +00:00
|
|
|
|
|
|
|
import (
|
2018-01-14 14:30:11 +00:00
|
|
|
"bytes"
|
2014-08-25 15:45:13 +00:00
|
|
|
"fmt"
|
2020-04-26 22:13:18 +00:00
|
|
|
"io"
|
2014-03-28 13:36:57 +00:00
|
|
|
"path/filepath"
|
2016-08-05 07:13:52 +00:00
|
|
|
"runtime"
|
2015-06-26 11:31:30 +00:00
|
|
|
"sort"
|
2016-01-03 20:15:02 +00:00
|
|
|
"strings"
|
2014-03-28 13:36:57 +00:00
|
|
|
"time"
|
2014-06-19 22:27:54 +00:00
|
|
|
|
2019-01-25 08:52:21 +00:00
|
|
|
"github.com/pkg/errors"
|
|
|
|
|
2015-08-06 09:29:25 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/config"
|
|
|
|
"github.com/syncthing/syncthing/lib/db"
|
|
|
|
"github.com/syncthing/syncthing/lib/events"
|
2016-08-05 17:45:45 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/fs"
|
2015-08-06 09:29:25 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/ignore"
|
|
|
|
"github.com/syncthing/syncthing/lib/osutil"
|
2015-09-22 17:38:46 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/protocol"
|
2015-08-06 09:29:25 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/scanner"
|
2018-01-14 14:30:11 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/sha256"
|
2015-08-06 09:29:25 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/sync"
|
|
|
|
"github.com/syncthing/syncthing/lib/versioner"
|
2016-12-14 23:30:29 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/weakhash"
|
2014-03-28 13:36:57 +00:00
|
|
|
)
|
|
|
|
|
2017-11-09 21:16:29 +00:00
|
|
|
var (
|
|
|
|
blockStats = make(map[string]int)
|
|
|
|
blockStatsMut = sync.NewMutex()
|
|
|
|
)
|
|
|
|
|
2016-05-04 10:47:33 +00:00
|
|
|
func init() {
|
2016-12-16 22:23:35 +00:00
|
|
|
folderFactories[config.FolderTypeSendReceive] = newSendReceiveFolder
|
2016-05-04 10:47:33 +00:00
|
|
|
}
|
|
|
|
|
2014-09-27 12:44:15 +00:00
|
|
|
// A pullBlockState is passed to the puller routine for each block that needs
|
|
|
|
// to be fetched.
|
|
|
|
type pullBlockState struct {
|
|
|
|
*sharedPullerState
|
|
|
|
block protocol.BlockInfo
|
2014-03-28 13:36:57 +00:00
|
|
|
}
|
|
|
|
|
2014-09-27 12:44:15 +00:00
|
|
|
// A copyBlocksState is passed to copy routine if the file has blocks to be
|
2014-10-08 22:41:23 +00:00
|
|
|
// copied.
|
2014-09-27 12:44:15 +00:00
|
|
|
type copyBlocksState struct {
|
|
|
|
*sharedPullerState
|
|
|
|
blocks []protocol.BlockInfo
|
2017-01-04 21:04:13 +00:00
|
|
|
have int
|
2014-03-28 13:36:57 +00:00
|
|
|
}
|
|
|
|
|
2015-07-03 09:25:35 +00:00
|
|
|
// Which filemode bits to preserve
|
2017-08-19 14:36:56 +00:00
|
|
|
const retainBits = fs.ModeSetgid | fs.ModeSetuid | fs.ModeSticky
|
2015-07-03 09:25:35 +00:00
|
|
|
|
2014-09-27 12:44:15 +00:00
|
|
|
var (
|
2020-08-20 08:56:29 +00:00
|
|
|
activity = newDeviceActivity()
|
|
|
|
errNoDevice = errors.New("peers who had this file went away, or the file has changed while syncing. will retry later")
|
|
|
|
errDirPrefix = "directory has been deleted on a remote device but "
|
|
|
|
errDirHasToBeScanned = errors.New(errDirPrefix + "contains changed files, scheduling scan")
|
|
|
|
errDirHasIgnored = errors.New(errDirPrefix + "contains ignored files (see ignore documentation for (?d) prefix)")
|
|
|
|
errDirNotEmpty = errors.New(errDirPrefix + "is not empty; the contents are probably ignored on that remote device, but not locally")
|
|
|
|
errNotAvailable = errors.New("no connected device has the required version of this file")
|
|
|
|
errModified = errors.New("file modified but not rescanned; will try again later")
|
|
|
|
errUnexpectedDirOnFileDel = errors.New("encountered directory when trying to remove file/symlink")
|
|
|
|
errIncompatibleSymlink = errors.New("incompatible symlink entry; rescan with newer Syncthing on source")
|
|
|
|
contextRemovingOldItem = "removing item to be replaced"
|
2014-09-27 12:44:15 +00:00
|
|
|
)
|
|
|
|
|
2020-06-16 13:20:08 +00:00
|
|
|
type dbUpdateType int
|
|
|
|
|
|
|
|
func (d dbUpdateType) String() string {
|
|
|
|
switch d {
|
|
|
|
case dbUpdateHandleDir:
|
|
|
|
return "dbUpdateHandleDir"
|
|
|
|
case dbUpdateDeleteDir:
|
|
|
|
return "dbUpdateDeleteDir"
|
|
|
|
case dbUpdateHandleFile:
|
|
|
|
return "dbUpdateHandleFile"
|
|
|
|
case dbUpdateDeleteFile:
|
|
|
|
return "dbUpdateDeleteFile"
|
|
|
|
case dbUpdateShortcutFile:
|
|
|
|
return "dbUpdateShourtcutFile"
|
|
|
|
case dbUpdateHandleSymlink:
|
|
|
|
return "dbUpdateHandleSymlink"
|
|
|
|
case dbUpdateInvalidate:
|
|
|
|
return "dbUpdateHandleInvalidate"
|
|
|
|
}
|
|
|
|
panic(fmt.Sprintf("unknown dbUpdateType %d", d))
|
|
|
|
}
|
|
|
|
|
2015-06-16 11:12:34 +00:00
|
|
|
const (
|
2020-06-16 13:20:08 +00:00
|
|
|
dbUpdateHandleDir dbUpdateType = iota
|
2015-06-16 11:12:34 +00:00
|
|
|
dbUpdateDeleteDir
|
|
|
|
dbUpdateHandleFile
|
|
|
|
dbUpdateDeleteFile
|
|
|
|
dbUpdateShortcutFile
|
2016-12-09 18:02:18 +00:00
|
|
|
dbUpdateHandleSymlink
|
2017-11-11 19:18:17 +00:00
|
|
|
dbUpdateInvalidate
|
2015-06-16 11:12:34 +00:00
|
|
|
)
|
|
|
|
|
2015-08-14 07:37:04 +00:00
|
|
|
const (
|
2018-02-25 09:14:02 +00:00
|
|
|
defaultCopiers = 2
|
|
|
|
defaultPullerPause = 60 * time.Second
|
2018-05-01 21:50:23 +00:00
|
|
|
defaultPullerPendingKiB = 2 * protocol.MaxBlockSize / 1024
|
2018-02-25 09:14:02 +00:00
|
|
|
|
2017-11-07 06:59:35 +00:00
|
|
|
maxPullerIterations = 3
|
2015-08-14 07:37:04 +00:00
|
|
|
)
|
|
|
|
|
2015-06-16 11:12:34 +00:00
|
|
|
type dbUpdateJob struct {
|
|
|
|
file protocol.FileInfo
|
2020-06-16 13:20:08 +00:00
|
|
|
jobType dbUpdateType
|
2015-06-16 11:12:34 +00:00
|
|
|
}
|
|
|
|
|
2016-12-16 22:23:35 +00:00
|
|
|
type sendReceiveFolder struct {
|
2016-04-26 14:01:46 +00:00
|
|
|
folder
|
2015-03-16 20:14:19 +00:00
|
|
|
|
2020-05-11 21:44:04 +00:00
|
|
|
queue *jobQueue
|
|
|
|
blockPullReorderer blockPullReorderer
|
|
|
|
writeLimiter *byteSemaphore
|
2015-06-26 11:31:30 +00:00
|
|
|
|
2020-07-28 09:13:15 +00:00
|
|
|
tempPullErrors map[string]string // pull errors that might be just transient
|
2015-03-16 20:14:19 +00:00
|
|
|
}
|
|
|
|
|
2020-12-27 21:26:25 +00:00
|
|
|
func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *byteSemaphore) service {
|
2016-12-16 22:23:35 +00:00
|
|
|
f := &sendReceiveFolder{
|
2020-07-14 08:48:50 +00:00
|
|
|
folder: newFolder(model, fset, ignores, cfg, evLogger, ioLimiter, ver),
|
2020-05-11 21:44:04 +00:00
|
|
|
queue: newJobQueue(),
|
|
|
|
blockPullReorderer: newBlockPullReorderer(cfg.BlockPullOrder, model.id, cfg.DeviceIDs()),
|
|
|
|
writeLimiter: newByteSemaphore(cfg.MaxConcurrentWrites),
|
2015-03-16 20:14:19 +00:00
|
|
|
}
|
2018-05-11 08:45:13 +00:00
|
|
|
f.folder.puller = f
|
2015-08-14 07:37:04 +00:00
|
|
|
|
2016-12-21 11:23:20 +00:00
|
|
|
if f.Copiers == 0 {
|
|
|
|
f.Copiers = defaultCopiers
|
2015-08-14 07:37:04 +00:00
|
|
|
}
|
2018-02-25 09:14:02 +00:00
|
|
|
|
|
|
|
// If the configured max amount of pending data is zero, we use the
|
|
|
|
// default. If it's configured to something non-zero but less than the
|
|
|
|
// protocol block size we adjust it upwards accordingly.
|
|
|
|
if f.PullerMaxPendingKiB == 0 {
|
|
|
|
f.PullerMaxPendingKiB = defaultPullerPendingKiB
|
|
|
|
}
|
2018-04-16 18:08:50 +00:00
|
|
|
if blockSizeKiB := protocol.MaxBlockSize / 1024; f.PullerMaxPendingKiB < blockSizeKiB {
|
2018-02-25 09:14:02 +00:00
|
|
|
f.PullerMaxPendingKiB = blockSizeKiB
|
2015-08-14 07:37:04 +00:00
|
|
|
}
|
|
|
|
|
2018-02-25 09:14:02 +00:00
|
|
|
return f
|
2014-03-28 13:36:57 +00:00
|
|
|
}
|
|
|
|
|
2019-01-01 09:17:14 +00:00
|
|
|
// pull returns true if it manages to get all needed items from peers, i.e. get
|
|
|
|
// the device in sync with the global state.
|
2018-05-11 08:45:13 +00:00
|
|
|
func (f *sendReceiveFolder) pull() bool {
|
2019-01-01 09:17:14 +00:00
|
|
|
l.Debugf("%v pulling", f)
|
2017-11-07 06:59:35 +00:00
|
|
|
|
2017-12-07 08:42:03 +00:00
|
|
|
scanChan := make(chan string)
|
|
|
|
go f.pullScannerRoutine(scanChan)
|
|
|
|
|
2018-07-04 07:07:33 +00:00
|
|
|
defer func() {
|
|
|
|
close(scanChan)
|
|
|
|
f.setState(FolderIdle)
|
|
|
|
}()
|
|
|
|
|
2019-11-11 14:50:28 +00:00
|
|
|
changed := 0
|
|
|
|
|
2020-11-06 13:22:20 +00:00
|
|
|
f.errorsMut.Lock()
|
2020-07-28 09:13:15 +00:00
|
|
|
f.pullErrors = nil
|
2020-11-06 13:22:20 +00:00
|
|
|
f.errorsMut.Unlock()
|
2020-07-28 09:13:15 +00:00
|
|
|
|
2019-01-14 07:30:52 +00:00
|
|
|
for tries := 0; tries < maxPullerIterations; tries++ {
|
2018-07-04 07:07:33 +00:00
|
|
|
select {
|
|
|
|
case <-f.ctx.Done():
|
|
|
|
return false
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
2019-10-16 07:08:54 +00:00
|
|
|
// Needs to be set on every loop, as the puller might have set
|
|
|
|
// it to FolderSyncing during the last iteration.
|
|
|
|
f.setState(FolderSyncPreparing)
|
|
|
|
|
2019-11-11 14:50:28 +00:00
|
|
|
changed = f.pullerIteration(scanChan)
|
2019-01-14 07:30:52 +00:00
|
|
|
|
2019-02-12 15:05:20 +00:00
|
|
|
l.Debugln(f, "changed", changed, "on try", tries+1)
|
2017-11-07 06:59:35 +00:00
|
|
|
|
|
|
|
if changed == 0 {
|
|
|
|
// No files were changed by the puller, so we are in
|
2019-11-11 14:50:28 +00:00
|
|
|
// sync (except for unrecoverable stuff like invalid
|
|
|
|
// filenames on windows).
|
|
|
|
break
|
2017-11-07 06:59:35 +00:00
|
|
|
}
|
2019-01-14 07:30:52 +00:00
|
|
|
}
|
2017-11-07 06:59:35 +00:00
|
|
|
|
2020-11-06 13:22:20 +00:00
|
|
|
f.errorsMut.Lock()
|
|
|
|
pullErrNum := len(f.tempPullErrors)
|
|
|
|
if pullErrNum > 0 {
|
|
|
|
f.pullErrors = make([]FileError, 0, len(f.tempPullErrors))
|
|
|
|
for path, err := range f.tempPullErrors {
|
|
|
|
l.Infof("Puller (folder %s, item %q): %v", f.Description(), path, err)
|
|
|
|
f.pullErrors = append(f.pullErrors, FileError{
|
|
|
|
Err: err,
|
|
|
|
Path: path,
|
|
|
|
})
|
|
|
|
}
|
|
|
|
f.tempPullErrors = nil
|
2020-07-28 09:13:15 +00:00
|
|
|
}
|
2020-11-06 13:22:20 +00:00
|
|
|
f.errorsMut.Unlock()
|
2020-07-28 09:13:15 +00:00
|
|
|
|
2019-11-19 08:56:53 +00:00
|
|
|
if pullErrNum > 0 {
|
|
|
|
l.Infof("%v: Failed to sync %v items", f.Description(), pullErrNum)
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.FolderErrors, map[string]interface{}{
|
2019-01-14 07:30:52 +00:00
|
|
|
"folder": f.folderID,
|
2019-11-11 14:50:28 +00:00
|
|
|
"errors": f.Errors(),
|
2019-01-14 07:30:52 +00:00
|
|
|
})
|
2017-11-07 06:59:35 +00:00
|
|
|
}
|
|
|
|
|
2019-11-11 14:50:28 +00:00
|
|
|
return changed == 0
|
2017-11-07 06:59:35 +00:00
|
|
|
}
|
|
|
|
|
2014-09-28 11:00:38 +00:00
|
|
|
// pullerIteration runs a single puller iteration for the given folder and
|
2014-09-27 12:44:15 +00:00
|
|
|
// returns the number items that should have been synced (even those that
|
|
|
|
// might have failed). One puller iteration handles all files currently
|
2014-11-23 00:02:09 +00:00
|
|
|
// flagged as needed in the folder.
|
2019-03-11 06:28:54 +00:00
|
|
|
func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
|
2020-11-06 13:22:20 +00:00
|
|
|
f.errorsMut.Lock()
|
2020-07-28 09:13:15 +00:00
|
|
|
f.tempPullErrors = make(map[string]string)
|
2020-11-06 13:22:20 +00:00
|
|
|
f.errorsMut.Unlock()
|
2019-11-19 08:56:53 +00:00
|
|
|
|
2020-01-21 17:23:08 +00:00
|
|
|
snap := f.fset.Snapshot()
|
|
|
|
defer snap.Release()
|
|
|
|
|
2014-09-27 12:44:15 +00:00
|
|
|
pullChan := make(chan pullBlockState)
|
|
|
|
copyChan := make(chan copyBlocksState)
|
|
|
|
finisherChan := make(chan *sharedPullerState)
|
2017-12-07 08:42:03 +00:00
|
|
|
dbUpdateChan := make(chan dbUpdateJob)
|
2014-09-27 12:44:15 +00:00
|
|
|
|
2015-04-22 22:54:31 +00:00
|
|
|
pullWg := sync.NewWaitGroup()
|
2017-12-07 08:42:03 +00:00
|
|
|
copyWg := sync.NewWaitGroup()
|
2015-04-22 22:54:31 +00:00
|
|
|
doneWg := sync.NewWaitGroup()
|
2017-12-07 08:42:03 +00:00
|
|
|
updateWg := sync.NewWaitGroup()
|
2014-09-27 12:44:15 +00:00
|
|
|
|
2018-02-25 09:14:02 +00:00
|
|
|
l.Debugln(f, "copiers:", f.Copiers, "pullerPendingKiB:", f.PullerMaxPendingKiB)
|
2014-11-23 18:43:49 +00:00
|
|
|
|
2015-04-05 13:34:29 +00:00
|
|
|
updateWg.Add(1)
|
|
|
|
go func() {
|
2017-12-07 08:42:03 +00:00
|
|
|
// dbUpdaterRoutine finishes when dbUpdateChan is closed
|
|
|
|
f.dbUpdaterRoutine(dbUpdateChan)
|
2015-04-05 13:34:29 +00:00
|
|
|
updateWg.Done()
|
|
|
|
}()
|
|
|
|
|
2016-12-21 11:23:20 +00:00
|
|
|
for i := 0; i < f.Copiers; i++ {
|
2014-10-08 22:41:23 +00:00
|
|
|
copyWg.Add(1)
|
2014-09-27 12:44:15 +00:00
|
|
|
go func() {
|
|
|
|
// copierRoutine finishes when copyChan is closed
|
2016-04-26 14:01:46 +00:00
|
|
|
f.copierRoutine(copyChan, pullChan, finisherChan)
|
2014-10-08 22:41:23 +00:00
|
|
|
copyWg.Done()
|
2014-09-27 12:44:15 +00:00
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
2018-02-25 09:14:02 +00:00
|
|
|
pullWg.Add(1)
|
|
|
|
go func() {
|
|
|
|
// pullerRoutine finishes when pullChan is closed
|
|
|
|
f.pullerRoutine(pullChan, finisherChan)
|
|
|
|
pullWg.Done()
|
|
|
|
}()
|
2014-09-27 12:44:15 +00:00
|
|
|
|
2014-12-24 23:12:12 +00:00
|
|
|
doneWg.Add(1)
|
|
|
|
// finisherRoutine finishes when finisherChan is closed
|
|
|
|
go func() {
|
2020-01-21 17:23:08 +00:00
|
|
|
f.finisherRoutine(snap, finisherChan, dbUpdateChan, scanChan)
|
2014-12-24 23:12:12 +00:00
|
|
|
doneWg.Done()
|
|
|
|
}()
|
2014-09-27 12:44:15 +00:00
|
|
|
|
2020-01-21 17:23:08 +00:00
|
|
|
changed, fileDeletions, dirDeletions, err := f.processNeeded(snap, dbUpdateChan, copyChan, scanChan)
|
2018-07-04 07:07:33 +00:00
|
|
|
|
|
|
|
// Signal copy and puller routines that we are done with the in data for
|
|
|
|
// this iteration. Wait for them to finish.
|
|
|
|
close(copyChan)
|
|
|
|
copyWg.Wait()
|
|
|
|
close(pullChan)
|
|
|
|
pullWg.Wait()
|
|
|
|
|
|
|
|
// Signal the finisher chan that there will be no more input and wait
|
|
|
|
// for it to finish.
|
|
|
|
close(finisherChan)
|
|
|
|
doneWg.Wait()
|
|
|
|
|
|
|
|
if err == nil {
|
2020-01-21 17:23:08 +00:00
|
|
|
f.processDeletions(fileDeletions, dirDeletions, snap, dbUpdateChan, scanChan)
|
2018-07-04 07:07:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Wait for db updates and scan scheduling to complete
|
|
|
|
close(dbUpdateChan)
|
|
|
|
updateWg.Wait()
|
|
|
|
|
2020-01-08 11:21:22 +00:00
|
|
|
f.queue.Reset()
|
|
|
|
|
2018-07-04 07:07:33 +00:00
|
|
|
return changed
|
|
|
|
}
|
|
|
|
|
2020-01-21 17:23:08 +00:00
|
|
|
func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, copyChan chan<- copyBlocksState, scanChan chan<- string) (int, map[string]protocol.FileInfo, []protocol.FileInfo, error) {
|
2014-09-27 12:44:15 +00:00
|
|
|
changed := 0
|
2018-07-10 15:40:06 +00:00
|
|
|
var dirDeletions []protocol.FileInfo
|
|
|
|
fileDeletions := map[string]protocol.FileInfo{}
|
|
|
|
buckets := map[string][]protocol.FileInfo{}
|
2016-12-13 10:24:10 +00:00
|
|
|
|
|
|
|
// Iterate the list of items that we need and sort them into piles.
|
|
|
|
// Regular files to pull goes into the file queue, everything else
|
|
|
|
// (directories, symlinks and deletes) goes into the "process directly"
|
|
|
|
// pile.
|
2020-05-30 07:50:23 +00:00
|
|
|
snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
|
2018-07-04 07:07:33 +00:00
|
|
|
select {
|
|
|
|
case <-f.ctx.Done():
|
|
|
|
return false
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
2017-11-11 19:18:17 +00:00
|
|
|
if f.IgnoreDelete && intf.IsDeleted() {
|
2017-11-22 08:05:27 +00:00
|
|
|
l.Debugln(f, "ignore file deletion (config)", intf.FileName())
|
2016-12-13 10:24:10 +00:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2019-11-11 14:50:28 +00:00
|
|
|
changed++
|
|
|
|
|
2016-12-13 10:24:10 +00:00
|
|
|
file := intf.(protocol.FileInfo)
|
|
|
|
|
|
|
|
switch {
|
2019-03-11 06:28:54 +00:00
|
|
|
case f.ignores.ShouldIgnore(file.Name):
|
2018-06-24 07:50:18 +00:00
|
|
|
file.SetIgnored(f.shortID)
|
2017-11-11 19:18:17 +00:00
|
|
|
l.Debugln(f, "Handling ignored file", file)
|
2017-12-07 08:42:03 +00:00
|
|
|
dbUpdateChan <- dbUpdateJob{file, dbUpdateInvalidate}
|
2017-11-11 19:18:17 +00:00
|
|
|
|
2020-09-28 08:22:50 +00:00
|
|
|
case runtime.GOOS == "windows" && fs.WindowsInvalidFilename(file.Name) != nil:
|
2019-06-14 06:48:14 +00:00
|
|
|
if file.IsDeleted() {
|
|
|
|
// Just pretend we deleted it, no reason to create an error
|
|
|
|
// about a deleted file that we can't have anyway.
|
|
|
|
// Reason we need it in the first place is, that it was
|
|
|
|
// ignored at some point.
|
|
|
|
dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile}
|
|
|
|
} else {
|
2020-09-28 08:22:50 +00:00
|
|
|
// We can't pull an invalid file. Grab the error again since
|
|
|
|
// we couldn't assign it directly in the case clause.
|
|
|
|
f.newPullError(file.Name, fs.WindowsInvalidFilename(file.Name))
|
2019-11-11 14:50:28 +00:00
|
|
|
// No reason to retry for this
|
|
|
|
changed--
|
2019-06-14 06:48:14 +00:00
|
|
|
}
|
2017-12-25 17:54:34 +00:00
|
|
|
|
2016-12-13 10:24:10 +00:00
|
|
|
case file.IsDeleted():
|
2018-07-10 15:40:06 +00:00
|
|
|
if file.IsDirectory() {
|
|
|
|
// Perform directory deletions at the end, as we may have
|
|
|
|
// files to delete inside them before we get to that point.
|
|
|
|
dirDeletions = append(dirDeletions, file)
|
2019-05-17 16:29:54 +00:00
|
|
|
} else if file.IsSymlink() {
|
2020-01-21 17:23:08 +00:00
|
|
|
f.deleteFile(file, snap, dbUpdateChan, scanChan)
|
2018-07-10 15:40:06 +00:00
|
|
|
} else {
|
2020-01-21 17:23:08 +00:00
|
|
|
df, ok := snap.Get(protocol.LocalDeviceID, file.Name)
|
2018-07-10 15:40:06 +00:00
|
|
|
// Local file can be already deleted, but with a lower version
|
|
|
|
// number, hence the deletion coming in again as part of
|
|
|
|
// WithNeed, furthermore, the file can simply be of the wrong
|
|
|
|
// type if we haven't yet managed to pull it.
|
|
|
|
if ok && !df.IsDeleted() && !df.IsSymlink() && !df.IsDirectory() && !df.IsInvalid() {
|
2019-05-17 16:29:54 +00:00
|
|
|
fileDeletions[file.Name] = file
|
2018-07-10 15:40:06 +00:00
|
|
|
// Put files into buckets per first hash
|
2020-05-11 18:15:11 +00:00
|
|
|
key := string(df.BlocksHash)
|
2018-07-10 15:40:06 +00:00
|
|
|
buckets[key] = append(buckets[key], df)
|
2019-05-17 16:29:54 +00:00
|
|
|
} else {
|
|
|
|
f.deleteFileWithCurrent(file, df, ok, dbUpdateChan, scanChan)
|
2018-07-10 15:40:06 +00:00
|
|
|
}
|
|
|
|
}
|
2016-12-13 10:24:10 +00:00
|
|
|
|
|
|
|
case file.Type == protocol.FileInfoTypeFile:
|
2020-01-21 17:23:08 +00:00
|
|
|
curFile, hasCurFile := snap.Get(protocol.LocalDeviceID, file.Name)
|
2020-03-10 13:46:49 +00:00
|
|
|
if hasCurFile && file.BlocksEqual(curFile) {
|
2019-03-27 08:36:58 +00:00
|
|
|
// We are supposed to copy the entire file, and then fetch nothing. We
|
|
|
|
// are only updating metadata, so we don't actually *need* to make the
|
|
|
|
// copy.
|
|
|
|
f.shortcutFile(file, curFile, dbUpdateChan)
|
|
|
|
} else {
|
|
|
|
// Queue files for processing after directories and symlinks.
|
|
|
|
f.queue.Push(file.Name, file.Size, file.ModTime())
|
|
|
|
}
|
2017-11-11 19:18:17 +00:00
|
|
|
|
|
|
|
case runtime.GOOS == "windows" && file.IsSymlink():
|
2020-09-10 08:52:38 +00:00
|
|
|
if err := f.handleSymlinkCheckExisting(file, snap, scanChan); err != nil {
|
|
|
|
f.newPullError(file.Name, fmt.Errorf("handling unsupported symlink: %w", err))
|
|
|
|
break
|
|
|
|
}
|
2018-06-24 07:50:18 +00:00
|
|
|
file.SetUnsupported(f.shortID)
|
2017-11-11 19:18:17 +00:00
|
|
|
l.Debugln(f, "Invalidating symlink (unsupported)", file.Name)
|
2017-12-07 08:42:03 +00:00
|
|
|
dbUpdateChan <- dbUpdateJob{file, dbUpdateInvalidate}
|
2016-12-13 10:24:10 +00:00
|
|
|
|
2019-03-27 08:36:58 +00:00
|
|
|
case file.IsDirectory() && !file.IsSymlink():
|
|
|
|
l.Debugln(f, "Handling directory", file.Name)
|
|
|
|
if f.checkParent(file.Name, scanChan) {
|
2020-01-21 17:23:08 +00:00
|
|
|
f.handleDir(file, snap, dbUpdateChan, scanChan)
|
2019-03-27 08:36:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
case file.IsSymlink():
|
|
|
|
l.Debugln(f, "Handling symlink", file.Name)
|
|
|
|
if f.checkParent(file.Name, scanChan) {
|
2020-01-21 17:23:08 +00:00
|
|
|
f.handleSymlink(file, snap, dbUpdateChan, scanChan)
|
2019-03-27 08:36:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
default:
|
|
|
|
l.Warnln(file)
|
|
|
|
panic("unhandleable item type, can't happen")
|
2016-12-13 10:24:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return true
|
|
|
|
})
|
|
|
|
|
2018-07-04 07:07:33 +00:00
|
|
|
select {
|
|
|
|
case <-f.ctx.Done():
|
|
|
|
return changed, nil, nil, f.ctx.Err()
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
2016-12-13 10:24:10 +00:00
|
|
|
// Now do the file queue. Reorder it according to configuration.
|
2015-04-25 05:13:53 +00:00
|
|
|
|
2016-12-21 11:23:20 +00:00
|
|
|
switch f.Order {
|
2020-08-25 06:11:14 +00:00
|
|
|
case config.PullOrderRandom:
|
2016-04-26 14:01:46 +00:00
|
|
|
f.queue.Shuffle()
|
2020-08-25 06:11:14 +00:00
|
|
|
case config.PullOrderAlphabetic:
|
2016-04-26 14:01:46 +00:00
|
|
|
// The queue is already in alphabetic order.
|
2020-08-25 06:11:14 +00:00
|
|
|
case config.PullOrderSmallestFirst:
|
2016-04-26 14:01:46 +00:00
|
|
|
f.queue.SortSmallestFirst()
|
2020-08-25 06:11:14 +00:00
|
|
|
case config.PullOrderLargestFirst:
|
2016-04-26 14:01:46 +00:00
|
|
|
f.queue.SortLargestFirst()
|
2020-08-25 06:11:14 +00:00
|
|
|
case config.PullOrderOldestFirst:
|
2016-04-26 14:01:46 +00:00
|
|
|
f.queue.SortOldestFirst()
|
2020-08-25 06:11:14 +00:00
|
|
|
case config.PullOrderNewestFirst:
|
2016-04-26 14:01:46 +00:00
|
|
|
f.queue.SortNewestFirst()
|
2015-04-25 05:13:53 +00:00
|
|
|
}
|
|
|
|
|
2016-12-13 10:24:10 +00:00
|
|
|
// Process the file queue.
|
2015-04-25 05:13:53 +00:00
|
|
|
|
2014-12-19 23:12:12 +00:00
|
|
|
nextFile:
|
Add job queue (fixes #629)
Request to terminate currently ongoing downloads and jump to the bumped file
incoming in 3, 2, 1.
Also, has a slightly strange effect where we pop a job off the queue, but
the copyChannel is still busy and blocks, though it gets moved to the
progress slice in the jobqueue, and looks like it's in progress which it isn't
as it's waiting to be picked up from the copyChan.
As a result, the progress emitter doesn't register on the task, and hence the file
doesn't have a progress bar, but cannot be replaced by a bump.
I guess I can fix progress bar issue by moving the progressEmiter.Register just
before passing the file to the copyChan, but then we are back to the initial
problem of a file with a progress bar, but no progress happening as it's stuck
on write to copyChan
I checked if there is a way to check for channel writeability (before popping)
but got struck by lightning just for bringing the idea up in #go-nuts.
My ideal scenario would be to check if copyChan is writeable, pop job from the
queue and shove it down handleFile. This way jobs would stay in the queue while
they cannot be handled, meaning that the `Bump` could bring your file up higher.
2014-12-01 19:23:06 +00:00
|
|
|
for {
|
2016-01-16 20:42:32 +00:00
|
|
|
select {
|
2017-04-26 00:15:23 +00:00
|
|
|
case <-f.ctx.Done():
|
2018-07-04 07:07:33 +00:00
|
|
|
return changed, fileDeletions, dirDeletions, f.ctx.Err()
|
2016-01-16 20:42:32 +00:00
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
2016-04-26 14:01:46 +00:00
|
|
|
fileName, ok := f.queue.Pop()
|
2014-12-30 08:31:34 +00:00
|
|
|
if !ok {
|
Add job queue (fixes #629)
Request to terminate currently ongoing downloads and jump to the bumped file
incoming in 3, 2, 1.
Also, has a slightly strange effect where we pop a job off the queue, but
the copyChannel is still busy and blocks, though it gets moved to the
progress slice in the jobqueue, and looks like it's in progress which it isn't
as it's waiting to be picked up from the copyChan.
As a result, the progress emitter doesn't register on the task, and hence the file
doesn't have a progress bar, but cannot be replaced by a bump.
I guess I can fix progress bar issue by moving the progressEmiter.Register just
before passing the file to the copyChan, but then we are back to the initial
problem of a file with a progress bar, but no progress happening as it's stuck
on write to copyChan
I checked if there is a way to check for channel writeability (before popping)
but got struck by lightning just for bringing the idea up in #go-nuts.
My ideal scenario would be to check if copyChan is writeable, pop job from the
queue and shove it down handleFile. This way jobs would stay in the queue while
they cannot be handled, meaning that the `Bump` could bring your file up higher.
2014-12-01 19:23:06 +00:00
|
|
|
break
|
|
|
|
}
|
2014-12-19 23:12:12 +00:00
|
|
|
|
2020-01-21 17:23:08 +00:00
|
|
|
fi, ok := snap.GetGlobal(fileName)
|
2014-12-19 23:12:12 +00:00
|
|
|
if !ok {
|
2015-01-06 21:12:45 +00:00
|
|
|
// File is no longer in the index. Mark it as done and drop it.
|
2016-04-26 14:01:46 +00:00
|
|
|
f.queue.Done(fileName)
|
2014-12-19 23:12:12 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2018-10-26 18:13:35 +00:00
|
|
|
if fi.IsDeleted() || fi.IsInvalid() || fi.Type != protocol.FileInfoTypeFile {
|
2016-12-13 10:24:10 +00:00
|
|
|
// The item has changed type or status in the index while we
|
|
|
|
// were processing directories above.
|
|
|
|
f.queue.Done(fileName)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2018-07-10 15:40:06 +00:00
|
|
|
if !f.checkParent(fi.Name, scanChan) {
|
2018-08-19 21:34:26 +00:00
|
|
|
f.queue.Done(fileName)
|
2016-01-16 17:18:37 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2016-12-09 18:02:18 +00:00
|
|
|
// Check our list of files to be removed for a match, in which case
|
|
|
|
// we can just do a rename instead.
|
2020-05-11 18:15:11 +00:00
|
|
|
key := string(fi.BlocksHash)
|
2020-05-17 07:48:35 +00:00
|
|
|
for candidate, ok := popCandidate(buckets, key); ok; candidate, ok = popCandidate(buckets, key) {
|
2020-05-11 18:15:11 +00:00
|
|
|
// candidate is our current state of the file, where as the
|
|
|
|
// desired state with the delete bit set is in the deletion
|
|
|
|
// map.
|
|
|
|
desired := fileDeletions[candidate.Name]
|
|
|
|
if err := f.renameFile(candidate, desired, fi, snap, dbUpdateChan, scanChan); err != nil {
|
2020-05-13 22:31:05 +00:00
|
|
|
l.Debugf("rename shortcut for %s failed: %s", fi.Name, err.Error())
|
2020-05-11 18:15:11 +00:00
|
|
|
// Failed to rename, try next one.
|
|
|
|
continue
|
|
|
|
}
|
2016-12-09 18:02:18 +00:00
|
|
|
|
2020-05-11 18:15:11 +00:00
|
|
|
// Remove the pending deletion (as we performed it by renaming)
|
|
|
|
delete(fileDeletions, candidate.Name)
|
2016-12-09 18:02:18 +00:00
|
|
|
|
2020-05-11 18:15:11 +00:00
|
|
|
f.queue.Done(fileName)
|
|
|
|
continue nextFile
|
2015-01-06 21:12:45 +00:00
|
|
|
}
|
2014-12-19 23:12:12 +00:00
|
|
|
|
2020-01-21 17:23:08 +00:00
|
|
|
devices := snap.Availability(fileName)
|
2018-08-19 18:03:20 +00:00
|
|
|
for _, dev := range devices {
|
|
|
|
if _, ok := f.model.Connection(dev); ok {
|
|
|
|
// Handle the file normally, by coping and pulling, etc.
|
2020-01-21 17:23:08 +00:00
|
|
|
f.handleFile(fi, snap, copyChan)
|
2018-08-19 18:03:20 +00:00
|
|
|
continue nextFile
|
|
|
|
}
|
|
|
|
}
|
2019-03-04 13:01:52 +00:00
|
|
|
f.newPullError(fileName, errNotAvailable)
|
2019-03-27 19:19:35 +00:00
|
|
|
f.queue.Done(fileName)
|
Add job queue (fixes #629)
Request to terminate currently ongoing downloads and jump to the bumped file
incoming in 3, 2, 1.
Also, has a slightly strange effect where we pop a job off the queue, but
the copyChannel is still busy and blocks, though it gets moved to the
progress slice in the jobqueue, and looks like it's in progress which it isn't
as it's waiting to be picked up from the copyChan.
As a result, the progress emitter doesn't register on the task, and hence the file
doesn't have a progress bar, but cannot be replaced by a bump.
I guess I can fix progress bar issue by moving the progressEmiter.Register just
before passing the file to the copyChan, but then we are back to the initial
problem of a file with a progress bar, but no progress happening as it's stuck
on write to copyChan
I checked if there is a way to check for channel writeability (before popping)
but got struck by lightning just for bringing the idea up in #go-nuts.
My ideal scenario would be to check if copyChan is writeable, pop job from the
queue and shove it down handleFile. This way jobs would stay in the queue while
they cannot be handled, meaning that the `Bump` could bring your file up higher.
2014-12-01 19:23:06 +00:00
|
|
|
}
|
|
|
|
|
2018-07-04 07:07:33 +00:00
|
|
|
return changed, fileDeletions, dirDeletions, nil
|
|
|
|
}
|
2014-05-19 20:31:28 +00:00
|
|
|
|
2020-05-17 07:48:35 +00:00
|
|
|
func popCandidate(buckets map[string][]protocol.FileInfo, key string) (protocol.FileInfo, bool) {
|
|
|
|
cands := buckets[key]
|
|
|
|
if len(cands) == 0 {
|
|
|
|
return protocol.FileInfo{}, false
|
|
|
|
}
|
|
|
|
|
|
|
|
buckets[key] = cands[1:]
|
|
|
|
return cands[0], true
|
|
|
|
}
|
|
|
|
|
2020-01-21 17:23:08 +00:00
|
|
|
func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.FileInfo, dirDeletions []protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
|
2014-12-19 23:12:12 +00:00
|
|
|
for _, file := range fileDeletions {
|
2018-07-04 07:07:33 +00:00
|
|
|
select {
|
|
|
|
case <-f.ctx.Done():
|
|
|
|
return
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
2020-01-21 17:23:08 +00:00
|
|
|
f.deleteFile(file, snap, dbUpdateChan, scanChan)
|
2014-12-19 23:12:12 +00:00
|
|
|
}
|
|
|
|
|
2019-05-06 18:55:26 +00:00
|
|
|
// Process in reverse order to delete depth first
|
|
|
|
for i := range dirDeletions {
|
2018-07-04 07:07:33 +00:00
|
|
|
select {
|
|
|
|
case <-f.ctx.Done():
|
|
|
|
return
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
2019-05-06 18:55:26 +00:00
|
|
|
dir := dirDeletions[len(dirDeletions)-i-1]
|
2017-11-11 19:18:17 +00:00
|
|
|
l.Debugln(f, "Deleting dir", dir.Name)
|
2020-01-21 17:23:08 +00:00
|
|
|
f.deleteDir(dir, snap, dbUpdateChan, scanChan)
|
2014-10-12 21:01:57 +00:00
|
|
|
}
|
2014-09-27 12:44:15 +00:00
|
|
|
}
|
2014-04-01 21:18:32 +00:00
|
|
|
|
2014-09-27 12:44:15 +00:00
|
|
|
// handleDir creates or updates the given directory
|
2020-01-21 17:23:08 +00:00
|
|
|
func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
|
2016-12-09 18:02:18 +00:00
|
|
|
// Used in the defer closure below, updated by the function body. Take
|
|
|
|
// care not declare another err.
|
2015-02-01 17:31:19 +00:00
|
|
|
var err error
|
2016-12-09 18:02:18 +00:00
|
|
|
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.ItemStarted, map[string]string{
|
2016-04-26 14:01:46 +00:00
|
|
|
"folder": f.folderID,
|
2015-04-14 11:59:06 +00:00
|
|
|
"item": file.Name,
|
|
|
|
"type": "dir",
|
|
|
|
"action": "update",
|
2015-02-01 17:31:19 +00:00
|
|
|
})
|
2015-04-14 11:59:06 +00:00
|
|
|
|
2015-02-01 17:31:19 +00:00
|
|
|
defer func() {
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.ItemFinished, map[string]interface{}{
|
2016-04-26 14:01:46 +00:00
|
|
|
"folder": f.folderID,
|
2015-02-01 17:31:19 +00:00
|
|
|
"item": file.Name,
|
2015-05-27 09:14:39 +00:00
|
|
|
"error": events.Error(err),
|
2015-04-14 11:59:06 +00:00
|
|
|
"type": "dir",
|
|
|
|
"action": "update",
|
2015-02-01 17:31:19 +00:00
|
|
|
})
|
|
|
|
}()
|
|
|
|
|
2017-08-19 14:36:56 +00:00
|
|
|
mode := fs.FileMode(file.Permissions & 0777)
|
2018-02-25 08:39:00 +00:00
|
|
|
if f.IgnorePerms || file.NoPermissions {
|
2015-05-24 22:12:51 +00:00
|
|
|
mode = 0777
|
2014-10-09 22:34:32 +00:00
|
|
|
}
|
2014-05-19 20:31:28 +00:00
|
|
|
|
2015-10-03 15:25:21 +00:00
|
|
|
if shouldDebug() {
|
2020-01-21 17:23:08 +00:00
|
|
|
curFile, _ := snap.Get(protocol.LocalDeviceID, file.Name)
|
2014-09-27 12:44:15 +00:00
|
|
|
l.Debugf("need dir\n\t%v\n\t%v", file, curFile)
|
2014-04-01 21:18:32 +00:00
|
|
|
}
|
|
|
|
|
2020-12-27 21:26:25 +00:00
|
|
|
info, err := f.mtimefs.Lstat(file.Name)
|
2014-11-13 22:59:40 +00:00
|
|
|
switch {
|
2019-03-07 14:15:14 +00:00
|
|
|
// There is already something under that name, we need to handle that.
|
|
|
|
// Unless it already is a directory, as we only track permissions,
|
|
|
|
// that don't result in a conflict.
|
|
|
|
case err == nil && !info.IsDir():
|
|
|
|
// Check that it is what we have in the database.
|
|
|
|
curFile, hasCurFile := f.model.CurrentFolderFile(f.folderID, file.Name)
|
2020-06-16 13:20:08 +00:00
|
|
|
if err := f.scanIfItemChanged(file.Name, info, curFile, hasCurFile, scanChan); err != nil {
|
2019-06-29 05:45:41 +00:00
|
|
|
err = errors.Wrap(err, "handling dir")
|
2019-03-07 14:15:14 +00:00
|
|
|
f.newPullError(file.Name, err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Remove it to replace with the dir.
|
|
|
|
if !curFile.IsSymlink() && f.inConflict(curFile.Version, file.Version) {
|
|
|
|
// The new file has been changed in conflict with the existing one. We
|
|
|
|
// should file it away as a conflict instead of just removing or
|
2020-10-19 06:53:19 +00:00
|
|
|
// archiving.
|
2019-03-07 14:15:14 +00:00
|
|
|
// Symlinks aren't checked for conflicts.
|
|
|
|
|
2019-07-31 08:53:35 +00:00
|
|
|
err = f.inWritableDir(func(name string) error {
|
2019-03-07 14:15:14 +00:00
|
|
|
return f.moveForConflict(name, file.ModifiedBy.String(), scanChan)
|
2019-07-31 08:53:35 +00:00
|
|
|
}, curFile.Name)
|
2019-03-07 14:15:14 +00:00
|
|
|
} else {
|
2020-01-21 17:23:08 +00:00
|
|
|
err = f.deleteItemOnDisk(curFile, snap, scanChan)
|
2019-03-07 14:15:14 +00:00
|
|
|
}
|
2014-11-13 22:59:40 +00:00
|
|
|
if err != nil {
|
2019-03-07 14:15:14 +00:00
|
|
|
f.newPullError(file.Name, err)
|
2014-09-27 23:54:25 +00:00
|
|
|
return
|
|
|
|
}
|
2014-11-13 22:59:40 +00:00
|
|
|
fallthrough
|
|
|
|
// The directory doesn't exist, so we create it with the right
|
|
|
|
// mode bits from the start.
|
2017-08-19 14:36:56 +00:00
|
|
|
case err != nil && fs.IsNotExist(err):
|
2014-11-13 22:59:40 +00:00
|
|
|
// We declare a function that acts on only the path name, so
|
|
|
|
// we can pass it to InWritableDir. We use a regular Mkdir and
|
|
|
|
// not MkdirAll because the parent should already exist.
|
|
|
|
mkdir := func(path string) error {
|
2020-12-27 21:26:25 +00:00
|
|
|
err = f.mtimefs.Mkdir(path, mode)
|
2018-02-25 08:39:00 +00:00
|
|
|
if err != nil || f.IgnorePerms || file.NoPermissions {
|
2015-03-23 12:31:53 +00:00
|
|
|
return err
|
|
|
|
}
|
2015-07-03 09:25:35 +00:00
|
|
|
|
2019-01-25 08:52:21 +00:00
|
|
|
// Copy the parent owner and group, if we are supposed to do that.
|
|
|
|
if err := f.maybeCopyOwner(path); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2015-07-03 09:25:35 +00:00
|
|
|
// Stat the directory so we can check its permissions.
|
2020-12-27 21:26:25 +00:00
|
|
|
info, err := f.mtimefs.Lstat(path)
|
2015-07-03 09:25:35 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Mask for the bits we want to preserve and add them in to the
|
|
|
|
// directories permissions.
|
2020-12-27 21:26:25 +00:00
|
|
|
return f.mtimefs.Chmod(path, mode|(info.Mode()&retainBits))
|
2014-11-13 22:59:40 +00:00
|
|
|
}
|
2014-09-27 23:54:25 +00:00
|
|
|
|
2019-07-31 08:53:35 +00:00
|
|
|
if err = f.inWritableDir(mkdir, file.Name); err == nil {
|
2017-12-07 08:42:03 +00:00
|
|
|
dbUpdateChan <- dbUpdateJob{file, dbUpdateHandleDir}
|
2014-11-13 22:59:40 +00:00
|
|
|
} else {
|
2019-03-04 13:01:52 +00:00
|
|
|
f.newPullError(file.Name, errors.Wrap(err, "creating directory"))
|
2014-11-13 22:59:40 +00:00
|
|
|
}
|
2014-09-27 23:54:25 +00:00
|
|
|
return
|
2014-11-13 22:59:40 +00:00
|
|
|
// Weird error when stat()'ing the dir. Probably won't work to do
|
|
|
|
// anything else with it if we can't even stat() it.
|
|
|
|
case err != nil:
|
2019-03-04 13:01:52 +00:00
|
|
|
f.newPullError(file.Name, errors.Wrap(err, "checking file to be replaced"))
|
2014-03-28 13:36:57 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2014-09-27 23:54:25 +00:00
|
|
|
// The directory already exists, so we just correct the mode bits. (We
|
|
|
|
// don't handle modification times on directories, because that sucks...)
|
|
|
|
// It's OK to change mode bits on stuff within non-writable directories.
|
2019-03-04 13:01:52 +00:00
|
|
|
if !f.IgnorePerms && !file.NoPermissions {
|
2020-12-27 21:26:25 +00:00
|
|
|
if err := f.mtimefs.Chmod(file.Name, mode|(info.Mode()&retainBits)); err != nil {
|
2019-03-04 13:01:52 +00:00
|
|
|
f.newPullError(file.Name, err)
|
|
|
|
return
|
|
|
|
}
|
2014-07-15 11:04:37 +00:00
|
|
|
}
|
2019-03-04 13:01:52 +00:00
|
|
|
dbUpdateChan <- dbUpdateJob{file, dbUpdateHandleDir}
|
2014-09-27 12:44:15 +00:00
|
|
|
}
|
2014-03-28 13:36:57 +00:00
|
|
|
|
2018-07-10 15:40:06 +00:00
|
|
|
// checkParent verifies that the thing we are handling lives inside a directory,
|
|
|
|
// and not a symlink or regular file. It also resurrects missing parent dirs.
|
|
|
|
func (f *sendReceiveFolder) checkParent(file string, scanChan chan<- string) bool {
|
|
|
|
parent := filepath.Dir(file)
|
|
|
|
|
2020-12-27 21:26:25 +00:00
|
|
|
if err := osutil.TraversesSymlink(f.mtimefs, parent); err != nil {
|
2019-03-04 13:01:52 +00:00
|
|
|
f.newPullError(file, errors.Wrap(err, "checking parent dirs"))
|
2018-07-10 15:40:06 +00:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// issues #114 and #4475: This works around a race condition
|
|
|
|
// between two devices, when one device removes a directory and the
|
|
|
|
// other creates a file in it. However that happens, we end up with
|
|
|
|
// a directory for "foo" with the delete bit, but a file "foo/bar"
|
|
|
|
// that we want to sync. We never create the directory, and hence
|
|
|
|
// fail to create the file and end up looping forever on it. This
|
|
|
|
// breaks that by creating the directory and scheduling a scan,
|
|
|
|
// where it will be found and the delete bit on it removed. The
|
|
|
|
// user can then clean up as they like...
|
|
|
|
// This can also occur if an entire tree structure was deleted, but only
|
|
|
|
// a leave has been scanned.
|
2020-11-09 14:33:32 +00:00
|
|
|
//
|
|
|
|
// And if this is an encrypted folder:
|
|
|
|
// Encrypted files have made-up filenames with two synthetic parent
|
|
|
|
// directories which don't have any meaning. Create those if necessary.
|
2020-12-27 21:26:25 +00:00
|
|
|
if _, err := f.mtimefs.Lstat(parent); !fs.IsNotExist(err) {
|
2018-07-10 15:40:06 +00:00
|
|
|
l.Debugf("%v parent not missing %v", f, file)
|
|
|
|
return true
|
|
|
|
}
|
2020-11-09 14:33:32 +00:00
|
|
|
l.Debugf("%v creating parent directory of %v", f, file)
|
2020-12-27 21:26:25 +00:00
|
|
|
if err := f.mtimefs.MkdirAll(parent, 0755); err != nil {
|
2020-11-09 14:33:32 +00:00
|
|
|
f.newPullError(file, errors.Wrap(err, "creating parent dir"))
|
2018-07-10 15:40:06 +00:00
|
|
|
return false
|
|
|
|
}
|
2020-11-09 14:33:32 +00:00
|
|
|
if f.Type != config.FolderTypeReceiveEncrypted {
|
|
|
|
scanChan <- parent
|
|
|
|
}
|
2018-07-10 15:40:06 +00:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2016-12-09 18:02:18 +00:00
|
|
|
// handleSymlink creates or updates the given symlink
|
2020-01-21 17:23:08 +00:00
|
|
|
func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
|
2016-12-09 18:02:18 +00:00
|
|
|
// Used in the defer closure below, updated by the function body. Take
|
|
|
|
// care not declare another err.
|
|
|
|
var err error
|
|
|
|
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.ItemStarted, map[string]string{
|
2016-12-09 18:02:18 +00:00
|
|
|
"folder": f.folderID,
|
|
|
|
"item": file.Name,
|
|
|
|
"type": "symlink",
|
|
|
|
"action": "update",
|
|
|
|
})
|
|
|
|
|
|
|
|
defer func() {
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.ItemFinished, map[string]interface{}{
|
2016-12-09 18:02:18 +00:00
|
|
|
"folder": f.folderID,
|
|
|
|
"item": file.Name,
|
|
|
|
"error": events.Error(err),
|
|
|
|
"type": "symlink",
|
|
|
|
"action": "update",
|
|
|
|
})
|
|
|
|
}()
|
|
|
|
|
|
|
|
if shouldDebug() {
|
2020-01-21 17:23:08 +00:00
|
|
|
curFile, _ := snap.Get(protocol.LocalDeviceID, file.Name)
|
2016-12-09 18:02:18 +00:00
|
|
|
l.Debugf("need symlink\n\t%v\n\t%v", file, curFile)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(file.SymlinkTarget) == 0 {
|
|
|
|
// Index entry from a Syncthing predating the support for including
|
|
|
|
// the link target in the index entry. We log this as an error.
|
2019-03-04 13:01:52 +00:00
|
|
|
f.newPullError(file.Name, errIncompatibleSymlink)
|
2016-12-09 18:02:18 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2020-09-10 08:52:38 +00:00
|
|
|
if f.handleSymlinkCheckExisting(file, snap, scanChan); err != nil {
|
|
|
|
f.newPullError(file.Name, fmt.Errorf("handling symlink: %w", err))
|
2020-07-28 09:13:15 +00:00
|
|
|
return
|
2016-12-09 18:02:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// We declare a function that acts on only the path name, so
|
|
|
|
// we can pass it to InWritableDir.
|
|
|
|
createLink := func(path string) error {
|
2020-12-27 21:26:25 +00:00
|
|
|
if err := f.mtimefs.CreateSymlink(file.SymlinkTarget, path); err != nil {
|
2019-01-25 08:52:21 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
return f.maybeCopyOwner(path)
|
2016-12-09 18:02:18 +00:00
|
|
|
}
|
|
|
|
|
2019-07-31 08:53:35 +00:00
|
|
|
if err = f.inWritableDir(createLink, file.Name); err == nil {
|
2017-12-07 08:42:03 +00:00
|
|
|
dbUpdateChan <- dbUpdateJob{file, dbUpdateHandleSymlink}
|
2016-12-09 18:02:18 +00:00
|
|
|
} else {
|
2019-03-04 13:01:52 +00:00
|
|
|
f.newPullError(file.Name, errors.Wrap(err, "symlink create"))
|
2016-12-09 18:02:18 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-09-10 08:52:38 +00:00
|
|
|
func (f *sendReceiveFolder) handleSymlinkCheckExisting(file protocol.FileInfo, snap *db.Snapshot, scanChan chan<- string) error {
|
|
|
|
// If there is already something under that name, we need to handle that.
|
2020-12-27 21:26:25 +00:00
|
|
|
info, err := f.mtimefs.Lstat(file.Name)
|
2020-09-10 08:52:38 +00:00
|
|
|
if err != nil {
|
|
|
|
if fs.IsNotExist(err) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
// Check that it is what we have in the database.
|
|
|
|
curFile, hasCurFile := f.model.CurrentFolderFile(f.folderID, file.Name)
|
|
|
|
if err := f.scanIfItemChanged(file.Name, info, curFile, hasCurFile, scanChan); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
// Remove it to replace with the symlink. This also handles the
|
|
|
|
// "change symlink type" path.
|
|
|
|
if !curFile.IsDirectory() && !curFile.IsSymlink() && f.inConflict(curFile.Version, file.Version) {
|
|
|
|
// The new file has been changed in conflict with the existing one. We
|
|
|
|
// should file it away as a conflict instead of just removing or
|
2020-10-19 06:53:19 +00:00
|
|
|
// archiving.
|
2020-09-10 08:52:38 +00:00
|
|
|
// Directories and symlinks aren't checked for conflicts.
|
|
|
|
|
|
|
|
return f.inWritableDir(func(name string) error {
|
|
|
|
return f.moveForConflict(name, file.ModifiedBy.String(), scanChan)
|
|
|
|
}, curFile.Name)
|
|
|
|
} else {
|
|
|
|
return f.deleteItemOnDisk(curFile, snap, scanChan)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-03-07 14:15:14 +00:00
|
|
|
// deleteDir attempts to remove a directory that was deleted on a remote
|
2020-01-21 17:23:08 +00:00
|
|
|
func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
|
2016-12-09 18:02:18 +00:00
|
|
|
// Used in the defer closure below, updated by the function body. Take
|
|
|
|
// care not declare another err.
|
2015-02-01 17:31:19 +00:00
|
|
|
var err error
|
2016-12-09 18:02:18 +00:00
|
|
|
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.ItemStarted, map[string]string{
|
2016-04-26 14:01:46 +00:00
|
|
|
"folder": f.folderID,
|
2015-04-14 11:59:06 +00:00
|
|
|
"item": file.Name,
|
|
|
|
"type": "dir",
|
|
|
|
"action": "delete",
|
2015-02-01 17:31:19 +00:00
|
|
|
})
|
2016-12-09 18:02:18 +00:00
|
|
|
|
2015-02-01 17:31:19 +00:00
|
|
|
defer func() {
|
2020-06-16 13:20:08 +00:00
|
|
|
if err != nil {
|
|
|
|
f.newPullError(file.Name, errors.Wrap(err, "delete dir"))
|
|
|
|
}
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.ItemFinished, map[string]interface{}{
|
2016-04-26 14:01:46 +00:00
|
|
|
"folder": f.folderID,
|
2015-02-01 17:31:19 +00:00
|
|
|
"item": file.Name,
|
2015-05-27 09:14:39 +00:00
|
|
|
"error": events.Error(err),
|
2015-04-14 11:59:06 +00:00
|
|
|
"type": "dir",
|
|
|
|
"action": "delete",
|
2015-02-01 17:31:19 +00:00
|
|
|
})
|
|
|
|
}()
|
|
|
|
|
2020-06-16 13:20:08 +00:00
|
|
|
cur, hasCur := snap.Get(protocol.LocalDeviceID, file.Name)
|
|
|
|
|
2020-09-12 05:45:50 +00:00
|
|
|
if err = f.checkToBeDeleted(file, cur, hasCur, scanChan); err != nil {
|
|
|
|
if fs.IsNotExist(err) || fs.IsErrCaseConflict(err) {
|
2020-06-16 13:20:08 +00:00
|
|
|
err = nil
|
2020-09-12 05:45:50 +00:00
|
|
|
dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteDir}
|
2020-06-16 13:20:08 +00:00
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2020-01-21 17:23:08 +00:00
|
|
|
if err = f.deleteDirOnDisk(file.Name, snap, scanChan); err != nil {
|
2017-12-07 08:42:03 +00:00
|
|
|
return
|
2014-03-28 13:36:57 +00:00
|
|
|
}
|
2017-12-07 08:42:03 +00:00
|
|
|
|
|
|
|
dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteDir}
|
2014-03-28 13:36:57 +00:00
|
|
|
}
|
|
|
|
|
2014-09-27 12:44:15 +00:00
|
|
|
// deleteFile attempts to delete the given file
|
2020-01-21 17:23:08 +00:00
|
|
|
func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
|
|
|
|
cur, hasCur := snap.Get(protocol.LocalDeviceID, file.Name)
|
2019-05-17 16:29:54 +00:00
|
|
|
f.deleteFileWithCurrent(file, cur, hasCur, dbUpdateChan, scanChan)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *sendReceiveFolder) deleteFileWithCurrent(file, cur protocol.FileInfo, hasCur bool, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
|
2016-12-09 18:02:18 +00:00
|
|
|
// Used in the defer closure below, updated by the function body. Take
|
|
|
|
// care not declare another err.
|
2015-02-01 17:31:19 +00:00
|
|
|
var err error
|
2016-12-09 18:02:18 +00:00
|
|
|
|
2019-05-17 16:29:54 +00:00
|
|
|
l.Debugln(f, "Deleting file", file.Name)
|
|
|
|
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.ItemStarted, map[string]string{
|
2016-04-26 14:01:46 +00:00
|
|
|
"folder": f.folderID,
|
2015-04-14 11:59:06 +00:00
|
|
|
"item": file.Name,
|
|
|
|
"type": "file",
|
|
|
|
"action": "delete",
|
2015-02-01 17:31:19 +00:00
|
|
|
})
|
2016-12-09 18:02:18 +00:00
|
|
|
|
2015-02-01 17:31:19 +00:00
|
|
|
defer func() {
|
2019-05-17 16:29:54 +00:00
|
|
|
if err != nil {
|
|
|
|
f.newPullError(file.Name, errors.Wrap(err, "delete file"))
|
|
|
|
}
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.ItemFinished, map[string]interface{}{
|
2016-04-26 14:01:46 +00:00
|
|
|
"folder": f.folderID,
|
2015-02-01 17:31:19 +00:00
|
|
|
"item": file.Name,
|
2015-05-27 09:14:39 +00:00
|
|
|
"error": events.Error(err),
|
2015-04-14 11:59:06 +00:00
|
|
|
"type": "file",
|
|
|
|
"action": "delete",
|
2015-02-01 17:31:19 +00:00
|
|
|
})
|
|
|
|
}()
|
|
|
|
|
2020-09-12 05:45:50 +00:00
|
|
|
if err = f.checkToBeDeleted(file, cur, hasCur, scanChan); err != nil {
|
|
|
|
if fs.IsNotExist(err) || fs.IsErrCaseConflict(err) {
|
2020-06-16 13:20:08 +00:00
|
|
|
err = nil
|
2020-09-12 05:45:50 +00:00
|
|
|
dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile}
|
2020-06-16 13:20:08 +00:00
|
|
|
}
|
2019-05-17 16:29:54 +00:00
|
|
|
return
|
2018-09-16 07:48:14 +00:00
|
|
|
}
|
|
|
|
|
2019-03-07 14:15:14 +00:00
|
|
|
// We are asked to delete a file, but what we have on disk and in db
|
|
|
|
// is a directory. Something is wrong here, should probably not happen.
|
|
|
|
if cur.IsDirectory() {
|
2019-05-17 16:29:54 +00:00
|
|
|
err = errUnexpectedDirOnFileDel
|
|
|
|
return
|
2019-03-07 14:15:14 +00:00
|
|
|
}
|
|
|
|
|
2018-09-16 07:48:14 +00:00
|
|
|
if f.inConflict(cur.Version, file.Version) {
|
2019-03-07 14:15:14 +00:00
|
|
|
// There is a conflict here, which shouldn't happen as deletions
|
|
|
|
// always lose. Merge the version vector of the file we have
|
|
|
|
// locally and commit it to db to resolve the conflict.
|
|
|
|
cur.Version = cur.Version.Merge(file.Version)
|
2019-05-17 16:29:54 +00:00
|
|
|
dbUpdateChan <- dbUpdateJob{cur, dbUpdateHandleFile}
|
|
|
|
return
|
2019-03-07 14:15:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if f.versioner != nil && !cur.IsSymlink() {
|
2019-07-31 08:53:35 +00:00
|
|
|
err = f.inWritableDir(f.versioner.Archive, file.Name)
|
2014-09-27 12:44:15 +00:00
|
|
|
} else {
|
2020-12-27 21:26:25 +00:00
|
|
|
err = f.inWritableDir(f.mtimefs.Remove, file.Name)
|
2014-09-27 12:44:15 +00:00
|
|
|
}
|
2014-07-13 19:07:24 +00:00
|
|
|
|
2017-08-19 14:36:56 +00:00
|
|
|
if err == nil || fs.IsNotExist(err) {
|
2015-05-23 21:55:50 +00:00
|
|
|
// It was removed or it doesn't exist to start with
|
2019-05-17 16:29:54 +00:00
|
|
|
dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile}
|
|
|
|
return
|
2018-07-12 08:15:57 +00:00
|
|
|
}
|
|
|
|
|
2020-12-27 21:26:25 +00:00
|
|
|
if _, serr := f.mtimefs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) {
|
2015-05-23 21:55:50 +00:00
|
|
|
// We get an error just looking at the file, and it's not a permission
|
|
|
|
// problem. Lets assume the error is in fact some variant of "file
|
|
|
|
// does not exist" (possibly expressed as some parent being a file and
|
|
|
|
// not a directory etc) and that the delete is handled.
|
2018-09-16 07:48:14 +00:00
|
|
|
err = nil
|
2019-05-17 16:29:54 +00:00
|
|
|
dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile}
|
2014-05-28 09:45:45 +00:00
|
|
|
}
|
2014-09-27 12:44:15 +00:00
|
|
|
}
|
2014-05-28 09:45:45 +00:00
|
|
|
|
2014-12-19 23:12:12 +00:00
|
|
|
// renameFile attempts to rename an existing file to a destination
|
|
|
|
// and set the right attributes on it.
|
2020-01-21 17:23:08 +00:00
|
|
|
func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error {
|
2016-12-09 18:02:18 +00:00
|
|
|
// Used in the defer closure below, updated by the function body. Take
|
|
|
|
// care not declare another err.
|
2015-02-01 17:31:19 +00:00
|
|
|
var err error
|
2016-12-09 18:02:18 +00:00
|
|
|
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.ItemStarted, map[string]string{
|
2016-04-26 14:01:46 +00:00
|
|
|
"folder": f.folderID,
|
2015-04-14 11:59:06 +00:00
|
|
|
"item": source.Name,
|
|
|
|
"type": "file",
|
|
|
|
"action": "delete",
|
2015-02-01 17:31:19 +00:00
|
|
|
})
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.ItemStarted, map[string]string{
|
2016-04-26 14:01:46 +00:00
|
|
|
"folder": f.folderID,
|
2015-04-14 11:59:06 +00:00
|
|
|
"item": target.Name,
|
|
|
|
"type": "file",
|
|
|
|
"action": "update",
|
2015-02-01 17:31:19 +00:00
|
|
|
})
|
2016-12-09 18:02:18 +00:00
|
|
|
|
2015-02-01 17:31:19 +00:00
|
|
|
defer func() {
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.ItemFinished, map[string]interface{}{
|
2016-04-26 14:01:46 +00:00
|
|
|
"folder": f.folderID,
|
2015-02-01 17:31:19 +00:00
|
|
|
"item": source.Name,
|
2015-05-27 09:14:39 +00:00
|
|
|
"error": events.Error(err),
|
2015-04-14 11:59:06 +00:00
|
|
|
"type": "file",
|
|
|
|
"action": "delete",
|
2015-02-01 17:31:19 +00:00
|
|
|
})
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.ItemFinished, map[string]interface{}{
|
2016-04-26 14:01:46 +00:00
|
|
|
"folder": f.folderID,
|
2015-02-01 17:31:19 +00:00
|
|
|
"item": target.Name,
|
2015-05-27 09:14:39 +00:00
|
|
|
"error": events.Error(err),
|
2015-04-14 11:59:06 +00:00
|
|
|
"type": "file",
|
|
|
|
"action": "update",
|
2015-02-01 17:31:19 +00:00
|
|
|
})
|
|
|
|
}()
|
|
|
|
|
2016-04-26 14:01:46 +00:00
|
|
|
l.Debugln(f, "taking rename shortcut", source.Name, "->", target.Name)
|
2014-12-19 23:12:12 +00:00
|
|
|
|
2018-09-16 07:48:14 +00:00
|
|
|
// Check that source is compatible with what we have in the DB
|
2020-09-12 05:45:50 +00:00
|
|
|
if err = f.checkToBeDeleted(source, cur, true, scanChan); err != nil {
|
2018-10-10 09:37:20 +00:00
|
|
|
return err
|
2018-09-16 07:48:14 +00:00
|
|
|
}
|
|
|
|
// Check that the target corresponds to what we have in the DB
|
2020-01-21 17:23:08 +00:00
|
|
|
curTarget, ok := snap.Get(protocol.LocalDeviceID, target.Name)
|
2020-12-27 21:26:25 +00:00
|
|
|
switch stat, serr := f.mtimefs.Lstat(target.Name); {
|
2018-09-16 07:48:14 +00:00
|
|
|
case serr != nil:
|
2020-09-10 06:23:54 +00:00
|
|
|
var caseErr *fs.ErrCaseConflict
|
|
|
|
switch {
|
|
|
|
case errors.As(serr, &caseErr):
|
|
|
|
if caseErr.Real != source.Name {
|
|
|
|
err = serr
|
|
|
|
break
|
|
|
|
}
|
|
|
|
fallthrough // This is a case only rename
|
|
|
|
case fs.IsNotExist(serr):
|
|
|
|
if !ok || curTarget.IsDeleted() {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
scanChan <- target.Name
|
|
|
|
err = errModified
|
|
|
|
default:
|
|
|
|
// We can't check whether the file changed as compared to the db,
|
|
|
|
// do not delete.
|
|
|
|
err = serr
|
|
|
|
}
|
2018-09-16 07:48:14 +00:00
|
|
|
case !ok:
|
|
|
|
// Target appeared from nowhere
|
|
|
|
scanChan <- target.Name
|
|
|
|
err = errModified
|
|
|
|
default:
|
2019-02-02 09:11:42 +00:00
|
|
|
var fi protocol.FileInfo
|
2020-12-27 21:26:25 +00:00
|
|
|
if fi, err = scanner.CreateFileInfo(stat, target.Name, f.mtimefs); err == nil {
|
2020-08-25 06:11:14 +00:00
|
|
|
if !fi.IsEquivalentOptional(curTarget, f.modTimeWindow, f.IgnorePerms, true, protocol.LocalAllFlags) {
|
2018-09-16 07:48:14 +00:00
|
|
|
// Target changed
|
|
|
|
scanChan <- target.Name
|
|
|
|
err = errModified
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if err != nil {
|
2018-10-10 09:37:20 +00:00
|
|
|
return err
|
2018-09-16 07:48:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
tempName := fs.TempName(target.Name)
|
|
|
|
|
2016-04-26 14:01:46 +00:00
|
|
|
if f.versioner != nil {
|
2020-10-02 13:22:28 +00:00
|
|
|
err = f.CheckAvailableSpace(uint64(source.Size))
|
2014-12-19 23:12:12 +00:00
|
|
|
if err == nil {
|
2020-12-27 21:26:25 +00:00
|
|
|
err = osutil.Copy(f.CopyRangeMethod, f.mtimefs, f.mtimefs, source.Name, tempName)
|
2018-08-25 08:16:38 +00:00
|
|
|
if err == nil {
|
2019-07-31 08:53:35 +00:00
|
|
|
err = f.inWritableDir(f.versioner.Archive, source.Name)
|
2018-08-25 08:16:38 +00:00
|
|
|
}
|
2014-12-19 23:12:12 +00:00
|
|
|
}
|
|
|
|
} else {
|
2020-12-27 21:26:25 +00:00
|
|
|
err = osutil.RenameOrCopy(f.CopyRangeMethod, f.mtimefs, f.mtimefs, source.Name, tempName)
|
2014-12-19 23:12:12 +00:00
|
|
|
}
|
2018-10-10 09:37:20 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2014-12-19 23:12:12 +00:00
|
|
|
|
2018-10-10 09:37:20 +00:00
|
|
|
blockStatsMut.Lock()
|
2019-10-04 11:03:13 +00:00
|
|
|
minBlocksPerBlock := target.BlockSize() / protocol.MinBlockSize
|
|
|
|
blockStats["total"] += len(target.Blocks) * minBlocksPerBlock
|
|
|
|
blockStats["renamed"] += len(target.Blocks) * minBlocksPerBlock
|
2018-10-10 09:37:20 +00:00
|
|
|
blockStatsMut.Unlock()
|
2014-12-19 23:12:12 +00:00
|
|
|
|
2018-10-10 09:37:20 +00:00
|
|
|
// The file was renamed, so we have handled both the necessary delete
|
|
|
|
// of the source and the creation of the target temp file. Fix-up the metadata,
|
|
|
|
// update the local index of the target file and rename from temp to real name.
|
2015-03-01 09:46:28 +00:00
|
|
|
|
2020-01-21 17:23:08 +00:00
|
|
|
if err = f.performFinish(target, curTarget, true, tempName, snap, dbUpdateChan, scanChan); err != nil {
|
2018-10-10 09:37:20 +00:00
|
|
|
return err
|
|
|
|
}
|
2014-12-19 23:12:12 +00:00
|
|
|
|
2018-10-10 09:37:20 +00:00
|
|
|
dbUpdateChan <- dbUpdateJob{source, dbUpdateDeleteFile}
|
2015-03-01 09:46:28 +00:00
|
|
|
|
2018-10-10 09:37:20 +00:00
|
|
|
return nil
|
2014-12-19 23:12:12 +00:00
|
|
|
}
|
|
|
|
|
2015-05-27 09:14:39 +00:00
|
|
|
// This is the flow of data and events here, I think...
|
|
|
|
//
|
|
|
|
// +-----------------------+
|
|
|
|
// | | - - - - > ItemStarted
|
|
|
|
// | handleFile | - - - - > ItemFinished (on shortcuts)
|
|
|
|
// | |
|
|
|
|
// +-----------------------+
|
|
|
|
// |
|
|
|
|
// | copyChan (copyBlocksState; unless shortcut taken)
|
|
|
|
// |
|
|
|
|
// | +-----------------------+
|
|
|
|
// | | +-----------------------+
|
|
|
|
// +--->| | |
|
|
|
|
// | | copierRoutine |
|
|
|
|
// +-| |
|
|
|
|
// +-----------------------+
|
|
|
|
// |
|
|
|
|
// | pullChan (sharedPullerState)
|
|
|
|
// |
|
|
|
|
// | +-----------------------+
|
|
|
|
// | | +-----------------------+
|
|
|
|
// +-->| | |
|
|
|
|
// | | pullerRoutine |
|
|
|
|
// +-| |
|
|
|
|
// +-----------------------+
|
|
|
|
// |
|
|
|
|
// | finisherChan (sharedPullerState)
|
|
|
|
// |
|
|
|
|
// | +-----------------------+
|
|
|
|
// | | |
|
|
|
|
// +-->| finisherRoutine | - - - - > ItemFinished
|
|
|
|
// | |
|
|
|
|
// +-----------------------+
|
|
|
|
|
2014-09-27 12:44:15 +00:00
|
|
|
// handleFile queues the copies and pulls as necessary for a single new or
|
|
|
|
// changed file.
|
2020-01-21 17:23:08 +00:00
|
|
|
func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, snap *db.Snapshot, copyChan chan<- copyBlocksState) {
|
|
|
|
curFile, hasCurFile := snap.Get(protocol.LocalDeviceID, file.Name)
|
2014-03-28 13:36:57 +00:00
|
|
|
|
2019-03-27 08:36:58 +00:00
|
|
|
have, _ := blockDiff(curFile.Blocks, file.Blocks)
|
2014-03-28 13:36:57 +00:00
|
|
|
|
2017-09-02 05:52:38 +00:00
|
|
|
tempName := fs.TempName(file.Name)
|
2015-10-29 08:08:03 +00:00
|
|
|
|
2018-01-14 14:30:11 +00:00
|
|
|
populateOffsets(file.Blocks)
|
2014-10-17 22:16:29 +00:00
|
|
|
|
2018-01-14 21:52:41 +00:00
|
|
|
blocks := make([]protocol.BlockInfo, 0, len(file.Blocks))
|
2020-10-02 06:07:05 +00:00
|
|
|
reused := make([]int, 0, len(file.Blocks))
|
2014-10-03 22:15:54 +00:00
|
|
|
|
|
|
|
// Check for an old temporary file which might have some blocks we could
|
|
|
|
// reuse.
|
2020-12-27 21:26:25 +00:00
|
|
|
tempBlocks, err := scanner.HashFile(f.ctx, f.mtimefs, tempName, file.BlockSize(), nil, false)
|
2020-09-09 09:47:14 +00:00
|
|
|
if err != nil {
|
|
|
|
var caseErr *fs.ErrCaseConflict
|
|
|
|
if errors.As(err, &caseErr) {
|
2020-12-27 21:26:25 +00:00
|
|
|
if rerr := f.mtimefs.Rename(caseErr.Real, tempName); rerr == nil {
|
|
|
|
tempBlocks, err = scanner.HashFile(f.ctx, f.mtimefs, tempName, file.BlockSize(), nil, false)
|
2020-09-09 09:47:14 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2014-10-03 22:15:54 +00:00
|
|
|
if err == nil {
|
|
|
|
// Check for any reusable blocks in the temp file
|
2018-01-14 14:30:11 +00:00
|
|
|
tempCopyBlocks, _ := blockDiff(tempBlocks, file.Blocks)
|
2014-10-03 22:15:54 +00:00
|
|
|
|
|
|
|
// block.String() returns a string unique to the block
|
2015-01-14 23:00:00 +00:00
|
|
|
existingBlocks := make(map[string]struct{}, len(tempCopyBlocks))
|
2014-10-03 22:15:54 +00:00
|
|
|
for _, block := range tempCopyBlocks {
|
2015-01-14 23:00:00 +00:00
|
|
|
existingBlocks[block.String()] = struct{}{}
|
2014-10-03 22:15:54 +00:00
|
|
|
}
|
|
|
|
|
2014-10-08 22:41:23 +00:00
|
|
|
// Since the blocks are already there, we don't need to get them.
|
2016-04-15 10:59:41 +00:00
|
|
|
for i, block := range file.Blocks {
|
2014-10-03 22:15:54 +00:00
|
|
|
_, ok := existingBlocks[block.String()]
|
|
|
|
if !ok {
|
2014-10-08 22:41:23 +00:00
|
|
|
blocks = append(blocks, block)
|
2016-04-15 10:59:41 +00:00
|
|
|
} else {
|
2020-10-02 06:07:05 +00:00
|
|
|
reused = append(reused, i)
|
2014-10-03 22:15:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-10-12 20:38:22 +00:00
|
|
|
// The sharedpullerstate will know which flags to use when opening the
|
|
|
|
// temp file depending if we are reusing any blocks or not.
|
2016-04-15 10:59:41 +00:00
|
|
|
if len(reused) == 0 {
|
2014-10-03 22:15:54 +00:00
|
|
|
// Otherwise, discard the file ourselves in order for the
|
2015-04-28 15:34:55 +00:00
|
|
|
// sharedpuller not to panic when it fails to exclusively create a
|
2014-10-03 22:15:54 +00:00
|
|
|
// file which already exists
|
2020-12-27 21:26:25 +00:00
|
|
|
f.inWritableDir(f.mtimefs.Remove, tempName)
|
2014-10-03 22:15:54 +00:00
|
|
|
}
|
2014-10-08 22:41:23 +00:00
|
|
|
} else {
|
2016-04-15 10:59:41 +00:00
|
|
|
// Copy the blocks, as we don't want to shuffle them on the FileInfo
|
|
|
|
blocks = append(blocks, file.Blocks...)
|
2015-12-21 18:29:18 +00:00
|
|
|
}
|
|
|
|
|
2020-05-11 21:44:04 +00:00
|
|
|
// Reorder blocks
|
|
|
|
blocks = f.blockPullReorderer.Reorder(blocks)
|
2016-04-15 10:59:41 +00:00
|
|
|
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.ItemStarted, map[string]string{
|
2016-04-26 14:01:46 +00:00
|
|
|
"folder": f.folderID,
|
2015-12-21 18:29:18 +00:00
|
|
|
"item": file.Name,
|
|
|
|
"type": "file",
|
|
|
|
"action": "update",
|
|
|
|
})
|
|
|
|
|
2020-12-27 21:26:25 +00:00
|
|
|
s := newSharedPullerState(file, f.mtimefs, f.folderID, tempName, blocks, reused, f.IgnorePerms || file.NoPermissions, hasCurFile, curFile, !f.DisableSparseFiles, !f.DisableFsync)
|
2014-03-28 13:36:57 +00:00
|
|
|
|
2016-12-14 23:30:29 +00:00
|
|
|
l.Debugf("%v need file %s; copy %d, reused %v", f, file.Name, len(blocks), len(reused))
|
2014-03-28 13:36:57 +00:00
|
|
|
|
2014-10-08 22:41:23 +00:00
|
|
|
cs := copyBlocksState{
|
2020-04-21 17:55:14 +00:00
|
|
|
sharedPullerState: s,
|
2014-10-08 22:41:23 +00:00
|
|
|
blocks: blocks,
|
2017-01-04 21:04:13 +00:00
|
|
|
have: len(have),
|
2014-10-06 08:14:36 +00:00
|
|
|
}
|
2014-10-08 22:41:23 +00:00
|
|
|
copyChan <- cs
|
2014-03-28 13:36:57 +00:00
|
|
|
}
|
|
|
|
|
2018-01-14 14:30:11 +00:00
|
|
|
// blockDiff returns lists of common and missing (to transform src into tgt)
|
|
|
|
// blocks. Both block lists must have been created with the same block size.
|
2018-01-14 21:52:41 +00:00
|
|
|
func blockDiff(src, tgt []protocol.BlockInfo) ([]protocol.BlockInfo, []protocol.BlockInfo) {
|
|
|
|
if len(tgt) == 0 {
|
2018-01-14 14:30:11 +00:00
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
2018-01-14 21:52:41 +00:00
|
|
|
if len(src) == 0 {
|
2018-01-14 14:30:11 +00:00
|
|
|
// Copy the entire file
|
|
|
|
return nil, tgt
|
|
|
|
}
|
|
|
|
|
2018-01-14 21:52:41 +00:00
|
|
|
have := make([]protocol.BlockInfo, 0, len(src))
|
|
|
|
need := make([]protocol.BlockInfo, 0, len(tgt))
|
|
|
|
|
2018-01-14 14:30:11 +00:00
|
|
|
for i := range tgt {
|
2018-01-14 21:52:41 +00:00
|
|
|
if i >= len(src) {
|
|
|
|
return have, append(need, tgt[i:]...)
|
|
|
|
}
|
|
|
|
if !bytes.Equal(tgt[i].Hash, src[i].Hash) {
|
2018-01-14 14:30:11 +00:00
|
|
|
// Copy differing block
|
|
|
|
need = append(need, tgt[i])
|
|
|
|
} else {
|
|
|
|
have = append(have, tgt[i])
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return have, need
|
|
|
|
}
|
|
|
|
|
|
|
|
// populateOffsets sets the Offset field on each block
|
|
|
|
func populateOffsets(blocks []protocol.BlockInfo) {
|
|
|
|
var offset int64
|
|
|
|
for i := range blocks {
|
|
|
|
blocks[i].Offset = offset
|
|
|
|
offset += int64(blocks[i].Size)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-09-27 12:44:15 +00:00
|
|
|
// shortcutFile sets file mode and modification time, when that's the only
|
|
|
|
// thing that has changed.
|
2018-09-16 09:29:06 +00:00
|
|
|
func (f *sendReceiveFolder) shortcutFile(file, curFile protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob) {
|
|
|
|
l.Debugln(f, "taking shortcut on", file.Name)
|
|
|
|
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.ItemStarted, map[string]string{
|
2018-09-16 09:29:06 +00:00
|
|
|
"folder": f.folderID,
|
|
|
|
"item": file.Name,
|
|
|
|
"type": "file",
|
|
|
|
"action": "metadata",
|
|
|
|
})
|
|
|
|
|
|
|
|
var err error
|
2019-08-15 14:29:37 +00:00
|
|
|
defer f.evLogger.Log(events.ItemFinished, map[string]interface{}{
|
2018-09-16 09:29:06 +00:00
|
|
|
"folder": f.folderID,
|
|
|
|
"item": file.Name,
|
|
|
|
"error": events.Error(err),
|
|
|
|
"type": "file",
|
|
|
|
"action": "metadata",
|
|
|
|
})
|
|
|
|
|
|
|
|
f.queue.Done(file.Name)
|
|
|
|
|
2018-02-25 08:39:00 +00:00
|
|
|
if !f.IgnorePerms && !file.NoPermissions {
|
2020-12-27 21:26:25 +00:00
|
|
|
if err = f.mtimefs.Chmod(file.Name, fs.FileMode(file.Permissions&0777)); err != nil {
|
2019-03-04 13:01:52 +00:00
|
|
|
f.newPullError(file.Name, err)
|
2018-09-16 09:29:06 +00:00
|
|
|
return
|
2014-10-09 22:34:32 +00:00
|
|
|
}
|
2014-04-27 10:14:53 +00:00
|
|
|
}
|
2014-03-28 13:36:57 +00:00
|
|
|
|
2020-12-27 21:26:25 +00:00
|
|
|
f.mtimefs.Chtimes(file.Name, file.ModTime(), file.ModTime()) // never fails
|
2014-03-28 13:36:57 +00:00
|
|
|
|
2018-09-16 09:29:06 +00:00
|
|
|
dbUpdateChan <- dbUpdateJob{file, dbUpdateShortcutFile}
|
2014-03-28 13:36:57 +00:00
|
|
|
}
|
|
|
|
|
2014-10-08 22:41:23 +00:00
|
|
|
// copierRoutine reads copierStates until the in channel closes and performs
|
|
|
|
// the relevant copies when possible, or passes it to the puller routine.
|
2016-12-16 22:23:35 +00:00
|
|
|
func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState) {
|
2018-11-13 07:53:55 +00:00
|
|
|
buf := protocol.BufferPool.Get(protocol.MinBlockSize)
|
|
|
|
defer func() {
|
|
|
|
protocol.BufferPool.Put(buf)
|
|
|
|
}()
|
2014-03-28 13:36:57 +00:00
|
|
|
|
2020-05-11 18:15:11 +00:00
|
|
|
folderFilesystems := make(map[string]fs.Filesystem)
|
|
|
|
// Hope that it's usually in the same folder, so start with that one.
|
|
|
|
folders := []string{f.folderID}
|
|
|
|
for folder, cfg := range f.model.cfg.Folders() {
|
|
|
|
folderFilesystems[folder] = cfg.Filesystem()
|
|
|
|
if folder != f.folderID {
|
|
|
|
folders = append(folders, folder)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-09-27 12:44:15 +00:00
|
|
|
for state := range in {
|
2020-10-02 13:22:28 +00:00
|
|
|
if err := f.CheckAvailableSpace(uint64(state.file.Size)); err != nil {
|
2019-08-16 07:40:53 +00:00
|
|
|
state.fail(err)
|
|
|
|
// Nothing more to do for this failed file, since it would use to much disk space
|
|
|
|
out <- state.sharedPullerState
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2014-09-27 12:44:15 +00:00
|
|
|
dstFd, err := state.tempFile()
|
|
|
|
if err != nil {
|
2015-05-27 09:14:39 +00:00
|
|
|
// Nothing more to do for this failed file, since we couldn't create a temporary for it.
|
2015-01-07 23:12:12 +00:00
|
|
|
out <- state.sharedPullerState
|
|
|
|
continue
|
2014-11-16 23:18:59 +00:00
|
|
|
}
|
|
|
|
|
2020-11-09 14:33:32 +00:00
|
|
|
if f.Type != config.FolderTypeReceiveEncrypted {
|
|
|
|
f.model.progressEmitter.Register(state.sharedPullerState)
|
2018-05-05 08:24:44 +00:00
|
|
|
}
|
2016-12-14 23:30:29 +00:00
|
|
|
|
2020-11-09 14:33:32 +00:00
|
|
|
weakHashFinder, file := f.initWeakHashFinder(state)
|
2016-12-14 23:30:29 +00:00
|
|
|
|
2018-07-04 07:07:33 +00:00
|
|
|
blocks:
|
2014-09-27 12:44:15 +00:00
|
|
|
for _, block := range state.blocks {
|
2018-07-04 07:07:33 +00:00
|
|
|
select {
|
|
|
|
case <-f.ctx.Done():
|
2019-03-04 13:01:52 +00:00
|
|
|
state.fail(errors.Wrap(f.ctx.Err(), "folder stopped"))
|
2018-07-04 07:07:33 +00:00
|
|
|
break blocks
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
2016-12-21 11:23:20 +00:00
|
|
|
if !f.DisableSparseFiles && state.reused == 0 && block.IsEmpty() {
|
2015-11-21 15:30:53 +00:00
|
|
|
// The block is a block of all zeroes, and we are not reusing
|
|
|
|
// a temp file, so there is no need to do anything with it.
|
|
|
|
// If we were reusing a temp file and had this block to copy,
|
|
|
|
// it would be because the block in the temp file was *not* a
|
|
|
|
// block of all zeroes, so then we should not skip it.
|
|
|
|
|
|
|
|
// Pretend we copied it.
|
|
|
|
state.copiedFromOrigin()
|
2018-01-11 10:36:35 +00:00
|
|
|
state.copyDone(block)
|
2015-11-21 15:30:53 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2018-11-13 07:53:55 +00:00
|
|
|
buf = protocol.BufferPool.Upgrade(buf, int(block.Size))
|
2014-10-08 22:41:23 +00:00
|
|
|
|
2020-11-09 14:33:32 +00:00
|
|
|
var found bool
|
|
|
|
if f.Type != config.FolderTypeReceiveEncrypted {
|
|
|
|
found, err = weakHashFinder.Iterate(block.WeakHash, buf, func(offset int64) bool {
|
|
|
|
if f.verifyBuffer(buf, block) != nil {
|
|
|
|
return true
|
|
|
|
}
|
2014-10-24 22:20:08 +00:00
|
|
|
|
2020-11-09 14:33:32 +00:00
|
|
|
err = f.limitedWriteAt(dstFd, buf, block.Offset)
|
|
|
|
if err != nil {
|
|
|
|
state.fail(errors.Wrap(err, "dst write"))
|
|
|
|
}
|
|
|
|
if offset == block.Offset {
|
|
|
|
state.copiedFromOrigin()
|
|
|
|
} else {
|
|
|
|
state.copiedFromOriginShifted()
|
|
|
|
}
|
|
|
|
|
|
|
|
return false
|
|
|
|
})
|
2014-10-08 22:41:23 +00:00
|
|
|
if err != nil {
|
2020-11-09 14:33:32 +00:00
|
|
|
l.Debugln("weak hasher iter", err)
|
2014-10-08 22:41:23 +00:00
|
|
|
}
|
2016-12-14 23:30:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if !found {
|
2017-08-19 14:36:56 +00:00
|
|
|
found = f.model.finder.Iterate(folders, block.Hash, func(folder, path string, index int32) bool {
|
2020-06-18 06:15:47 +00:00
|
|
|
ffs := folderFilesystems[folder]
|
|
|
|
fd, err := ffs.Open(path)
|
2016-12-14 23:30:29 +00:00
|
|
|
if err != nil {
|
|
|
|
return false
|
|
|
|
}
|
2020-08-03 19:54:42 +00:00
|
|
|
defer fd.Close()
|
2016-12-14 23:30:29 +00:00
|
|
|
|
2020-06-18 06:15:47 +00:00
|
|
|
srcOffset := int64(state.file.BlockSize()) * int64(index)
|
|
|
|
_, err = fd.ReadAt(buf, srcOffset)
|
2016-12-14 23:30:29 +00:00
|
|
|
if err != nil {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2020-11-09 14:33:32 +00:00
|
|
|
// Hash is not SHA256 as it's an encrypted hash token. In that
|
|
|
|
// case we can't verify the block integrity so we'll take it on
|
|
|
|
// trust. (The other side can and will verify.)
|
|
|
|
if f.Type != config.FolderTypeReceiveEncrypted {
|
|
|
|
if err := f.verifyBuffer(buf, block); err != nil {
|
|
|
|
l.Debugln("Finder failed to verify buffer", err)
|
|
|
|
return false
|
|
|
|
}
|
2016-12-14 23:30:29 +00:00
|
|
|
}
|
|
|
|
|
2020-06-18 06:15:47 +00:00
|
|
|
if f.CopyRangeMethod != fs.CopyRangeMethodStandard {
|
|
|
|
err = f.withLimiter(func() error {
|
|
|
|
dstFd.mut.Lock()
|
|
|
|
defer dstFd.mut.Unlock()
|
|
|
|
return fs.CopyRange(f.CopyRangeMethod, fd, dstFd.fd, srcOffset, block.Offset, int64(block.Size))
|
|
|
|
})
|
|
|
|
} else {
|
|
|
|
err = f.limitedWriteAt(dstFd, buf, block.Offset)
|
|
|
|
}
|
2016-12-14 23:30:29 +00:00
|
|
|
if err != nil {
|
2019-03-04 13:01:52 +00:00
|
|
|
state.fail(errors.Wrap(err, "dst write"))
|
2016-12-14 23:30:29 +00:00
|
|
|
}
|
2017-08-19 14:36:56 +00:00
|
|
|
if path == state.file.Name {
|
2016-12-14 23:30:29 +00:00
|
|
|
state.copiedFromOrigin()
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
})
|
|
|
|
}
|
2014-10-08 22:41:23 +00:00
|
|
|
|
|
|
|
if state.failed() != nil {
|
|
|
|
break
|
2014-08-27 05:00:15 +00:00
|
|
|
}
|
2014-09-27 12:44:15 +00:00
|
|
|
|
2014-10-24 22:20:08 +00:00
|
|
|
if !found {
|
2014-10-08 22:41:23 +00:00
|
|
|
state.pullStarted()
|
|
|
|
ps := pullBlockState{
|
|
|
|
sharedPullerState: state.sharedPullerState,
|
|
|
|
block: block,
|
|
|
|
}
|
|
|
|
pullChan <- ps
|
2014-10-12 20:38:22 +00:00
|
|
|
} else {
|
2016-04-15 10:59:41 +00:00
|
|
|
state.copyDone(block)
|
2014-05-25 18:49:08 +00:00
|
|
|
}
|
2014-05-19 21:42:08 +00:00
|
|
|
}
|
2017-08-19 14:36:56 +00:00
|
|
|
if file != nil {
|
|
|
|
// os.File used to return invalid argument if nil.
|
|
|
|
// fs.File panics as it's an interface.
|
|
|
|
file.Close()
|
|
|
|
}
|
|
|
|
|
2014-09-27 12:44:15 +00:00
|
|
|
out <- state.sharedPullerState
|
2014-03-28 13:36:57 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-11-09 14:33:32 +00:00
|
|
|
func (f *sendReceiveFolder) initWeakHashFinder(state copyBlocksState) (*weakhash.Finder, fs.File) {
|
|
|
|
if f.Type == config.FolderTypeReceiveEncrypted {
|
|
|
|
l.Debugln("not weak hashing due to folder type", f.Type)
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
blocksPercentChanged := 0
|
|
|
|
if tot := len(state.file.Blocks); tot > 0 {
|
|
|
|
blocksPercentChanged = (tot - state.have) * 100 / tot
|
|
|
|
}
|
|
|
|
|
|
|
|
if blocksPercentChanged < f.WeakHashThresholdPct {
|
|
|
|
l.Debugf("not weak hashing %s. not enough changed %.02f < %d", state.file.Name, blocksPercentChanged, f.WeakHashThresholdPct)
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
hashesToFind := make([]uint32, 0, len(state.blocks))
|
|
|
|
for _, block := range state.blocks {
|
|
|
|
if block.WeakHash != 0 {
|
|
|
|
hashesToFind = append(hashesToFind, block.WeakHash)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(hashesToFind) == 0 {
|
|
|
|
l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name)
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
2020-12-27 21:26:25 +00:00
|
|
|
file, err := f.mtimefs.Open(state.file.Name)
|
2020-11-09 14:33:32 +00:00
|
|
|
if err != nil {
|
|
|
|
l.Debugln("weak hasher", err)
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
weakHashFinder, err := weakhash.NewFinder(f.ctx, file, state.file.BlockSize(), hashesToFind)
|
|
|
|
if err != nil {
|
|
|
|
l.Debugln("weak hasher", err)
|
|
|
|
return nil, file
|
|
|
|
}
|
|
|
|
return weakHashFinder, file
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *sendReceiveFolder) verifyBuffer(buf []byte, block protocol.BlockInfo) error {
|
2018-01-14 14:30:11 +00:00
|
|
|
if len(buf) != int(block.Size) {
|
2018-05-05 08:24:44 +00:00
|
|
|
return fmt.Errorf("length mismatch %d != %d", len(buf), block.Size)
|
2018-01-14 14:30:11 +00:00
|
|
|
}
|
|
|
|
|
2020-03-31 12:32:24 +00:00
|
|
|
hash := sha256.Sum256(buf)
|
|
|
|
if !bytes.Equal(hash[:], block.Hash) {
|
2018-05-05 08:24:44 +00:00
|
|
|
return fmt.Errorf("hash mismatch %x != %x", hash, block.Hash)
|
2018-01-14 14:30:11 +00:00
|
|
|
}
|
|
|
|
|
2018-05-05 08:24:44 +00:00
|
|
|
return nil
|
2018-01-14 14:30:11 +00:00
|
|
|
}
|
|
|
|
|
2016-12-16 22:23:35 +00:00
|
|
|
func (f *sendReceiveFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPullerState) {
|
2018-02-25 09:14:02 +00:00
|
|
|
requestLimiter := newByteSemaphore(f.PullerMaxPendingKiB * 1024)
|
|
|
|
wg := sync.NewWaitGroup()
|
|
|
|
|
2014-09-27 12:44:15 +00:00
|
|
|
for state := range in {
|
|
|
|
if state.failed() != nil {
|
2015-05-27 09:14:39 +00:00
|
|
|
out <- state.sharedPullerState
|
2014-12-28 23:11:32 +00:00
|
|
|
continue
|
2014-09-27 12:44:15 +00:00
|
|
|
}
|
2014-07-24 07:38:16 +00:00
|
|
|
|
2019-10-16 07:08:54 +00:00
|
|
|
f.setState(FolderSyncing) // Does nothing if already FolderSyncing
|
|
|
|
|
2018-02-25 09:14:02 +00:00
|
|
|
// The requestLimiter limits how many pending block requests we have
|
|
|
|
// ongoing at any given time, based on the size of the blocks
|
|
|
|
// themselves.
|
2014-08-05 07:46:11 +00:00
|
|
|
|
2018-02-25 09:14:02 +00:00
|
|
|
state := state
|
|
|
|
bytes := int(state.block.Size)
|
2015-11-21 15:30:53 +00:00
|
|
|
|
2020-02-24 20:57:15 +00:00
|
|
|
if err := requestLimiter.takeWithContext(f.ctx, bytes); err != nil {
|
2020-04-21 17:55:14 +00:00
|
|
|
state.fail(err)
|
|
|
|
out <- state.sharedPullerState
|
|
|
|
continue
|
2020-02-24 20:57:15 +00:00
|
|
|
}
|
|
|
|
|
2018-02-25 09:14:02 +00:00
|
|
|
wg.Add(1)
|
2014-08-05 07:46:11 +00:00
|
|
|
|
2018-02-25 09:14:02 +00:00
|
|
|
go func() {
|
|
|
|
defer wg.Done()
|
|
|
|
defer requestLimiter.give(bytes)
|
2014-07-24 07:38:16 +00:00
|
|
|
|
2018-02-25 09:14:02 +00:00
|
|
|
f.pullBlock(state, out)
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
}
|
2014-12-28 23:11:32 +00:00
|
|
|
|
2018-02-25 09:14:02 +00:00
|
|
|
func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPullerState) {
|
|
|
|
// Get an fd to the temporary file. Technically we don't need it until
|
|
|
|
// after fetching the block, but if we run into an error here there is
|
|
|
|
// no point in issuing the request to the network.
|
|
|
|
fd, err := state.tempFile()
|
|
|
|
if err != nil {
|
|
|
|
out <- state.sharedPullerState
|
|
|
|
return
|
|
|
|
}
|
2014-12-28 23:11:32 +00:00
|
|
|
|
2018-02-25 09:14:02 +00:00
|
|
|
if !f.DisableSparseFiles && state.reused == 0 && state.block.IsEmpty() {
|
|
|
|
// There is no need to request a block of all zeroes. Pretend we
|
|
|
|
// requested it and handled it correctly.
|
|
|
|
state.pullDone(state.block)
|
|
|
|
out <- state.sharedPullerState
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
var lastError error
|
2018-04-16 18:08:50 +00:00
|
|
|
candidates := f.model.Availability(f.folderID, state.file, state.block)
|
2018-02-25 09:14:02 +00:00
|
|
|
for {
|
2018-07-04 07:07:33 +00:00
|
|
|
select {
|
|
|
|
case <-f.ctx.Done():
|
2019-03-04 13:01:52 +00:00
|
|
|
state.fail(errors.Wrap(f.ctx.Err(), "folder stopped"))
|
2019-11-19 08:56:53 +00:00
|
|
|
break
|
2018-07-04 07:07:33 +00:00
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
2018-02-25 09:14:02 +00:00
|
|
|
// Select the least busy device to pull the block from. If we found no
|
|
|
|
// feasible device at all, fail the block (and in the long run, the
|
|
|
|
// file).
|
|
|
|
selected, found := activity.leastBusy(candidates)
|
|
|
|
if !found {
|
|
|
|
if lastError != nil {
|
2019-03-04 13:01:52 +00:00
|
|
|
state.fail(errors.Wrap(lastError, "pull"))
|
2014-12-28 23:11:32 +00:00
|
|
|
} else {
|
2019-03-04 13:01:52 +00:00
|
|
|
state.fail(errors.Wrap(errNoDevice, "pull"))
|
2014-12-28 23:11:32 +00:00
|
|
|
}
|
|
|
|
break
|
|
|
|
}
|
2018-02-25 09:14:02 +00:00
|
|
|
|
|
|
|
candidates = removeAvailability(candidates, selected)
|
|
|
|
|
|
|
|
// Fetch the block, while marking the selected device as in use so that
|
|
|
|
// leastBusy can select another device when someone else asks.
|
|
|
|
activity.using(selected)
|
2019-01-24 07:18:55 +00:00
|
|
|
var buf []byte
|
2020-11-09 14:33:32 +00:00
|
|
|
blockNo := int(state.block.Offset / int64(state.file.BlockSize()))
|
|
|
|
buf, lastError = f.model.requestGlobal(f.ctx, selected.ID, f.folderID, state.file.Name, blockNo, state.block.Offset, int(state.block.Size), state.block.Hash, state.block.WeakHash, selected.FromTemporary)
|
2018-02-25 09:14:02 +00:00
|
|
|
activity.done(selected)
|
|
|
|
if lastError != nil {
|
|
|
|
l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, "returned error:", lastError)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
// Verify that the received block matches the desired hash, if not
|
|
|
|
// try pulling it from another device.
|
2020-11-09 14:33:32 +00:00
|
|
|
// For receive-only folders, the hash is not SHA256 as it's an
|
|
|
|
// encrypted hash token. In that case we can't verify the block
|
|
|
|
// integrity so we'll take it on trust. (The other side can and
|
|
|
|
// will verify.)
|
|
|
|
if f.Type != config.FolderTypeReceiveEncrypted {
|
|
|
|
lastError = f.verifyBuffer(buf, state.block)
|
|
|
|
}
|
2018-02-25 09:14:02 +00:00
|
|
|
if lastError != nil {
|
|
|
|
l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, "hash mismatch")
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
// Save the block data we got from the cluster
|
2020-06-18 06:15:47 +00:00
|
|
|
err = f.limitedWriteAt(fd, buf, state.block.Offset)
|
2018-02-25 09:14:02 +00:00
|
|
|
if err != nil {
|
2019-03-04 13:01:52 +00:00
|
|
|
state.fail(errors.Wrap(err, "save"))
|
2018-02-25 09:14:02 +00:00
|
|
|
} else {
|
|
|
|
state.pullDone(state.block)
|
|
|
|
}
|
|
|
|
break
|
2014-07-24 07:38:16 +00:00
|
|
|
}
|
2018-02-25 09:14:02 +00:00
|
|
|
out <- state.sharedPullerState
|
2014-03-28 13:36:57 +00:00
|
|
|
}
|
2014-04-27 10:14:53 +00:00
|
|
|
|
2020-01-21 17:23:08 +00:00
|
|
|
func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCurFile bool, tempName string, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error {
|
2014-11-29 22:18:56 +00:00
|
|
|
// Set the correct permission bits on the new file
|
2018-09-16 07:48:14 +00:00
|
|
|
if !f.IgnorePerms && !file.NoPermissions {
|
2020-12-27 21:26:25 +00:00
|
|
|
if err := f.mtimefs.Chmod(tempName, fs.FileMode(file.Permissions&0777)); err != nil {
|
2015-05-27 09:14:39 +00:00
|
|
|
return err
|
2014-11-16 23:18:59 +00:00
|
|
|
}
|
2014-11-29 22:18:56 +00:00
|
|
|
}
|
2014-11-16 23:18:59 +00:00
|
|
|
|
2019-01-25 08:52:21 +00:00
|
|
|
// Copy the parent owner and group, if we are supposed to do that.
|
|
|
|
if err := f.maybeCopyOwner(tempName); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2020-12-27 21:26:25 +00:00
|
|
|
if stat, err := f.mtimefs.Lstat(file.Name); err == nil {
|
2015-08-08 10:44:17 +00:00
|
|
|
// There is an old file or directory already in place. We need to
|
|
|
|
// handle that.
|
|
|
|
|
2020-06-16 13:20:08 +00:00
|
|
|
if err := f.scanIfItemChanged(file.Name, stat, curFile, hasCurFile, scanChan); err != nil {
|
2019-03-07 14:15:14 +00:00
|
|
|
return err
|
2017-11-13 15:16:27 +00:00
|
|
|
}
|
|
|
|
|
2019-03-07 14:15:14 +00:00
|
|
|
if !curFile.IsDirectory() && !curFile.IsSymlink() && f.inConflict(curFile.Version, file.Version) {
|
2015-08-08 10:44:17 +00:00
|
|
|
// The new file has been changed in conflict with the existing one. We
|
|
|
|
// should file it away as a conflict instead of just removing or
|
2020-10-19 06:53:19 +00:00
|
|
|
// archiving.
|
2019-03-07 14:15:14 +00:00
|
|
|
// Directories and symlinks aren't checked for conflicts.
|
2015-08-08 10:44:17 +00:00
|
|
|
|
2019-07-31 08:53:35 +00:00
|
|
|
err = f.inWritableDir(func(name string) error {
|
2019-02-12 15:05:20 +00:00
|
|
|
return f.moveForConflict(name, file.ModifiedBy.String(), scanChan)
|
2019-07-31 08:53:35 +00:00
|
|
|
}, curFile.Name)
|
2019-03-07 14:15:14 +00:00
|
|
|
} else {
|
2020-01-21 17:23:08 +00:00
|
|
|
err = f.deleteItemOnDisk(curFile, snap, scanChan)
|
2019-03-07 14:15:14 +00:00
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return err
|
2015-08-08 10:44:17 +00:00
|
|
|
}
|
2020-06-19 21:47:29 +00:00
|
|
|
} else if !fs.IsNotExist(err) {
|
|
|
|
return err
|
2014-11-29 22:18:56 +00:00
|
|
|
}
|
2015-08-08 10:44:17 +00:00
|
|
|
|
2016-05-22 09:06:07 +00:00
|
|
|
// Replace the original content with the new one. If it didn't work,
|
|
|
|
// leave the temp file in place for reuse.
|
2020-12-27 21:26:25 +00:00
|
|
|
if err := osutil.RenameOrCopy(f.CopyRangeMethod, f.mtimefs, f.mtimefs, tempName, file.Name); err != nil {
|
2015-05-27 09:14:39 +00:00
|
|
|
return err
|
2014-11-29 22:18:56 +00:00
|
|
|
}
|
2014-04-27 10:14:53 +00:00
|
|
|
|
2016-08-16 18:22:19 +00:00
|
|
|
// Set the correct timestamp on the new file
|
2020-12-27 21:26:25 +00:00
|
|
|
f.mtimefs.Chtimes(file.Name, file.ModTime(), file.ModTime()) // never fails
|
2016-08-16 18:22:19 +00:00
|
|
|
|
2014-11-29 22:18:56 +00:00
|
|
|
// Record the updated file in the index
|
2018-09-16 07:48:14 +00:00
|
|
|
dbUpdateChan <- dbUpdateJob{file, dbUpdateHandleFile}
|
2015-05-27 09:14:39 +00:00
|
|
|
return nil
|
2014-11-16 23:18:59 +00:00
|
|
|
}
|
2014-11-09 04:26:52 +00:00
|
|
|
|
2020-01-21 17:23:08 +00:00
|
|
|
func (f *sendReceiveFolder) finisherRoutine(snap *db.Snapshot, in <-chan *sharedPullerState, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
|
2014-11-16 23:18:59 +00:00
|
|
|
for state := range in {
|
2014-11-29 22:18:56 +00:00
|
|
|
if closed, err := state.finalClose(); closed {
|
2016-04-26 15:11:19 +00:00
|
|
|
l.Debugln(f, "closing", state.file.Name)
|
2014-11-29 22:18:56 +00:00
|
|
|
|
2016-04-26 15:11:19 +00:00
|
|
|
f.queue.Done(state.file.Name)
|
2015-05-27 09:14:39 +00:00
|
|
|
|
|
|
|
if err == nil {
|
2020-01-21 17:23:08 +00:00
|
|
|
err = f.performFinish(state.file, state.curFile, state.hasCurFile, state.tempName, snap, dbUpdateChan, scanChan)
|
2015-01-07 23:12:12 +00:00
|
|
|
}
|
2015-05-27 09:14:39 +00:00
|
|
|
|
|
|
|
if err != nil {
|
2019-03-04 13:01:52 +00:00
|
|
|
f.newPullError(state.file.Name, err)
|
2017-10-12 06:16:46 +00:00
|
|
|
} else {
|
2019-10-04 11:03:13 +00:00
|
|
|
minBlocksPerBlock := state.file.BlockSize() / protocol.MinBlockSize
|
2017-11-09 21:16:29 +00:00
|
|
|
blockStatsMut.Lock()
|
2019-10-04 11:03:13 +00:00
|
|
|
blockStats["total"] += (state.reused + state.copyTotal + state.pullTotal) * minBlocksPerBlock
|
|
|
|
blockStats["reused"] += state.reused * minBlocksPerBlock
|
|
|
|
blockStats["pulled"] += state.pullTotal * minBlocksPerBlock
|
2017-11-09 21:16:29 +00:00
|
|
|
// copyOriginShifted is counted towards copyOrigin due to progress bar reasons
|
|
|
|
// for reporting reasons we want to separate these.
|
2019-10-04 11:03:13 +00:00
|
|
|
blockStats["copyOrigin"] += (state.copyOrigin - state.copyOriginShifted) * minBlocksPerBlock
|
|
|
|
blockStats["copyOriginShifted"] += state.copyOriginShifted * minBlocksPerBlock
|
|
|
|
blockStats["copyElsewhere"] += (state.copyTotal - state.copyOrigin) * minBlocksPerBlock
|
2017-11-09 21:16:29 +00:00
|
|
|
blockStatsMut.Unlock()
|
2015-05-27 09:14:39 +00:00
|
|
|
}
|
2017-09-23 14:22:26 +00:00
|
|
|
|
2020-11-09 14:33:32 +00:00
|
|
|
if f.Type != config.FolderTypeReceiveEncrypted {
|
|
|
|
f.model.progressEmitter.Deregister(state)
|
|
|
|
}
|
2019-03-26 20:31:33 +00:00
|
|
|
|
2019-08-15 14:29:37 +00:00
|
|
|
f.evLogger.Log(events.ItemFinished, map[string]interface{}{
|
2016-04-26 15:11:19 +00:00
|
|
|
"folder": f.folderID,
|
2015-05-27 09:14:39 +00:00
|
|
|
"item": state.file.Name,
|
|
|
|
"error": events.Error(err),
|
|
|
|
"type": "file",
|
|
|
|
"action": "update",
|
|
|
|
})
|
2014-04-27 10:14:53 +00:00
|
|
|
}
|
|
|
|
}
|
2014-09-27 12:44:15 +00:00
|
|
|
}
|
2014-04-27 10:14:53 +00:00
|
|
|
|
Add job queue (fixes #629)
Request to terminate currently ongoing downloads and jump to the bumped file
incoming in 3, 2, 1.
Also, has a slightly strange effect where we pop a job off the queue, but
the copyChannel is still busy and blocks, though it gets moved to the
progress slice in the jobqueue, and looks like it's in progress which it isn't
as it's waiting to be picked up from the copyChan.
As a result, the progress emitter doesn't register on the task, and hence the file
doesn't have a progress bar, but cannot be replaced by a bump.
I guess I can fix progress bar issue by moving the progressEmiter.Register just
before passing the file to the copyChan, but then we are back to the initial
problem of a file with a progress bar, but no progress happening as it's stuck
on write to copyChan
I checked if there is a way to check for channel writeability (before popping)
but got struck by lightning just for bringing the idea up in #go-nuts.
My ideal scenario would be to check if copyChan is writeable, pop job from the
queue and shove it down handleFile. This way jobs would stay in the queue while
they cannot be handled, meaning that the `Bump` could bring your file up higher.
2014-12-01 19:23:06 +00:00
|
|
|
// Moves the given filename to the front of the job queue
|
2016-12-16 22:23:35 +00:00
|
|
|
func (f *sendReceiveFolder) BringToFront(filename string) {
|
2016-04-26 15:11:19 +00:00
|
|
|
f.queue.BringToFront(filename)
|
Add job queue (fixes #629)
Request to terminate currently ongoing downloads and jump to the bumped file
incoming in 3, 2, 1.
Also, has a slightly strange effect where we pop a job off the queue, but
the copyChannel is still busy and blocks, though it gets moved to the
progress slice in the jobqueue, and looks like it's in progress which it isn't
as it's waiting to be picked up from the copyChan.
As a result, the progress emitter doesn't register on the task, and hence the file
doesn't have a progress bar, but cannot be replaced by a bump.
I guess I can fix progress bar issue by moving the progressEmiter.Register just
before passing the file to the copyChan, but then we are back to the initial
problem of a file with a progress bar, but no progress happening as it's stuck
on write to copyChan
I checked if there is a way to check for channel writeability (before popping)
but got struck by lightning just for bringing the idea up in #go-nuts.
My ideal scenario would be to check if copyChan is writeable, pop job from the
queue and shove it down handleFile. This way jobs would stay in the queue while
they cannot be handled, meaning that the `Bump` could bring your file up higher.
2014-12-01 19:23:06 +00:00
|
|
|
}
|
|
|
|
|
2019-06-27 18:25:38 +00:00
|
|
|
func (f *sendReceiveFolder) Jobs(page, perpage int) ([]string, []string, int) {
|
|
|
|
return f.queue.Jobs(page, perpage)
|
Add job queue (fixes #629)
Request to terminate currently ongoing downloads and jump to the bumped file
incoming in 3, 2, 1.
Also, has a slightly strange effect where we pop a job off the queue, but
the copyChannel is still busy and blocks, though it gets moved to the
progress slice in the jobqueue, and looks like it's in progress which it isn't
as it's waiting to be picked up from the copyChan.
As a result, the progress emitter doesn't register on the task, and hence the file
doesn't have a progress bar, but cannot be replaced by a bump.
I guess I can fix progress bar issue by moving the progressEmiter.Register just
before passing the file to the copyChan, but then we are back to the initial
problem of a file with a progress bar, but no progress happening as it's stuck
on write to copyChan
I checked if there is a way to check for channel writeability (before popping)
but got struck by lightning just for bringing the idea up in #go-nuts.
My ideal scenario would be to check if copyChan is writeable, pop job from the
queue and shove it down handleFile. This way jobs would stay in the queue while
they cannot be handled, meaning that the `Bump` could bring your file up higher.
2014-12-01 19:23:06 +00:00
|
|
|
}
|
|
|
|
|
2015-04-05 13:34:29 +00:00
|
|
|
// dbUpdaterRoutine aggregates db updates and commits them in batches no
|
|
|
|
// larger than 1000 items, and no more delayed than 2 seconds.
|
2017-12-07 08:42:03 +00:00
|
|
|
func (f *sendReceiveFolder) dbUpdaterRoutine(dbUpdateChan <-chan dbUpdateJob) {
|
2017-08-31 08:47:39 +00:00
|
|
|
const maxBatchTime = 2 * time.Second
|
2015-04-05 13:34:29 +00:00
|
|
|
|
2019-03-05 20:32:37 +00:00
|
|
|
batch := newFileInfoBatch(nil)
|
2015-04-05 13:34:29 +00:00
|
|
|
tick := time.NewTicker(maxBatchTime)
|
|
|
|
defer tick.Stop()
|
|
|
|
|
2017-08-19 14:36:56 +00:00
|
|
|
changedDirs := make(map[string]struct{})
|
2019-03-05 20:32:37 +00:00
|
|
|
found := false
|
|
|
|
var lastFile protocol.FileInfo
|
2016-11-21 17:09:29 +00:00
|
|
|
|
2019-03-05 20:32:37 +00:00
|
|
|
batch.flushFn = func(files []protocol.FileInfo) error {
|
2017-08-19 14:36:56 +00:00
|
|
|
// sync directories
|
|
|
|
for dir := range changedDirs {
|
|
|
|
delete(changedDirs, dir)
|
2020-05-01 07:36:46 +00:00
|
|
|
if !f.FolderConfiguration.DisableFsync {
|
2020-12-27 21:26:25 +00:00
|
|
|
fd, err := f.mtimefs.Open(dir)
|
2020-05-01 07:36:46 +00:00
|
|
|
if err != nil {
|
|
|
|
l.Debugf("fsync %q failed: %v", dir, err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if err := fd.Sync(); err != nil {
|
|
|
|
l.Debugf("fsync %q failed: %v", dir, err)
|
|
|
|
}
|
|
|
|
fd.Close()
|
2017-08-19 14:36:56 +00:00
|
|
|
}
|
2016-11-21 17:09:29 +00:00
|
|
|
}
|
|
|
|
|
2016-05-19 00:19:26 +00:00
|
|
|
// All updates to file/folder objects that originated remotely
|
|
|
|
// (across the network) use this call to updateLocals
|
2019-04-07 11:29:17 +00:00
|
|
|
f.updateLocalsFromPulling(files)
|
2015-06-16 11:12:34 +00:00
|
|
|
|
|
|
|
if found {
|
2019-03-11 16:57:21 +00:00
|
|
|
f.ReceivedFile(lastFile.Name, lastFile.IsDeleted())
|
2019-03-05 20:32:37 +00:00
|
|
|
found = false
|
2015-06-16 11:12:34 +00:00
|
|
|
}
|
|
|
|
|
2019-03-05 20:32:37 +00:00
|
|
|
return nil
|
2015-06-16 11:12:34 +00:00
|
|
|
}
|
|
|
|
|
2020-11-25 21:57:25 +00:00
|
|
|
recvEnc := f.Type == config.FolderTypeReceiveEncrypted
|
2015-04-05 13:34:29 +00:00
|
|
|
loop:
|
|
|
|
for {
|
|
|
|
select {
|
2017-12-07 08:42:03 +00:00
|
|
|
case job, ok := <-dbUpdateChan:
|
2015-04-05 13:34:29 +00:00
|
|
|
if !ok {
|
|
|
|
break loop
|
|
|
|
}
|
|
|
|
|
2019-03-05 20:32:37 +00:00
|
|
|
switch job.jobType {
|
|
|
|
case dbUpdateHandleFile, dbUpdateShortcutFile:
|
|
|
|
changedDirs[filepath.Dir(job.file.Name)] = struct{}{}
|
2020-11-25 21:57:25 +00:00
|
|
|
if recvEnc {
|
|
|
|
job.file.Size += encryptionTrailerSize(job.file)
|
|
|
|
}
|
2019-03-05 20:32:37 +00:00
|
|
|
case dbUpdateHandleDir:
|
|
|
|
changedDirs[job.file.Name] = struct{}{}
|
|
|
|
case dbUpdateHandleSymlink, dbUpdateInvalidate:
|
|
|
|
// fsyncing symlinks is only supported by MacOS
|
|
|
|
// and invalidated files are db only changes -> no sync
|
|
|
|
}
|
2015-04-05 13:34:29 +00:00
|
|
|
|
2019-03-05 20:32:37 +00:00
|
|
|
// For some reason we seem to care about file deletions and
|
|
|
|
// content modification, but not about metadata and dirs/symlinks.
|
|
|
|
if !job.file.IsInvalid() && job.jobType&(dbUpdateHandleFile|dbUpdateDeleteFile) != 0 {
|
|
|
|
found = true
|
|
|
|
lastFile = job.file
|
2015-04-05 13:34:29 +00:00
|
|
|
}
|
|
|
|
|
2019-03-05 20:32:37 +00:00
|
|
|
job.file.Sequence = 0
|
|
|
|
|
|
|
|
batch.append(job.file)
|
|
|
|
|
|
|
|
batch.flushIfFull()
|
|
|
|
|
2015-04-05 13:34:29 +00:00
|
|
|
case <-tick.C:
|
2019-03-05 20:32:37 +00:00
|
|
|
batch.flush()
|
2015-04-05 13:34:29 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-03-05 20:32:37 +00:00
|
|
|
batch.flush()
|
2015-04-05 13:34:29 +00:00
|
|
|
}
|
|
|
|
|
2017-12-07 08:42:03 +00:00
|
|
|
// pullScannerRoutine aggregates paths to be scanned after pulling. The scan is
|
|
|
|
// scheduled once when scanChan is closed (scanning can not happen during pulling).
|
|
|
|
func (f *sendReceiveFolder) pullScannerRoutine(scanChan <-chan string) {
|
|
|
|
toBeScanned := make(map[string]struct{})
|
|
|
|
|
|
|
|
for path := range scanChan {
|
|
|
|
toBeScanned[path] = struct{}{}
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(toBeScanned) != 0 {
|
|
|
|
scanList := make([]string, 0, len(toBeScanned))
|
|
|
|
for path := range toBeScanned {
|
|
|
|
l.Debugln(f, "scheduling scan after pulling for", path)
|
|
|
|
scanList = append(scanList, path)
|
|
|
|
}
|
2019-02-02 11:16:27 +00:00
|
|
|
f.Scan(scanList)
|
2017-12-07 08:42:03 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-12-16 22:23:35 +00:00
|
|
|
func (f *sendReceiveFolder) inConflict(current, replacement protocol.Vector) bool {
|
2015-04-09 10:53:41 +00:00
|
|
|
if current.Concurrent(replacement) {
|
|
|
|
// Obvious case
|
|
|
|
return true
|
|
|
|
}
|
2018-02-25 08:39:00 +00:00
|
|
|
if replacement.Counter(f.shortID) > current.Counter(f.shortID) {
|
2015-04-09 10:53:41 +00:00
|
|
|
// The replacement file contains a higher version for ourselves than
|
|
|
|
// what we have. This isn't supposed to be possible, since it's only
|
|
|
|
// we who can increment that counter. We take it as a sign that
|
|
|
|
// something is wrong (our index may have been corrupted or removed)
|
|
|
|
// and flag it as a conflict.
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2016-04-15 10:59:41 +00:00
|
|
|
func removeAvailability(availabilities []Availability, availability Availability) []Availability {
|
|
|
|
for i := range availabilities {
|
|
|
|
if availabilities[i] == availability {
|
|
|
|
availabilities[i] = availabilities[len(availabilities)-1]
|
|
|
|
return availabilities[:len(availabilities)-1]
|
2014-12-28 23:11:32 +00:00
|
|
|
}
|
|
|
|
}
|
2016-04-15 10:59:41 +00:00
|
|
|
return availabilities
|
2014-12-28 23:11:32 +00:00
|
|
|
}
|
2015-03-29 14:16:36 +00:00
|
|
|
|
2019-03-04 12:20:40 +00:00
|
|
|
func (f *sendReceiveFolder) moveForConflict(name, lastModBy string, scanChan chan<- string) error {
|
|
|
|
if isConflict(name) {
|
2016-01-03 20:15:02 +00:00
|
|
|
l.Infoln("Conflict for", name, "which is already a conflict copy; not copying again.")
|
2020-12-27 21:26:25 +00:00
|
|
|
if err := f.mtimefs.Remove(name); err != nil && !fs.IsNotExist(err) {
|
2019-03-04 13:01:52 +00:00
|
|
|
return errors.Wrap(err, contextRemovingOldItem)
|
2016-01-03 20:15:02 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-12-21 11:23:20 +00:00
|
|
|
if f.MaxConflicts == 0 {
|
2020-12-27 21:26:25 +00:00
|
|
|
if err := f.mtimefs.Remove(name); err != nil && !fs.IsNotExist(err) {
|
2019-03-04 13:01:52 +00:00
|
|
|
return errors.Wrap(err, contextRemovingOldItem)
|
2015-10-13 19:50:58 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-03-04 12:20:40 +00:00
|
|
|
newName := conflictName(name, lastModBy)
|
2020-12-27 21:26:25 +00:00
|
|
|
err := f.mtimefs.Rename(name, newName)
|
2017-08-19 14:36:56 +00:00
|
|
|
if fs.IsNotExist(err) {
|
2015-04-28 09:33:54 +00:00
|
|
|
// We were supposed to move a file away but it does not exist. Either
|
|
|
|
// the user has already moved it away, or the conflict was between a
|
|
|
|
// remote modification and a local delete. In either way it does not
|
|
|
|
// matter, go ahead as if the move succeeded.
|
2015-10-13 19:50:58 +00:00
|
|
|
err = nil
|
|
|
|
}
|
2016-12-21 11:23:20 +00:00
|
|
|
if f.MaxConflicts > -1 {
|
2020-12-27 21:26:25 +00:00
|
|
|
matches := existingConflicts(name, f.mtimefs)
|
2019-03-04 12:20:40 +00:00
|
|
|
if len(matches) > f.MaxConflicts {
|
2015-10-13 19:50:58 +00:00
|
|
|
sort.Sort(sort.Reverse(sort.StringSlice(matches)))
|
2016-12-21 11:23:20 +00:00
|
|
|
for _, match := range matches[f.MaxConflicts:] {
|
2020-12-27 21:26:25 +00:00
|
|
|
if gerr := f.mtimefs.Remove(match); gerr != nil {
|
2016-04-26 15:11:19 +00:00
|
|
|
l.Debugln(f, "removing extra conflict", gerr)
|
2015-10-13 19:50:58 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2015-04-28 09:33:54 +00:00
|
|
|
}
|
2019-02-12 15:05:20 +00:00
|
|
|
if err == nil {
|
|
|
|
scanChan <- newName
|
|
|
|
}
|
2015-04-28 09:33:54 +00:00
|
|
|
return err
|
2015-03-29 14:16:36 +00:00
|
|
|
}
|
2015-06-26 11:31:30 +00:00
|
|
|
|
2019-03-04 13:01:52 +00:00
|
|
|
func (f *sendReceiveFolder) newPullError(path string, err error) {
|
2019-11-19 08:56:53 +00:00
|
|
|
if errors.Cause(err) == f.ctx.Err() {
|
|
|
|
// Error because the folder stopped - no point logging/tracking
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2020-11-06 13:22:20 +00:00
|
|
|
f.errorsMut.Lock()
|
|
|
|
defer f.errorsMut.Unlock()
|
2015-06-26 11:31:30 +00:00
|
|
|
|
|
|
|
// We might get more than one error report for a file (i.e. error on
|
|
|
|
// Write() followed by Close()); we keep the first error as that is
|
|
|
|
// probably closer to the root cause.
|
2020-07-28 09:13:15 +00:00
|
|
|
if _, ok := f.tempPullErrors[path]; ok {
|
2015-06-26 11:31:30 +00:00
|
|
|
return
|
|
|
|
}
|
2019-03-04 13:01:52 +00:00
|
|
|
|
|
|
|
// Establish context to differentiate from errors while scanning.
|
|
|
|
// Use "syncing" as opposed to "pulling" as the latter might be used
|
|
|
|
// for errors occurring specificly in the puller routine.
|
2019-11-19 08:56:53 +00:00
|
|
|
errStr := fmt.Sprintln("syncing:", err)
|
2020-07-28 09:13:15 +00:00
|
|
|
f.tempPullErrors[path] = errStr
|
2019-11-19 08:56:53 +00:00
|
|
|
|
2020-07-28 09:13:15 +00:00
|
|
|
l.Debugf("%v new error for %v: %v", f, path, err)
|
2015-06-26 11:31:30 +00:00
|
|
|
}
|
|
|
|
|
2019-03-07 14:15:14 +00:00
|
|
|
// deleteItemOnDisk deletes the file represented by old that is about to be replaced by new.
|
2020-01-21 17:23:08 +00:00
|
|
|
func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, snap *db.Snapshot, scanChan chan<- string) (err error) {
|
2019-03-07 14:15:14 +00:00
|
|
|
defer func() {
|
|
|
|
err = errors.Wrap(err, contextRemovingOldItem)
|
|
|
|
}()
|
|
|
|
|
|
|
|
switch {
|
|
|
|
case item.IsDirectory():
|
|
|
|
// Directories aren't archived and need special treatment due
|
|
|
|
// to potential children.
|
2020-01-21 17:23:08 +00:00
|
|
|
return f.deleteDirOnDisk(item.Name, snap, scanChan)
|
2019-03-07 14:15:14 +00:00
|
|
|
|
|
|
|
case !item.IsSymlink() && f.versioner != nil:
|
|
|
|
// If we should use versioning, let the versioner archive the
|
|
|
|
// file before we replace it. Archiving a non-existent file is not
|
|
|
|
// an error.
|
|
|
|
// Symlinks aren't archived.
|
|
|
|
|
2019-07-31 08:53:35 +00:00
|
|
|
return f.inWritableDir(f.versioner.Archive, item.Name)
|
2019-03-07 14:15:14 +00:00
|
|
|
}
|
|
|
|
|
2020-12-27 21:26:25 +00:00
|
|
|
return f.inWritableDir(f.mtimefs.Remove, item.Name)
|
2019-03-07 14:15:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// deleteDirOnDisk attempts to delete a directory. It checks for files/dirs inside
|
2017-12-07 08:42:03 +00:00
|
|
|
// the directory and removes them if possible or returns an error if it fails
|
2020-01-21 17:23:08 +00:00
|
|
|
func (f *sendReceiveFolder) deleteDirOnDisk(dir string, snap *db.Snapshot, scanChan chan<- string) error {
|
2020-12-27 21:26:25 +00:00
|
|
|
if err := osutil.TraversesSymlink(f.mtimefs, filepath.Dir(dir)); err != nil {
|
2019-10-22 19:55:51 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2021-01-02 20:40:37 +00:00
|
|
|
if err := f.deleteDirOnDiskHandleChildren(dir, snap, scanChan); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
err := f.inWritableDir(f.mtimefs.Remove, dir)
|
|
|
|
if err == nil || fs.IsNotExist(err) {
|
|
|
|
// It was removed or it doesn't exist to start with
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
if _, serr := f.mtimefs.Lstat(dir); serr != nil && !fs.IsPermission(serr) {
|
|
|
|
// We get an error just looking at the directory, and it's not a
|
|
|
|
// permission problem. Lets assume the error is in fact some variant
|
|
|
|
// of "file does not exist" (possibly expressed as some parent being a
|
|
|
|
// file and not a directory etc) and that the delete is handled.
|
|
|
|
return nil
|
|
|
|
}
|
2017-12-07 08:42:03 +00:00
|
|
|
|
2021-01-02 20:40:37 +00:00
|
|
|
return err
|
|
|
|
}
|
2017-12-07 08:42:03 +00:00
|
|
|
|
2021-01-02 20:40:37 +00:00
|
|
|
func (f *sendReceiveFolder) deleteDirOnDiskHandleChildren(dir string, snap *db.Snapshot, scanChan chan<- string) error {
|
|
|
|
var dirsToDelete []string
|
2020-05-25 06:46:24 +00:00
|
|
|
var hasIgnored, hasKnown, hasToBeScanned, hasReceiveOnlyChanged bool
|
2021-01-02 20:40:37 +00:00
|
|
|
var delErr error
|
2017-12-07 08:42:03 +00:00
|
|
|
|
2021-01-02 20:40:37 +00:00
|
|
|
err := f.mtimefs.Walk(dir, func(path string, info fs.FileInfo, err error) error {
|
|
|
|
if path == dir {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
switch match := f.ignores.Match(path); {
|
|
|
|
case match.IsDeletable():
|
|
|
|
if info.IsDir() {
|
|
|
|
dirsToDelete = append(dirsToDelete, path)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
fallthrough
|
|
|
|
case fs.IsTemporary(path):
|
|
|
|
if err := f.mtimefs.Remove(path); err != nil && delErr == nil {
|
|
|
|
delErr = err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
case match.IsIgnored():
|
2017-12-07 08:42:03 +00:00
|
|
|
hasIgnored = true
|
2021-01-02 20:40:37 +00:00
|
|
|
return nil
|
2020-05-25 06:46:24 +00:00
|
|
|
}
|
2021-01-02 20:40:37 +00:00
|
|
|
cf, ok := snap.Get(protocol.LocalDeviceID, path)
|
2020-05-25 06:46:24 +00:00
|
|
|
switch {
|
|
|
|
case !ok || cf.IsDeleted():
|
|
|
|
// Something appeared in the dir that we either are not
|
|
|
|
// aware of at all or that we think should be deleted
|
|
|
|
// -> schedule scan.
|
2021-01-02 20:40:37 +00:00
|
|
|
scanChan <- path
|
2017-12-07 08:42:03 +00:00
|
|
|
hasToBeScanned = true
|
2021-01-02 20:40:37 +00:00
|
|
|
return nil
|
2020-05-25 06:46:24 +00:00
|
|
|
case ok && f.Type == config.FolderTypeReceiveOnly && cf.IsReceiveOnlyChanged():
|
|
|
|
hasReceiveOnlyChanged = true
|
2021-01-02 20:40:37 +00:00
|
|
|
return nil
|
2020-05-25 06:46:24 +00:00
|
|
|
}
|
2021-01-02 20:40:37 +00:00
|
|
|
diskFile, err := scanner.CreateFileInfo(info, path, f.mtimefs)
|
2020-05-25 06:46:24 +00:00
|
|
|
if err != nil {
|
|
|
|
// Lets just assume the file has changed.
|
2021-01-02 20:40:37 +00:00
|
|
|
scanChan <- path
|
2020-05-25 06:46:24 +00:00
|
|
|
hasToBeScanned = true
|
2021-01-02 20:40:37 +00:00
|
|
|
return nil
|
2020-05-25 06:46:24 +00:00
|
|
|
}
|
2020-08-25 06:11:14 +00:00
|
|
|
if !cf.IsEquivalentOptional(diskFile, f.modTimeWindow, f.IgnorePerms, true, protocol.LocalAllFlags) {
|
2020-05-25 06:46:24 +00:00
|
|
|
// File on disk changed compared to what we have in db
|
|
|
|
// -> schedule scan.
|
2021-01-02 20:40:37 +00:00
|
|
|
scanChan <- path
|
2020-05-25 06:46:24 +00:00
|
|
|
hasToBeScanned = true
|
2021-01-02 20:40:37 +00:00
|
|
|
return nil
|
2020-05-25 06:46:24 +00:00
|
|
|
}
|
|
|
|
// Dir contains file that is valid according to db and
|
|
|
|
// not ignored -> something weird is going on
|
|
|
|
hasKnown = true
|
2021-01-02 20:40:37 +00:00
|
|
|
return nil
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
for i := range dirsToDelete {
|
|
|
|
if err := f.mtimefs.Remove(dirsToDelete[len(dirsToDelete)-1-i]); err != nil && delErr == nil {
|
|
|
|
delErr = err
|
|
|
|
}
|
2017-12-07 08:42:03 +00:00
|
|
|
}
|
|
|
|
|
2021-01-02 20:40:37 +00:00
|
|
|
// "Error precedence":
|
|
|
|
// Something changed on disk, check that and maybe all else gets resolved
|
2017-12-07 08:42:03 +00:00
|
|
|
if hasToBeScanned {
|
|
|
|
return errDirHasToBeScanned
|
|
|
|
}
|
2021-01-02 20:40:37 +00:00
|
|
|
// Ignored files will never be touched, i.e. this will keep failing until
|
|
|
|
// user acts.
|
2017-12-07 08:42:03 +00:00
|
|
|
if hasIgnored {
|
|
|
|
return errDirHasIgnored
|
|
|
|
}
|
2020-05-25 06:46:24 +00:00
|
|
|
if hasReceiveOnlyChanged {
|
2020-08-20 08:56:29 +00:00
|
|
|
// Pretend we deleted the directory. It will be resurrected as a
|
|
|
|
// receive-only changed item on scan.
|
|
|
|
scanChan <- dir
|
|
|
|
return nil
|
2020-05-25 06:46:24 +00:00
|
|
|
}
|
2017-12-07 08:42:03 +00:00
|
|
|
if hasKnown {
|
|
|
|
return errDirNotEmpty
|
|
|
|
}
|
2021-01-02 20:40:37 +00:00
|
|
|
// All good, except maybe failing to remove a (?d) ignored item
|
|
|
|
return delErr
|
2017-12-07 08:42:03 +00:00
|
|
|
}
|
|
|
|
|
2019-06-29 05:45:41 +00:00
|
|
|
// scanIfItemChanged schedules the given file for scanning and returns errModified
|
|
|
|
// if it differs from the information in the database. Returns nil if the file has
|
|
|
|
// not changed.
|
2020-06-16 13:20:08 +00:00
|
|
|
func (f *sendReceiveFolder) scanIfItemChanged(name string, stat fs.FileInfo, item protocol.FileInfo, hasItem bool, scanChan chan<- string) (err error) {
|
2019-03-07 14:15:14 +00:00
|
|
|
defer func() {
|
2019-06-29 05:45:41 +00:00
|
|
|
if err == errModified {
|
2020-06-16 13:20:08 +00:00
|
|
|
scanChan <- name
|
2019-03-07 14:15:14 +00:00
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
if !hasItem || item.Deleted {
|
|
|
|
// The item appeared from nowhere
|
2019-06-29 05:45:41 +00:00
|
|
|
return errModified
|
2019-03-07 14:15:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Check that the item on disk is what we expect it to be according
|
|
|
|
// to the database. If there's a mismatch here, there might be local
|
|
|
|
// changes that we don't know about yet and we should scan before
|
|
|
|
// touching the item.
|
2020-12-27 21:26:25 +00:00
|
|
|
statItem, err := scanner.CreateFileInfo(stat, item.Name, f.mtimefs)
|
2019-03-07 14:15:14 +00:00
|
|
|
if err != nil {
|
2019-06-29 05:45:41 +00:00
|
|
|
return errors.Wrap(err, "comparing item on disk to db")
|
|
|
|
}
|
|
|
|
|
2020-08-25 06:11:14 +00:00
|
|
|
if !statItem.IsEquivalentOptional(item, f.modTimeWindow, f.IgnorePerms, true, protocol.LocalAllFlags) {
|
2019-06-29 05:45:41 +00:00
|
|
|
return errModified
|
2019-03-07 14:15:14 +00:00
|
|
|
}
|
|
|
|
|
2019-06-29 05:45:41 +00:00
|
|
|
return nil
|
2019-03-07 14:15:14 +00:00
|
|
|
}
|
|
|
|
|
2018-09-16 07:48:14 +00:00
|
|
|
// checkToBeDeleted makes sure the file on disk is compatible with what there is
|
|
|
|
// in the DB before the caller proceeds with actually deleting it.
|
2020-09-12 05:45:50 +00:00
|
|
|
// I.e. non-nil error status means "Do not delete!" or "is already deleted".
|
|
|
|
func (f *sendReceiveFolder) checkToBeDeleted(file, cur protocol.FileInfo, hasCur bool, scanChan chan<- string) error {
|
2020-12-27 21:26:25 +00:00
|
|
|
if err := osutil.TraversesSymlink(f.mtimefs, filepath.Dir(file.Name)); err != nil {
|
2020-06-16 13:20:08 +00:00
|
|
|
l.Debugln(f, "not deleting item behind symlink on disk, but update db", file.Name)
|
|
|
|
return fs.ErrNotExist
|
|
|
|
}
|
|
|
|
|
2020-12-27 21:26:25 +00:00
|
|
|
stat, err := f.mtimefs.Lstat(file.Name)
|
2020-09-12 05:45:50 +00:00
|
|
|
deleted := fs.IsNotExist(err) || fs.IsErrCaseConflict(err)
|
|
|
|
if !deleted && err != nil {
|
2018-09-16 07:48:14 +00:00
|
|
|
return err
|
|
|
|
}
|
2020-09-12 05:45:50 +00:00
|
|
|
if deleted {
|
2020-09-10 06:25:44 +00:00
|
|
|
if hasCur && !cur.Deleted && !cur.IsUnsupported() {
|
2020-06-16 13:20:08 +00:00
|
|
|
scanChan <- file.Name
|
|
|
|
return errModified
|
|
|
|
}
|
|
|
|
l.Debugln(f, "not deleting item we don't have, but update db", file.Name)
|
2020-09-12 05:45:50 +00:00
|
|
|
return err
|
2020-06-16 13:20:08 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return f.scanIfItemChanged(file.Name, stat, cur, hasCur, scanChan)
|
2018-09-16 07:48:14 +00:00
|
|
|
}
|
|
|
|
|
2019-01-25 08:52:21 +00:00
|
|
|
func (f *sendReceiveFolder) maybeCopyOwner(path string) error {
|
|
|
|
if !f.CopyOwnershipFromParent {
|
|
|
|
// Not supposed to do anything.
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
if runtime.GOOS == "windows" {
|
|
|
|
// Can't do anything.
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-12-27 21:26:25 +00:00
|
|
|
info, err := f.mtimefs.Lstat(filepath.Dir(path))
|
2019-01-25 08:52:21 +00:00
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "copy owner from parent")
|
|
|
|
}
|
2020-12-27 21:26:25 +00:00
|
|
|
if err := f.mtimefs.Lchown(path, info.Owner(), info.Group()); err != nil {
|
2019-01-25 08:52:21 +00:00
|
|
|
return errors.Wrap(err, "copy owner from parent")
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-07-31 08:53:35 +00:00
|
|
|
func (f *sendReceiveFolder) inWritableDir(fn func(string) error, path string) error {
|
2020-12-27 21:26:25 +00:00
|
|
|
return inWritableDir(fn, f.mtimefs, path, f.IgnorePerms)
|
2019-07-31 08:53:35 +00:00
|
|
|
}
|
|
|
|
|
2020-06-18 06:15:47 +00:00
|
|
|
func (f *sendReceiveFolder) limitedWriteAt(fd io.WriterAt, data []byte, offset int64) error {
|
|
|
|
return f.withLimiter(func() error {
|
|
|
|
_, err := fd.WriteAt(data, offset)
|
|
|
|
return err
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *sendReceiveFolder) withLimiter(fn func() error) error {
|
2020-04-26 22:13:18 +00:00
|
|
|
if err := f.writeLimiter.takeWithContext(f.ctx, 1); err != nil {
|
2020-06-18 06:15:47 +00:00
|
|
|
return err
|
2020-04-26 22:13:18 +00:00
|
|
|
}
|
|
|
|
defer f.writeLimiter.give(1)
|
2020-06-18 06:15:47 +00:00
|
|
|
return fn()
|
2020-04-26 22:13:18 +00:00
|
|
|
}
|
|
|
|
|
2018-01-14 17:01:06 +00:00
|
|
|
// A []FileError is sent as part of an event and will be JSON serialized.
|
|
|
|
type FileError struct {
|
2015-06-26 11:31:30 +00:00
|
|
|
Path string `json:"path"`
|
|
|
|
Err string `json:"error"`
|
|
|
|
}
|
|
|
|
|
2018-01-14 17:01:06 +00:00
|
|
|
type fileErrorList []FileError
|
2015-06-26 11:31:30 +00:00
|
|
|
|
|
|
|
func (l fileErrorList) Len() int {
|
|
|
|
return len(l)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (l fileErrorList) Less(a, b int) bool {
|
|
|
|
return l[a].Path < l[b].Path
|
|
|
|
}
|
|
|
|
|
|
|
|
func (l fileErrorList) Swap(a, b int) {
|
|
|
|
l[a], l[b] = l[b], l[a]
|
|
|
|
}
|
2016-08-05 07:13:52 +00:00
|
|
|
|
2019-03-04 12:20:40 +00:00
|
|
|
func conflictName(name, lastModBy string) string {
|
|
|
|
ext := filepath.Ext(name)
|
|
|
|
return name[:len(name)-len(ext)] + time.Now().Format(".sync-conflict-20060102-150405-") + lastModBy + ext
|
|
|
|
}
|
|
|
|
|
|
|
|
func isConflict(name string) bool {
|
|
|
|
return strings.Contains(filepath.Base(name), ".sync-conflict-")
|
|
|
|
}
|
|
|
|
|
|
|
|
func existingConflicts(name string, fs fs.Filesystem) []string {
|
|
|
|
ext := filepath.Ext(name)
|
|
|
|
matches, err := fs.Glob(name[:len(name)-len(ext)] + ".sync-conflict-????????-??????*" + ext)
|
|
|
|
if err != nil {
|
|
|
|
l.Debugln("globbing for conflicts", err)
|
|
|
|
}
|
|
|
|
return matches
|
|
|
|
}
|