mirror of
https://github.com/octoleo/syncthing.git
synced 2024-11-10 07:11:08 +00:00
5118538179
Instead of separately tracking the token.
Also changes serviceMap to have a channel version of RemoveAndWait, so
that it's possible to do the removal under a lock but wait outside of
the lock. And changed where we do that in connection close, reversing
the change that happened when I added the serviceMap in 40b3b9ad1.
3335 lines
104 KiB
Go
3335 lines
104 KiB
Go
// Copyright (C) 2014 The Syncthing Authors.
|
|
//
|
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
|
|
//go:generate -command counterfeiter go run github.com/maxbrunsfeld/counterfeiter/v6
|
|
//go:generate counterfeiter -o mocks/model.go --fake-name Model . Model
|
|
|
|
package model
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"runtime"
|
|
"strings"
|
|
stdsync "sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/thejerf/suture/v4"
|
|
|
|
"github.com/syncthing/syncthing/lib/build"
|
|
"github.com/syncthing/syncthing/lib/config"
|
|
"github.com/syncthing/syncthing/lib/connections"
|
|
"github.com/syncthing/syncthing/lib/db"
|
|
"github.com/syncthing/syncthing/lib/events"
|
|
"github.com/syncthing/syncthing/lib/fs"
|
|
"github.com/syncthing/syncthing/lib/ignore"
|
|
"github.com/syncthing/syncthing/lib/osutil"
|
|
"github.com/syncthing/syncthing/lib/protocol"
|
|
"github.com/syncthing/syncthing/lib/scanner"
|
|
"github.com/syncthing/syncthing/lib/semaphore"
|
|
"github.com/syncthing/syncthing/lib/stats"
|
|
"github.com/syncthing/syncthing/lib/svcutil"
|
|
"github.com/syncthing/syncthing/lib/sync"
|
|
"github.com/syncthing/syncthing/lib/ur/contract"
|
|
"github.com/syncthing/syncthing/lib/versioner"
|
|
)
|
|
|
|
type service interface {
|
|
suture.Service
|
|
BringToFront(string)
|
|
Override()
|
|
Revert()
|
|
DelayScan(d time.Duration)
|
|
ScheduleScan()
|
|
SchedulePull() // something relevant changed, we should try a pull
|
|
Jobs(page, perpage int) ([]string, []string, int) // In progress, Queued, skipped
|
|
Scan(subs []string) error
|
|
Errors() []FileError
|
|
WatchError() error
|
|
ScheduleForceRescan(path string)
|
|
GetStatistics() (stats.FolderStatistics, error)
|
|
|
|
getState() (folderState, time.Time, error)
|
|
}
|
|
|
|
type Availability struct {
|
|
ID protocol.DeviceID `json:"id"`
|
|
FromTemporary bool `json:"fromTemporary"`
|
|
}
|
|
|
|
type Model interface {
|
|
suture.Service
|
|
|
|
connections.Model
|
|
|
|
ResetFolder(folder string) error
|
|
DelayScan(folder string, next time.Duration)
|
|
ScanFolder(folder string) error
|
|
ScanFolders() map[string]error
|
|
ScanFolderSubdirs(folder string, subs []string) error
|
|
State(folder string) (string, time.Time, error)
|
|
FolderErrors(folder string) ([]FileError, error)
|
|
WatchError(folder string) error
|
|
Override(folder string)
|
|
Revert(folder string)
|
|
BringToFront(folder, file string)
|
|
LoadIgnores(folder string) ([]string, []string, error)
|
|
CurrentIgnores(folder string) ([]string, []string, error)
|
|
SetIgnores(folder string, content []string) error
|
|
|
|
GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error)
|
|
RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]error, error)
|
|
|
|
DBSnapshot(folder string) (*db.Snapshot, error)
|
|
NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated, error)
|
|
RemoteNeedFolderFiles(folder string, device protocol.DeviceID, page, perpage int) ([]db.FileInfoTruncated, error)
|
|
LocalChangedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, error)
|
|
FolderProgressBytesCompleted(folder string) int64
|
|
|
|
CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool, error)
|
|
CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool, error)
|
|
GetMtimeMapping(folder string, file string) (fs.MtimeMapping, error)
|
|
Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) ([]Availability, error)
|
|
|
|
Completion(device protocol.DeviceID, folder string) (FolderCompletion, error)
|
|
ConnectionStats() map[string]interface{}
|
|
DeviceStatistics() (map[protocol.DeviceID]stats.DeviceStatistics, error)
|
|
FolderStatistics() (map[string]stats.FolderStatistics, error)
|
|
UsageReportingStats(report *contract.Report, version int, preview bool)
|
|
|
|
PendingDevices() (map[protocol.DeviceID]db.ObservedDevice, error)
|
|
PendingFolders(device protocol.DeviceID) (map[string]db.PendingFolder, error)
|
|
DismissPendingDevice(device protocol.DeviceID) error
|
|
DismissPendingFolder(device protocol.DeviceID, folder string) error
|
|
|
|
StartDeadlockDetector(timeout time.Duration)
|
|
GlobalDirectoryTree(folder, prefix string, levels int, dirsOnly bool) ([]*TreeEntry, error)
|
|
}
|
|
|
|
type model struct {
|
|
*suture.Supervisor
|
|
|
|
// constructor parameters
|
|
cfg config.Wrapper
|
|
id protocol.DeviceID
|
|
clientName string
|
|
clientVersion string
|
|
db *db.Lowlevel
|
|
protectedFiles []string
|
|
evLogger events.Logger
|
|
|
|
// constant or concurrency safe fields
|
|
finder *db.BlockFinder
|
|
progressEmitter *ProgressEmitter
|
|
shortID protocol.ShortID
|
|
// globalRequestLimiter limits the amount of data in concurrent incoming
|
|
// requests
|
|
globalRequestLimiter *semaphore.Semaphore
|
|
// folderIOLimiter limits the number of concurrent I/O heavy operations,
|
|
// such as scans and pulls.
|
|
folderIOLimiter *semaphore.Semaphore
|
|
fatalChan chan error
|
|
started chan struct{}
|
|
keyGen *protocol.KeyGenerator
|
|
|
|
// fields protected by fmut
|
|
fmut sync.RWMutex
|
|
folderCfgs map[string]config.FolderConfiguration // folder -> cfg
|
|
folderFiles map[string]*db.FileSet // folder -> files
|
|
deviceStatRefs map[protocol.DeviceID]*stats.DeviceStatisticsReference // deviceID -> statsRef
|
|
folderIgnores map[string]*ignore.Matcher // folder -> matcher object
|
|
folderRunners *serviceMap[string, service] // folder -> puller or scanner
|
|
folderRestartMuts syncMutexMap // folder -> restart mutex
|
|
folderVersioners map[string]versioner.Versioner // folder -> versioner (may be nil)
|
|
folderEncryptionPasswordTokens map[string][]byte // folder -> encryption token (may be missing, and only for encryption type folders)
|
|
folderEncryptionFailures map[string]map[protocol.DeviceID]error // folder -> device -> error regarding encryption consistency (may be missing)
|
|
|
|
// fields protected by pmut
|
|
pmut sync.RWMutex
|
|
conn map[protocol.DeviceID]protocol.Connection
|
|
connRequestLimiters map[protocol.DeviceID]*semaphore.Semaphore
|
|
closed map[protocol.DeviceID]chan struct{}
|
|
helloMessages map[protocol.DeviceID]protocol.Hello
|
|
deviceDownloads map[protocol.DeviceID]*deviceDownloadState
|
|
remoteFolderStates map[protocol.DeviceID]map[string]remoteFolderState // deviceID -> folders
|
|
indexHandlers *serviceMap[protocol.DeviceID, *indexHandlerRegistry]
|
|
|
|
// for testing only
|
|
foldersRunning atomic.Int32
|
|
}
|
|
|
|
var _ config.Verifier = &model{}
|
|
|
|
type folderFactory func(*model, *db.FileSet, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, events.Logger, *semaphore.Semaphore) service
|
|
|
|
var folderFactories = make(map[config.FolderType]folderFactory)
|
|
|
|
var (
|
|
errDeviceUnknown = errors.New("unknown device")
|
|
errDevicePaused = errors.New("device is paused")
|
|
errDeviceRemoved = errors.New("device has been removed")
|
|
ErrFolderPaused = errors.New("folder is paused")
|
|
ErrFolderNotRunning = errors.New("folder is not running")
|
|
ErrFolderMissing = errors.New("no such folder")
|
|
errNoVersioner = errors.New("folder has no versioner")
|
|
// errors about why a connection is closed
|
|
errReplacingConnection = errors.New("replacing connection")
|
|
errStopped = errors.New("Syncthing is being stopped")
|
|
errEncryptionInvConfigLocal = errors.New("can't encrypt outgoing data because local data is encrypted (folder-type receive-encrypted)")
|
|
errEncryptionInvConfigRemote = errors.New("remote has encrypted data and encrypts that data for us - this is impossible")
|
|
errEncryptionNotEncryptedLocal = errors.New("remote expects to exchange encrypted data, but is configured for plain data")
|
|
errEncryptionPlainForReceiveEncrypted = errors.New("remote expects to exchange plain data, but is configured to be encrypted")
|
|
errEncryptionPlainForRemoteEncrypted = errors.New("remote expects to exchange plain data, but local data is encrypted (folder-type receive-encrypted)")
|
|
errEncryptionNotEncryptedUntrusted = errors.New("device is untrusted, but configured to receive plain data")
|
|
errEncryptionPassword = errors.New("different encryption passwords used")
|
|
errEncryptionTokenRead = errors.New("failed to read encryption token")
|
|
errEncryptionTokenWrite = errors.New("failed to write encryption token")
|
|
errMissingRemoteInClusterConfig = errors.New("remote device missing in cluster config")
|
|
errMissingLocalInClusterConfig = errors.New("local device missing in cluster config")
|
|
)
|
|
|
|
// NewModel creates and starts a new model. The model starts in read-only mode,
|
|
// where it sends index information to connected peers and responds to requests
|
|
// for file data without altering the local folder in any way.
|
|
func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Lowlevel, protectedFiles []string, evLogger events.Logger, keyGen *protocol.KeyGenerator) Model {
|
|
spec := svcutil.SpecWithDebugLogger(l)
|
|
m := &model{
|
|
Supervisor: suture.New("model", spec),
|
|
|
|
// constructor parameters
|
|
cfg: cfg,
|
|
id: id,
|
|
clientName: clientName,
|
|
clientVersion: clientVersion,
|
|
db: ldb,
|
|
protectedFiles: protectedFiles,
|
|
evLogger: evLogger,
|
|
|
|
// constant or concurrency safe fields
|
|
finder: db.NewBlockFinder(ldb),
|
|
progressEmitter: NewProgressEmitter(cfg, evLogger),
|
|
shortID: id.Short(),
|
|
globalRequestLimiter: semaphore.New(1024 * cfg.Options().MaxConcurrentIncomingRequestKiB()),
|
|
folderIOLimiter: semaphore.New(cfg.Options().MaxFolderConcurrency()),
|
|
fatalChan: make(chan error),
|
|
started: make(chan struct{}),
|
|
keyGen: keyGen,
|
|
|
|
// fields protected by fmut
|
|
fmut: sync.NewRWMutex(),
|
|
folderCfgs: make(map[string]config.FolderConfiguration),
|
|
folderFiles: make(map[string]*db.FileSet),
|
|
deviceStatRefs: make(map[protocol.DeviceID]*stats.DeviceStatisticsReference),
|
|
folderIgnores: make(map[string]*ignore.Matcher),
|
|
folderRunners: newServiceMap[string, service](evLogger),
|
|
folderVersioners: make(map[string]versioner.Versioner),
|
|
folderEncryptionPasswordTokens: make(map[string][]byte),
|
|
folderEncryptionFailures: make(map[string]map[protocol.DeviceID]error),
|
|
|
|
// fields protected by pmut
|
|
pmut: sync.NewRWMutex(),
|
|
conn: make(map[protocol.DeviceID]protocol.Connection),
|
|
connRequestLimiters: make(map[protocol.DeviceID]*semaphore.Semaphore),
|
|
closed: make(map[protocol.DeviceID]chan struct{}),
|
|
helloMessages: make(map[protocol.DeviceID]protocol.Hello),
|
|
deviceDownloads: make(map[protocol.DeviceID]*deviceDownloadState),
|
|
remoteFolderStates: make(map[protocol.DeviceID]map[string]remoteFolderState),
|
|
indexHandlers: newServiceMap[protocol.DeviceID, *indexHandlerRegistry](evLogger),
|
|
}
|
|
for devID := range cfg.Devices() {
|
|
m.deviceStatRefs[devID] = stats.NewDeviceStatisticsReference(m.db, devID)
|
|
}
|
|
m.Add(m.folderRunners)
|
|
m.Add(m.progressEmitter)
|
|
m.Add(m.indexHandlers)
|
|
m.Add(svcutil.AsService(m.serve, m.String()))
|
|
|
|
return m
|
|
}
|
|
|
|
func (m *model) serve(ctx context.Context) error {
|
|
defer m.closeAllConnectionsAndWait()
|
|
|
|
cfg := m.cfg.Subscribe(m)
|
|
defer m.cfg.Unsubscribe(m)
|
|
|
|
if err := m.initFolders(cfg); err != nil {
|
|
close(m.started)
|
|
return svcutil.AsFatalErr(err, svcutil.ExitError)
|
|
}
|
|
|
|
close(m.started)
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case err := <-m.fatalChan:
|
|
return svcutil.AsFatalErr(err, svcutil.ExitError)
|
|
}
|
|
}
|
|
|
|
func (m *model) initFolders(cfg config.Configuration) error {
|
|
clusterConfigDevices := make(deviceIDSet, len(cfg.Devices))
|
|
for _, folderCfg := range cfg.Folders {
|
|
if folderCfg.Paused {
|
|
folderCfg.CreateRoot()
|
|
continue
|
|
}
|
|
err := m.newFolder(folderCfg, cfg.Options.CacheIgnoredFiles)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
clusterConfigDevices.add(folderCfg.DeviceIDs())
|
|
}
|
|
|
|
ignoredDevices := observedDeviceSet(m.cfg.IgnoredDevices())
|
|
m.cleanPending(cfg.DeviceMap(), cfg.FolderMap(), ignoredDevices, nil)
|
|
|
|
m.sendClusterConfig(clusterConfigDevices.AsSlice())
|
|
return nil
|
|
}
|
|
|
|
func (m *model) closeAllConnectionsAndWait() {
|
|
m.pmut.RLock()
|
|
closed := make([]chan struct{}, 0, len(m.conn))
|
|
for id, conn := range m.conn {
|
|
closed = append(closed, m.closed[id])
|
|
go conn.Close(errStopped)
|
|
}
|
|
m.pmut.RUnlock()
|
|
for _, c := range closed {
|
|
<-c
|
|
}
|
|
}
|
|
|
|
func (m *model) fatal(err error) {
|
|
select {
|
|
case m.fatalChan <- err:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// StartDeadlockDetector starts a deadlock detector on the models locks which
|
|
// causes panics in case the locks cannot be acquired in the given timeout
|
|
// period.
|
|
func (m *model) StartDeadlockDetector(timeout time.Duration) {
|
|
l.Infof("Starting deadlock detector with %v timeout", timeout)
|
|
detector := newDeadlockDetector(timeout, m.evLogger, m.fatal)
|
|
detector.Watch("fmut", m.fmut)
|
|
detector.Watch("pmut", m.pmut)
|
|
}
|
|
|
|
// Need to hold lock on m.fmut when calling this.
|
|
func (m *model) addAndStartFolderLocked(cfg config.FolderConfiguration, fset *db.FileSet, cacheIgnoredFiles bool) {
|
|
ignores := ignore.New(cfg.Filesystem(nil), ignore.WithCache(cacheIgnoredFiles))
|
|
if cfg.Type != config.FolderTypeReceiveEncrypted {
|
|
if err := ignores.Load(".stignore"); err != nil && !fs.IsNotExist(err) {
|
|
l.Warnln("Loading ignores:", err)
|
|
}
|
|
}
|
|
|
|
m.addAndStartFolderLockedWithIgnores(cfg, fset, ignores)
|
|
}
|
|
|
|
// Only needed for testing, use addAndStartFolderLocked instead.
|
|
func (m *model) addAndStartFolderLockedWithIgnores(cfg config.FolderConfiguration, fset *db.FileSet, ignores *ignore.Matcher) {
|
|
m.folderCfgs[cfg.ID] = cfg
|
|
m.folderFiles[cfg.ID] = fset
|
|
m.folderIgnores[cfg.ID] = ignores
|
|
|
|
_, ok := m.folderRunners.Get(cfg.ID)
|
|
if ok {
|
|
l.Warnln("Cannot start already running folder", cfg.Description())
|
|
panic("cannot start already running folder")
|
|
}
|
|
|
|
folderFactory, ok := folderFactories[cfg.Type]
|
|
if !ok {
|
|
panic(fmt.Sprintf("unknown folder type 0x%x", cfg.Type))
|
|
}
|
|
|
|
folder := cfg.ID
|
|
|
|
// Find any devices for which we hold the index in the db, but the folder
|
|
// is not shared, and drop it.
|
|
expected := mapDevices(cfg.DeviceIDs())
|
|
for _, available := range fset.ListDevices() {
|
|
if _, ok := expected[available]; !ok {
|
|
l.Debugln("dropping", folder, "state for", available)
|
|
fset.Drop(available)
|
|
}
|
|
}
|
|
|
|
v, ok := fset.Sequence(protocol.LocalDeviceID), true
|
|
indexHasFiles := ok && v > 0
|
|
if !indexHasFiles {
|
|
// It's a blank folder, so this may the first time we're looking at
|
|
// it. Attempt to create and tag with our marker as appropriate. We
|
|
// don't really do anything with errors at this point except warn -
|
|
// if these things don't work, we still want to start the folder and
|
|
// it'll show up as errored later.
|
|
|
|
if err := cfg.CreateRoot(); err != nil {
|
|
l.Warnln("Failed to create folder root directory", err)
|
|
} else if err = cfg.CreateMarker(); err != nil {
|
|
l.Warnln("Failed to create folder marker:", err)
|
|
}
|
|
}
|
|
|
|
if cfg.Type == config.FolderTypeReceiveEncrypted {
|
|
if encryptionToken, err := readEncryptionToken(cfg); err == nil {
|
|
m.folderEncryptionPasswordTokens[folder] = encryptionToken
|
|
} else if !fs.IsNotExist(err) {
|
|
l.Warnf("Failed to read encryption token: %v", err)
|
|
}
|
|
}
|
|
|
|
// These are our metadata files, and they should always be hidden.
|
|
ffs := cfg.Filesystem(nil)
|
|
_ = ffs.Hide(config.DefaultMarkerName)
|
|
_ = ffs.Hide(versioner.DefaultPath)
|
|
_ = ffs.Hide(".stignore")
|
|
|
|
var ver versioner.Versioner
|
|
if cfg.Versioning.Type != "" {
|
|
var err error
|
|
ver, err = versioner.New(cfg)
|
|
if err != nil {
|
|
panic(fmt.Errorf("creating versioner: %w", err))
|
|
}
|
|
}
|
|
m.folderVersioners[folder] = ver
|
|
|
|
m.warnAboutOverwritingProtectedFiles(cfg, ignores)
|
|
|
|
p := folderFactory(m, fset, ignores, cfg, ver, m.evLogger, m.folderIOLimiter)
|
|
m.folderRunners.Add(folder, p)
|
|
|
|
l.Infof("Ready to synchronize %s (%s)", cfg.Description(), cfg.Type)
|
|
}
|
|
|
|
func (m *model) warnAboutOverwritingProtectedFiles(cfg config.FolderConfiguration, ignores *ignore.Matcher) {
|
|
if cfg.Type == config.FolderTypeSendOnly {
|
|
return
|
|
}
|
|
|
|
// This is a bit of a hack.
|
|
ffs := cfg.Filesystem(nil)
|
|
if ffs.Type() != fs.FilesystemTypeBasic {
|
|
return
|
|
}
|
|
folderLocation := ffs.URI()
|
|
|
|
var filesAtRisk []string
|
|
for _, protectedFilePath := range m.protectedFiles {
|
|
// check if file is synced in this folder
|
|
if protectedFilePath != folderLocation && !fs.IsParent(protectedFilePath, folderLocation) {
|
|
continue
|
|
}
|
|
|
|
// check if file is ignored
|
|
relPath, _ := filepath.Rel(folderLocation, protectedFilePath)
|
|
if ignores.Match(relPath).IsIgnored() {
|
|
continue
|
|
}
|
|
|
|
filesAtRisk = append(filesAtRisk, protectedFilePath)
|
|
}
|
|
|
|
if len(filesAtRisk) > 0 {
|
|
l.Warnln("Some protected files may be overwritten and cause issues. See https://docs.syncthing.net/users/config.html#syncing-configuration-files for more information. The at risk files are:", strings.Join(filesAtRisk, ", "))
|
|
}
|
|
}
|
|
|
|
func (m *model) removeFolder(cfg config.FolderConfiguration) {
|
|
m.fmut.RLock()
|
|
wait := m.folderRunners.RemoveAndWaitChan(cfg.ID, 0)
|
|
m.fmut.RUnlock()
|
|
<-wait
|
|
|
|
// We need to hold both fmut and pmut and must acquire locks in the same
|
|
// order always. (The locks can be *released* in any order.)
|
|
m.fmut.Lock()
|
|
m.pmut.RLock()
|
|
|
|
isPathUnique := true
|
|
for folderID, folderCfg := range m.folderCfgs {
|
|
if folderID != cfg.ID && folderCfg.Path == cfg.Path {
|
|
isPathUnique = false
|
|
break
|
|
}
|
|
}
|
|
if isPathUnique {
|
|
// Remove (if empty and removable) or move away (if non-empty or
|
|
// otherwise not removable) Syncthing-specific marker files.
|
|
fs := cfg.Filesystem(nil)
|
|
if err := fs.Remove(config.DefaultMarkerName); err != nil {
|
|
moved := config.DefaultMarkerName + time.Now().Format(".removed-20060102-150405")
|
|
_ = fs.Rename(config.DefaultMarkerName, moved)
|
|
}
|
|
}
|
|
|
|
m.cleanupFolderLocked(cfg)
|
|
m.indexHandlers.Each(func(_ protocol.DeviceID, r *indexHandlerRegistry) error {
|
|
r.Remove(cfg.ID)
|
|
return nil
|
|
})
|
|
|
|
m.fmut.Unlock()
|
|
m.pmut.RUnlock()
|
|
|
|
// Remove it from the database
|
|
db.DropFolder(m.db, cfg.ID)
|
|
}
|
|
|
|
// Need to hold lock on m.fmut when calling this.
|
|
func (m *model) cleanupFolderLocked(cfg config.FolderConfiguration) {
|
|
// clear up our config maps
|
|
delete(m.folderCfgs, cfg.ID)
|
|
delete(m.folderFiles, cfg.ID)
|
|
delete(m.folderIgnores, cfg.ID)
|
|
delete(m.folderVersioners, cfg.ID)
|
|
delete(m.folderEncryptionPasswordTokens, cfg.ID)
|
|
delete(m.folderEncryptionFailures, cfg.ID)
|
|
}
|
|
|
|
func (m *model) restartFolder(from, to config.FolderConfiguration, cacheIgnoredFiles bool) error {
|
|
if to.ID == "" {
|
|
panic("bug: cannot restart empty folder ID")
|
|
}
|
|
if to.ID != from.ID {
|
|
l.Warnf("bug: folder restart cannot change ID %q -> %q", from.ID, to.ID)
|
|
panic("bug: folder restart cannot change ID")
|
|
}
|
|
folder := to.ID
|
|
|
|
// This mutex protects the entirety of the restart operation, preventing
|
|
// there from being more than one folder restart operation in progress
|
|
// at any given time. The usual fmut/pmut stuff doesn't cover this,
|
|
// because those locks are released while we are waiting for the folder
|
|
// to shut down (and must be so because the folder might need them as
|
|
// part of its operations before shutting down).
|
|
restartMut := m.folderRestartMuts.Get(folder)
|
|
restartMut.Lock()
|
|
defer restartMut.Unlock()
|
|
|
|
m.fmut.RLock()
|
|
wait := m.folderRunners.RemoveAndWaitChan(from.ID, 0)
|
|
m.fmut.RUnlock()
|
|
<-wait
|
|
|
|
m.fmut.Lock()
|
|
defer m.fmut.Unlock()
|
|
|
|
// Cache the (maybe) existing fset before it's removed by cleanupFolderLocked
|
|
fset := m.folderFiles[folder]
|
|
fsetNil := fset == nil
|
|
|
|
m.cleanupFolderLocked(from)
|
|
if !to.Paused {
|
|
if fsetNil {
|
|
// Create a new fset. Might take a while and we do it under
|
|
// locking, but it's unsafe to create fset:s concurrently so
|
|
// that's the price we pay.
|
|
var err error
|
|
fset, err = db.NewFileSet(folder, m.db)
|
|
if err != nil {
|
|
return fmt.Errorf("restarting %v: %w", to.Description(), err)
|
|
}
|
|
}
|
|
m.addAndStartFolderLocked(to, fset, cacheIgnoredFiles)
|
|
}
|
|
|
|
// Care needs to be taken because we already hold fmut and the lock order
|
|
// must be the same everywhere. As fmut is acquired first, this is fine.
|
|
m.pmut.RLock()
|
|
runner, _ := m.folderRunners.Get(to.ID)
|
|
m.indexHandlers.Each(func(_ protocol.DeviceID, r *indexHandlerRegistry) error {
|
|
r.RegisterFolderState(to, fset, runner)
|
|
return nil
|
|
})
|
|
m.pmut.RUnlock()
|
|
|
|
var infoMsg string
|
|
switch {
|
|
case to.Paused:
|
|
infoMsg = "Paused"
|
|
case from.Paused:
|
|
infoMsg = "Unpaused"
|
|
default:
|
|
infoMsg = "Restarted"
|
|
}
|
|
l.Infof("%v folder %v (%v)", infoMsg, to.Description(), to.Type)
|
|
|
|
return nil
|
|
}
|
|
|
|
func (m *model) newFolder(cfg config.FolderConfiguration, cacheIgnoredFiles bool) error {
|
|
// Creating the fileset can take a long time (metadata calculation) so
|
|
// we do it outside of the lock.
|
|
fset, err := db.NewFileSet(cfg.ID, m.db)
|
|
if err != nil {
|
|
return fmt.Errorf("adding %v: %w", cfg.Description(), err)
|
|
}
|
|
|
|
m.fmut.Lock()
|
|
defer m.fmut.Unlock()
|
|
|
|
m.addAndStartFolderLocked(cfg, fset, cacheIgnoredFiles)
|
|
|
|
// Cluster configs might be received and processed before reaching this
|
|
// point, i.e. before the folder is started. If that's the case, start
|
|
// index senders here.
|
|
// Care needs to be taken because we already hold fmut and the lock order
|
|
// must be the same everywhere. As fmut is acquired first, this is fine.
|
|
m.pmut.RLock()
|
|
m.indexHandlers.Each(func(_ protocol.DeviceID, r *indexHandlerRegistry) error {
|
|
runner, _ := m.folderRunners.Get(cfg.ID)
|
|
r.RegisterFolderState(cfg, fset, runner)
|
|
return nil
|
|
})
|
|
m.pmut.RUnlock()
|
|
|
|
return nil
|
|
}
|
|
|
|
func (m *model) UsageReportingStats(report *contract.Report, version int, preview bool) {
|
|
if version >= 3 {
|
|
// Block stats
|
|
blockStatsMut.Lock()
|
|
for k, v := range blockStats {
|
|
switch k {
|
|
case "total":
|
|
report.BlockStats.Total = v
|
|
case "renamed":
|
|
report.BlockStats.Renamed = v
|
|
case "reused":
|
|
report.BlockStats.Reused = v
|
|
case "pulled":
|
|
report.BlockStats.Pulled = v
|
|
case "copyOrigin":
|
|
report.BlockStats.CopyOrigin = v
|
|
case "copyOriginShifted":
|
|
report.BlockStats.CopyOriginShifted = v
|
|
case "copyElsewhere":
|
|
report.BlockStats.CopyElsewhere = v
|
|
}
|
|
// Reset counts, as these are incremental
|
|
if !preview {
|
|
blockStats[k] = 0
|
|
}
|
|
}
|
|
blockStatsMut.Unlock()
|
|
|
|
// Transport stats
|
|
m.pmut.RLock()
|
|
for _, conn := range m.conn {
|
|
report.TransportStats[conn.Transport()]++
|
|
}
|
|
m.pmut.RUnlock()
|
|
|
|
// Ignore stats
|
|
var seenPrefix [3]bool
|
|
for folder := range m.cfg.Folders() {
|
|
lines, _, err := m.CurrentIgnores(folder)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
report.IgnoreStats.Lines += len(lines)
|
|
|
|
for _, line := range lines {
|
|
// Allow prefixes to be specified in any order, but only once.
|
|
for {
|
|
if strings.HasPrefix(line, "!") && !seenPrefix[0] {
|
|
seenPrefix[0] = true
|
|
line = line[1:]
|
|
report.IgnoreStats.Inverts++
|
|
} else if strings.HasPrefix(line, "(?i)") && !seenPrefix[1] {
|
|
seenPrefix[1] = true
|
|
line = line[4:]
|
|
report.IgnoreStats.Folded++
|
|
} else if strings.HasPrefix(line, "(?d)") && !seenPrefix[2] {
|
|
seenPrefix[2] = true
|
|
line = line[4:]
|
|
report.IgnoreStats.Deletable++
|
|
} else {
|
|
seenPrefix[0] = false
|
|
seenPrefix[1] = false
|
|
seenPrefix[2] = false
|
|
break
|
|
}
|
|
}
|
|
|
|
// Noops, remove
|
|
line = strings.TrimSuffix(line, "**")
|
|
line = strings.TrimPrefix(line, "**/")
|
|
|
|
if strings.HasPrefix(line, "/") {
|
|
report.IgnoreStats.Rooted++
|
|
} else if strings.HasPrefix(line, "#include ") {
|
|
report.IgnoreStats.Includes++
|
|
if strings.Contains(line, "..") {
|
|
report.IgnoreStats.EscapedIncludes++
|
|
}
|
|
}
|
|
|
|
if strings.Contains(line, "**") {
|
|
report.IgnoreStats.DoubleStars++
|
|
// Remove not to trip up star checks.
|
|
line = strings.ReplaceAll(line, "**", "")
|
|
}
|
|
|
|
if strings.Contains(line, "*") {
|
|
report.IgnoreStats.Stars++
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
type ConnectionInfo struct {
|
|
protocol.Statistics
|
|
Connected bool `json:"connected"`
|
|
Paused bool `json:"paused"`
|
|
Address string `json:"address"`
|
|
ClientVersion string `json:"clientVersion"`
|
|
Type string `json:"type"`
|
|
IsLocal bool `json:"isLocal"`
|
|
Crypto string `json:"crypto"`
|
|
}
|
|
|
|
// NumConnections returns the current number of active connected devices.
|
|
func (m *model) NumConnections() int {
|
|
m.pmut.RLock()
|
|
defer m.pmut.RUnlock()
|
|
return len(m.conn)
|
|
}
|
|
|
|
// ConnectionStats returns a map with connection statistics for each device.
|
|
func (m *model) ConnectionStats() map[string]interface{} {
|
|
m.pmut.RLock()
|
|
defer m.pmut.RUnlock()
|
|
|
|
res := make(map[string]interface{})
|
|
devs := m.cfg.Devices()
|
|
conns := make(map[string]ConnectionInfo, len(devs))
|
|
for device, deviceCfg := range devs {
|
|
hello := m.helloMessages[device]
|
|
versionString := hello.ClientVersion
|
|
if hello.ClientName != "syncthing" {
|
|
versionString = hello.ClientName + " " + hello.ClientVersion
|
|
}
|
|
ci := ConnectionInfo{
|
|
ClientVersion: strings.TrimSpace(versionString),
|
|
Paused: deviceCfg.Paused,
|
|
}
|
|
if conn, ok := m.conn[device]; ok {
|
|
ci.Type = conn.Type()
|
|
ci.IsLocal = conn.IsLocal()
|
|
ci.Crypto = conn.Crypto()
|
|
ci.Connected = ok
|
|
ci.Statistics = conn.Statistics()
|
|
if addr := conn.RemoteAddr(); addr != nil {
|
|
ci.Address = addr.String()
|
|
}
|
|
}
|
|
|
|
conns[device.String()] = ci
|
|
}
|
|
|
|
res["connections"] = conns
|
|
|
|
in, out := protocol.TotalInOut()
|
|
res["total"] = map[string]interface{}{
|
|
"at": time.Now().Truncate(time.Second),
|
|
"inBytesTotal": in,
|
|
"outBytesTotal": out,
|
|
}
|
|
|
|
return res
|
|
}
|
|
|
|
// DeviceStatistics returns statistics about each device
|
|
func (m *model) DeviceStatistics() (map[protocol.DeviceID]stats.DeviceStatistics, error) {
|
|
m.fmut.RLock()
|
|
defer m.fmut.RUnlock()
|
|
res := make(map[protocol.DeviceID]stats.DeviceStatistics, len(m.deviceStatRefs))
|
|
for id, sr := range m.deviceStatRefs {
|
|
stats, err := sr.GetStatistics()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
res[id] = stats
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
// FolderStatistics returns statistics about each folder
|
|
func (m *model) FolderStatistics() (map[string]stats.FolderStatistics, error) {
|
|
res := make(map[string]stats.FolderStatistics)
|
|
m.fmut.RLock()
|
|
defer m.fmut.RUnlock()
|
|
err := m.folderRunners.Each(func(id string, runner service) error {
|
|
stats, err := runner.GetStatistics()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
res[id] = stats
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
type FolderCompletion struct {
|
|
CompletionPct float64
|
|
GlobalBytes int64
|
|
NeedBytes int64
|
|
GlobalItems int
|
|
NeedItems int
|
|
NeedDeletes int
|
|
Sequence int64
|
|
RemoteState remoteFolderState
|
|
}
|
|
|
|
func newFolderCompletion(global, need db.Counts, sequence int64, state remoteFolderState) FolderCompletion {
|
|
comp := FolderCompletion{
|
|
GlobalBytes: global.Bytes,
|
|
NeedBytes: need.Bytes,
|
|
GlobalItems: global.Files + global.Directories + global.Symlinks,
|
|
NeedItems: need.Files + need.Directories + need.Symlinks,
|
|
NeedDeletes: need.Deleted,
|
|
Sequence: sequence,
|
|
RemoteState: state,
|
|
}
|
|
comp.setComplectionPct()
|
|
return comp
|
|
}
|
|
|
|
func (comp *FolderCompletion) add(other FolderCompletion) {
|
|
comp.GlobalBytes += other.GlobalBytes
|
|
comp.NeedBytes += other.NeedBytes
|
|
comp.GlobalItems += other.GlobalItems
|
|
comp.NeedItems += other.NeedItems
|
|
comp.NeedDeletes += other.NeedDeletes
|
|
comp.setComplectionPct()
|
|
}
|
|
|
|
func (comp *FolderCompletion) setComplectionPct() {
|
|
if comp.GlobalBytes == 0 {
|
|
comp.CompletionPct = 100
|
|
} else {
|
|
needRatio := float64(comp.NeedBytes) / float64(comp.GlobalBytes)
|
|
comp.CompletionPct = 100 * (1 - needRatio)
|
|
}
|
|
|
|
// If the completion is 100% but there are deletes we need to handle,
|
|
// drop it down a notch. Hack for consumers that look only at the
|
|
// percentage (our own GUI does the same calculation as here on its own
|
|
// and needs the same fixup).
|
|
if comp.NeedBytes == 0 && comp.NeedDeletes > 0 {
|
|
comp.CompletionPct = 95 // chosen by fair dice roll
|
|
}
|
|
}
|
|
|
|
// Map returns the members as a map, e.g. used in api to serialize as JSON.
|
|
func (comp *FolderCompletion) Map() map[string]interface{} {
|
|
return map[string]interface{}{
|
|
"completion": comp.CompletionPct,
|
|
"globalBytes": comp.GlobalBytes,
|
|
"needBytes": comp.NeedBytes,
|
|
"globalItems": comp.GlobalItems,
|
|
"needItems": comp.NeedItems,
|
|
"needDeletes": comp.NeedDeletes,
|
|
"sequence": comp.Sequence,
|
|
"remoteState": comp.RemoteState,
|
|
}
|
|
}
|
|
|
|
// Completion returns the completion status, in percent with some counters,
|
|
// for the given device and folder. The device can be any known device ID
|
|
// (including the local device) or explicitly protocol.LocalDeviceID. An
|
|
// empty folder string means the aggregate of all folders shared with the
|
|
// given device.
|
|
func (m *model) Completion(device protocol.DeviceID, folder string) (FolderCompletion, error) {
|
|
// The user specifically asked for our own device ID. Internally that is
|
|
// known as protocol.LocalDeviceID so translate.
|
|
if device == m.id {
|
|
device = protocol.LocalDeviceID
|
|
}
|
|
|
|
if folder != "" {
|
|
// We want completion for a specific folder.
|
|
return m.folderCompletion(device, folder)
|
|
}
|
|
|
|
// We want completion for all (shared) folders as an aggregate.
|
|
var comp FolderCompletion
|
|
for _, fcfg := range m.cfg.FolderList() {
|
|
if fcfg.Paused {
|
|
continue
|
|
}
|
|
if device == protocol.LocalDeviceID || fcfg.SharedWith(device) {
|
|
folderComp, err := m.folderCompletion(device, fcfg.ID)
|
|
if errors.Is(err, ErrFolderPaused) {
|
|
continue
|
|
} else if err != nil {
|
|
return FolderCompletion{}, err
|
|
}
|
|
comp.add(folderComp)
|
|
}
|
|
}
|
|
return comp, nil
|
|
}
|
|
|
|
func (m *model) folderCompletion(device protocol.DeviceID, folder string) (FolderCompletion, error) {
|
|
m.fmut.RLock()
|
|
err := m.checkFolderRunningLocked(folder)
|
|
rf := m.folderFiles[folder]
|
|
m.fmut.RUnlock()
|
|
if err != nil {
|
|
return FolderCompletion{}, err
|
|
}
|
|
|
|
snap, err := rf.Snapshot()
|
|
if err != nil {
|
|
return FolderCompletion{}, err
|
|
}
|
|
defer snap.Release()
|
|
|
|
m.pmut.RLock()
|
|
state := m.remoteFolderStates[device][folder]
|
|
downloaded := m.deviceDownloads[device].BytesDownloaded(folder)
|
|
m.pmut.RUnlock()
|
|
|
|
need := snap.NeedSize(device)
|
|
need.Bytes -= downloaded
|
|
// This might might be more than it really is, because some blocks can be of a smaller size.
|
|
if need.Bytes < 0 {
|
|
need.Bytes = 0
|
|
}
|
|
|
|
comp := newFolderCompletion(snap.GlobalSize(), need, snap.Sequence(device), state)
|
|
|
|
l.Debugf("%v Completion(%s, %q): %v", m, device, folder, comp.Map())
|
|
return comp, nil
|
|
}
|
|
|
|
// DBSnapshot returns a snapshot of the database content relevant to the given folder.
|
|
func (m *model) DBSnapshot(folder string) (*db.Snapshot, error) {
|
|
m.fmut.RLock()
|
|
err := m.checkFolderRunningLocked(folder)
|
|
rf := m.folderFiles[folder]
|
|
m.fmut.RUnlock()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return rf.Snapshot()
|
|
}
|
|
|
|
func (m *model) FolderProgressBytesCompleted(folder string) int64 {
|
|
return m.progressEmitter.BytesCompleted(folder)
|
|
}
|
|
|
|
// NeedFolderFiles returns paginated list of currently needed files in
|
|
// progress, queued, and to be queued on next puller iteration.
|
|
func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated, error) {
|
|
m.fmut.RLock()
|
|
rf, rfOk := m.folderFiles[folder]
|
|
runner, runnerOk := m.folderRunners.Get(folder)
|
|
cfg := m.folderCfgs[folder]
|
|
m.fmut.RUnlock()
|
|
|
|
if !rfOk {
|
|
return nil, nil, nil, ErrFolderMissing
|
|
}
|
|
|
|
snap, err := rf.Snapshot()
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
defer snap.Release()
|
|
var progress, queued, rest []db.FileInfoTruncated
|
|
var seen map[string]struct{}
|
|
|
|
p := newPager(page, perpage)
|
|
|
|
if runnerOk {
|
|
progressNames, queuedNames, skipped := runner.Jobs(page, perpage)
|
|
|
|
progress = make([]db.FileInfoTruncated, len(progressNames))
|
|
queued = make([]db.FileInfoTruncated, len(queuedNames))
|
|
seen = make(map[string]struct{}, len(progressNames)+len(queuedNames))
|
|
|
|
for i, name := range progressNames {
|
|
if f, ok := snap.GetGlobalTruncated(name); ok {
|
|
progress[i] = f
|
|
seen[name] = struct{}{}
|
|
}
|
|
}
|
|
|
|
for i, name := range queuedNames {
|
|
if f, ok := snap.GetGlobalTruncated(name); ok {
|
|
queued[i] = f
|
|
seen[name] = struct{}{}
|
|
}
|
|
}
|
|
|
|
p.get -= len(seen)
|
|
if p.get == 0 {
|
|
return progress, queued, nil, nil
|
|
}
|
|
p.toSkip -= skipped
|
|
}
|
|
|
|
rest = make([]db.FileInfoTruncated, 0, perpage)
|
|
snap.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
|
|
if cfg.IgnoreDelete && f.IsDeleted() {
|
|
return true
|
|
}
|
|
|
|
if p.skip() {
|
|
return true
|
|
}
|
|
ft := f.(db.FileInfoTruncated)
|
|
if _, ok := seen[ft.Name]; !ok {
|
|
rest = append(rest, ft)
|
|
p.get--
|
|
}
|
|
return p.get > 0
|
|
})
|
|
|
|
return progress, queued, rest, nil
|
|
}
|
|
|
|
// RemoteNeedFolderFiles returns paginated list of currently needed files for a
|
|
// remote device to become synced with a folder.
|
|
func (m *model) RemoteNeedFolderFiles(folder string, device protocol.DeviceID, page, perpage int) ([]db.FileInfoTruncated, error) {
|
|
m.fmut.RLock()
|
|
rf, ok := m.folderFiles[folder]
|
|
m.fmut.RUnlock()
|
|
|
|
if !ok {
|
|
return nil, ErrFolderMissing
|
|
}
|
|
|
|
snap, err := rf.Snapshot()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer snap.Release()
|
|
|
|
files := make([]db.FileInfoTruncated, 0, perpage)
|
|
p := newPager(page, perpage)
|
|
snap.WithNeedTruncated(device, func(f protocol.FileIntf) bool {
|
|
if p.skip() {
|
|
return true
|
|
}
|
|
files = append(files, f.(db.FileInfoTruncated))
|
|
return !p.done()
|
|
})
|
|
return files, nil
|
|
}
|
|
|
|
func (m *model) LocalChangedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, error) {
|
|
m.fmut.RLock()
|
|
rf, ok := m.folderFiles[folder]
|
|
m.fmut.RUnlock()
|
|
|
|
if !ok {
|
|
return nil, ErrFolderMissing
|
|
}
|
|
|
|
snap, err := rf.Snapshot()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer snap.Release()
|
|
|
|
if snap.ReceiveOnlyChangedSize().TotalItems() == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
p := newPager(page, perpage)
|
|
files := make([]db.FileInfoTruncated, 0, perpage)
|
|
|
|
snap.WithHaveTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
|
|
if !f.IsReceiveOnlyChanged() {
|
|
return true
|
|
}
|
|
if p.skip() {
|
|
return true
|
|
}
|
|
ft := f.(db.FileInfoTruncated)
|
|
files = append(files, ft)
|
|
return !p.done()
|
|
})
|
|
|
|
return files, nil
|
|
}
|
|
|
|
type pager struct {
|
|
toSkip, get int
|
|
}
|
|
|
|
func newPager(page, perpage int) *pager {
|
|
return &pager{
|
|
toSkip: (page - 1) * perpage,
|
|
get: perpage,
|
|
}
|
|
}
|
|
|
|
func (p *pager) skip() bool {
|
|
if p.toSkip == 0 {
|
|
return false
|
|
}
|
|
p.toSkip--
|
|
return true
|
|
}
|
|
|
|
func (p *pager) done() bool {
|
|
if p.get > 0 {
|
|
p.get--
|
|
}
|
|
return p.get == 0
|
|
}
|
|
|
|
// Index is called when a new device is connected and we receive their full index.
|
|
// Implements the protocol.Model interface.
|
|
func (m *model) Index(conn protocol.Connection, folder string, fs []protocol.FileInfo) error {
|
|
return m.handleIndex(conn, folder, fs, false)
|
|
}
|
|
|
|
// IndexUpdate is called for incremental updates to connected devices' indexes.
|
|
// Implements the protocol.Model interface.
|
|
func (m *model) IndexUpdate(conn protocol.Connection, folder string, fs []protocol.FileInfo) error {
|
|
return m.handleIndex(conn, folder, fs, true)
|
|
}
|
|
|
|
func (m *model) handleIndex(conn protocol.Connection, folder string, fs []protocol.FileInfo, update bool) error {
|
|
op := "Index"
|
|
if update {
|
|
op += " update"
|
|
}
|
|
|
|
deviceID := conn.DeviceID()
|
|
l.Debugf("%v (in): %s / %q: %d files", op, deviceID, folder, len(fs))
|
|
|
|
if cfg, ok := m.cfg.Folder(folder); !ok || !cfg.SharedWith(deviceID) {
|
|
l.Warnf("%v for unexpected folder ID %q sent from device %q; ensure that the folder exists and that this device is selected under \"Share With\" in the folder configuration.", op, folder, deviceID)
|
|
return fmt.Errorf("%s: %w", folder, ErrFolderMissing)
|
|
} else if cfg.Paused {
|
|
l.Debugf("%v for paused folder (ID %q) sent from device %q.", op, folder, deviceID)
|
|
return fmt.Errorf("%s: %w", folder, ErrFolderPaused)
|
|
}
|
|
|
|
m.pmut.RLock()
|
|
indexHandler, ok := m.indexHandlers.Get(deviceID)
|
|
m.pmut.RUnlock()
|
|
if !ok {
|
|
// This should be impossible, as an index handler always exists for an
|
|
// open connection, and this method can't be called on a closed
|
|
// connection
|
|
m.evLogger.Log(events.Failure, "index sender does not exist for connection on which indexes were received")
|
|
l.Debugf("%v for folder (ID %q) sent from device %q: missing index handler", op, folder, deviceID)
|
|
return fmt.Errorf("index handler missing: %s", folder)
|
|
}
|
|
|
|
return indexHandler.ReceiveIndex(folder, fs, update, op)
|
|
}
|
|
|
|
type clusterConfigDeviceInfo struct {
|
|
local, remote protocol.Device
|
|
}
|
|
|
|
type ClusterConfigReceivedEventData struct {
|
|
Device protocol.DeviceID `json:"device"`
|
|
}
|
|
|
|
func (m *model) ClusterConfig(conn protocol.Connection, cm protocol.ClusterConfig) error {
|
|
// Check the peer device's announced folders against our own. Emits events
|
|
// for folders that we don't expect (unknown or not shared).
|
|
// Also, collect a list of folders we do share, and if he's interested in
|
|
// temporary indexes, subscribe the connection.
|
|
|
|
deviceID := conn.DeviceID()
|
|
l.Debugf("Handling ClusterConfig from %v", deviceID.Short())
|
|
|
|
m.pmut.RLock()
|
|
indexHandlerRegistry, ok := m.indexHandlers.Get(deviceID)
|
|
m.pmut.RUnlock()
|
|
if !ok {
|
|
panic("bug: ClusterConfig called on closed or nonexistent connection")
|
|
}
|
|
|
|
deviceCfg, ok := m.cfg.Device(deviceID)
|
|
if !ok {
|
|
l.Debugln("Device disappeared from config while processing cluster-config")
|
|
return errDeviceUnknown
|
|
}
|
|
|
|
// Assemble the device information from the connected device about
|
|
// themselves and us for all folders.
|
|
ccDeviceInfos := make(map[string]*clusterConfigDeviceInfo, len(cm.Folders))
|
|
for _, folder := range cm.Folders {
|
|
info := &clusterConfigDeviceInfo{}
|
|
for _, dev := range folder.Devices {
|
|
if dev.ID == m.id {
|
|
info.local = dev
|
|
} else if dev.ID == deviceID {
|
|
info.remote = dev
|
|
}
|
|
if info.local.ID != protocol.EmptyDeviceID && info.remote.ID != protocol.EmptyDeviceID {
|
|
break
|
|
}
|
|
}
|
|
if info.remote.ID == protocol.EmptyDeviceID {
|
|
l.Infof("Device %v sent cluster-config without the device info for the remote on folder %v", deviceID, folder.Description())
|
|
return errMissingRemoteInClusterConfig
|
|
}
|
|
if info.local.ID == protocol.EmptyDeviceID {
|
|
l.Infof("Device %v sent cluster-config without the device info for us locally on folder %v", deviceID, folder.Description())
|
|
return errMissingLocalInClusterConfig
|
|
}
|
|
ccDeviceInfos[folder.ID] = info
|
|
}
|
|
|
|
for _, info := range ccDeviceInfos {
|
|
if deviceCfg.Introducer && info.local.Introducer {
|
|
l.Warnf("Remote %v is an introducer to us, and we are to them - only one should be introducer to the other, see https://docs.syncthing.net/users/introducer.html", deviceCfg.Description())
|
|
}
|
|
break
|
|
}
|
|
|
|
// Needs to happen outside of the fmut, as can cause CommitConfiguration
|
|
if deviceCfg.AutoAcceptFolders {
|
|
w, _ := m.cfg.Modify(func(cfg *config.Configuration) {
|
|
changedFcfg := make(map[string]config.FolderConfiguration)
|
|
haveFcfg := cfg.FolderMap()
|
|
for _, folder := range cm.Folders {
|
|
from, ok := haveFcfg[folder.ID]
|
|
if to, changed := m.handleAutoAccepts(deviceID, folder, ccDeviceInfos[folder.ID], from, ok, cfg.Defaults.Folder); changed {
|
|
changedFcfg[folder.ID] = to
|
|
}
|
|
}
|
|
if len(changedFcfg) == 0 {
|
|
return
|
|
}
|
|
for i := range cfg.Folders {
|
|
if fcfg, ok := changedFcfg[cfg.Folders[i].ID]; ok {
|
|
cfg.Folders[i] = fcfg
|
|
delete(changedFcfg, cfg.Folders[i].ID)
|
|
}
|
|
}
|
|
for _, fcfg := range changedFcfg {
|
|
cfg.Folders = append(cfg.Folders, fcfg)
|
|
}
|
|
})
|
|
// Need to wait for the waiter, as this calls CommitConfiguration,
|
|
// which sets up the folder and as we return from this call,
|
|
// ClusterConfig starts poking at m.folderFiles and other things
|
|
// that might not exist until the config is committed.
|
|
w.Wait()
|
|
}
|
|
|
|
tempIndexFolders, states, err := m.ccHandleFolders(cm.Folders, deviceCfg, ccDeviceInfos, indexHandlerRegistry)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
m.pmut.Lock()
|
|
m.remoteFolderStates[deviceID] = states
|
|
m.pmut.Unlock()
|
|
|
|
m.evLogger.Log(events.ClusterConfigReceived, ClusterConfigReceivedEventData{
|
|
Device: deviceID,
|
|
})
|
|
|
|
if len(tempIndexFolders) > 0 {
|
|
m.pmut.RLock()
|
|
conn, ok := m.conn[deviceID]
|
|
m.pmut.RUnlock()
|
|
// In case we've got ClusterConfig, and the connection disappeared
|
|
// from infront of our nose.
|
|
if ok {
|
|
m.progressEmitter.temporaryIndexSubscribe(conn, tempIndexFolders)
|
|
}
|
|
}
|
|
|
|
if deviceCfg.Introducer {
|
|
m.cfg.Modify(func(cfg *config.Configuration) {
|
|
folders, devices, foldersDevices, introduced := m.handleIntroductions(deviceCfg, cm, cfg.FolderMap(), cfg.DeviceMap())
|
|
folders, devices, deintroduced := m.handleDeintroductions(deviceCfg, foldersDevices, folders, devices)
|
|
if !introduced && !deintroduced {
|
|
return
|
|
}
|
|
cfg.Folders = make([]config.FolderConfiguration, 0, len(folders))
|
|
for _, fcfg := range folders {
|
|
cfg.Folders = append(cfg.Folders, fcfg)
|
|
}
|
|
cfg.Devices = make([]config.DeviceConfiguration, 0, len(devices))
|
|
for _, dcfg := range devices {
|
|
cfg.Devices = append(cfg.Devices, dcfg)
|
|
}
|
|
})
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (m *model) ccHandleFolders(folders []protocol.Folder, deviceCfg config.DeviceConfiguration, ccDeviceInfos map[string]*clusterConfigDeviceInfo, indexHandlers *indexHandlerRegistry) ([]string, map[string]remoteFolderState, error) {
|
|
var folderDevice config.FolderDeviceConfiguration
|
|
tempIndexFolders := make([]string, 0, len(folders))
|
|
seenFolders := make(map[string]remoteFolderState, len(folders))
|
|
updatedPending := make([]updatedPendingFolder, 0, len(folders))
|
|
deviceID := deviceCfg.DeviceID
|
|
expiredPending, err := m.db.PendingFoldersForDevice(deviceID)
|
|
if err != nil {
|
|
l.Infof("Could not get pending folders for cleanup: %v", err)
|
|
}
|
|
of := db.ObservedFolder{Time: time.Now().Truncate(time.Second)}
|
|
for _, folder := range folders {
|
|
seenFolders[folder.ID] = remoteFolderValid
|
|
|
|
cfg, ok := m.cfg.Folder(folder.ID)
|
|
if ok {
|
|
folderDevice, ok = cfg.Device(deviceID)
|
|
}
|
|
if !ok {
|
|
indexHandlers.Remove(folder.ID)
|
|
if deviceCfg.IgnoredFolder(folder.ID) {
|
|
l.Infof("Ignoring folder %s from device %s since we are configured to", folder.Description(), deviceID)
|
|
continue
|
|
}
|
|
delete(expiredPending, folder.ID)
|
|
of.Label = folder.Label
|
|
of.ReceiveEncrypted = len(ccDeviceInfos[folder.ID].local.EncryptionPasswordToken) > 0
|
|
of.RemoteEncrypted = len(ccDeviceInfos[folder.ID].remote.EncryptionPasswordToken) > 0
|
|
if err := m.db.AddOrUpdatePendingFolder(folder.ID, of, deviceID); err != nil {
|
|
l.Warnf("Failed to persist pending folder entry to database: %v", err)
|
|
}
|
|
if !folder.Paused {
|
|
indexHandlers.AddIndexInfo(folder.ID, ccDeviceInfos[folder.ID])
|
|
}
|
|
updatedPending = append(updatedPending, updatedPendingFolder{
|
|
FolderID: folder.ID,
|
|
FolderLabel: folder.Label,
|
|
DeviceID: deviceID,
|
|
ReceiveEncrypted: of.ReceiveEncrypted,
|
|
RemoteEncrypted: of.RemoteEncrypted,
|
|
})
|
|
// DEPRECATED: Only for backwards compatibility, should be removed.
|
|
m.evLogger.Log(events.FolderRejected, map[string]string{
|
|
"folder": folder.ID,
|
|
"folderLabel": folder.Label,
|
|
"device": deviceID.String(),
|
|
})
|
|
l.Infof("Unexpected folder %s sent from device %q; ensure that the folder exists and that this device is selected under \"Share With\" in the folder configuration.", folder.Description(), deviceID)
|
|
continue
|
|
}
|
|
|
|
if folder.Paused {
|
|
indexHandlers.Remove(folder.ID)
|
|
seenFolders[cfg.ID] = remoteFolderPaused
|
|
continue
|
|
}
|
|
|
|
if cfg.Paused {
|
|
indexHandlers.AddIndexInfo(folder.ID, ccDeviceInfos[folder.ID])
|
|
continue
|
|
}
|
|
|
|
if err := m.ccCheckEncryption(cfg, folderDevice, ccDeviceInfos[folder.ID], deviceCfg.Untrusted); err != nil {
|
|
sameError := false
|
|
m.fmut.Lock()
|
|
if devs, ok := m.folderEncryptionFailures[folder.ID]; ok {
|
|
sameError = devs[deviceID] == err
|
|
} else {
|
|
m.folderEncryptionFailures[folder.ID] = make(map[protocol.DeviceID]error)
|
|
}
|
|
m.folderEncryptionFailures[folder.ID][deviceID] = err
|
|
m.fmut.Unlock()
|
|
msg := fmt.Sprintf("Failure checking encryption consistency with device %v for folder %v: %v", deviceID, cfg.Description(), err)
|
|
if sameError {
|
|
l.Debugln(msg)
|
|
} else {
|
|
if rerr, ok := err.(*redactedError); ok {
|
|
err = rerr.redacted
|
|
}
|
|
m.evLogger.Log(events.Failure, err.Error())
|
|
l.Warnln(msg)
|
|
}
|
|
return tempIndexFolders, seenFolders, err
|
|
}
|
|
m.fmut.Lock()
|
|
if devErrs, ok := m.folderEncryptionFailures[folder.ID]; ok {
|
|
if len(devErrs) == 1 {
|
|
delete(m.folderEncryptionFailures, folder.ID)
|
|
} else {
|
|
delete(m.folderEncryptionFailures[folder.ID], deviceID)
|
|
}
|
|
}
|
|
m.fmut.Unlock()
|
|
|
|
// Handle indexes
|
|
|
|
if !folder.DisableTempIndexes {
|
|
tempIndexFolders = append(tempIndexFolders, folder.ID)
|
|
}
|
|
|
|
indexHandlers.AddIndexInfo(folder.ID, ccDeviceInfos[folder.ID])
|
|
}
|
|
|
|
indexHandlers.RemoveAllExcept(seenFolders)
|
|
|
|
// Explicitly mark folders we offer, but the remote has not accepted
|
|
for folderID, cfg := range m.cfg.Folders() {
|
|
if _, seen := seenFolders[folderID]; !seen && cfg.SharedWith(deviceID) {
|
|
l.Debugf("Remote device %v has not accepted sharing folder %s", deviceID.Short(), cfg.Description())
|
|
seenFolders[folderID] = remoteFolderNotSharing
|
|
}
|
|
}
|
|
|
|
expiredPendingList := make([]map[string]string, 0, len(expiredPending))
|
|
for folder := range expiredPending {
|
|
if err = m.db.RemovePendingFolderForDevice(folder, deviceID); err != nil {
|
|
msg := "Failed to remove pending folder-device entry"
|
|
l.Warnf("%v (%v, %v): %v", msg, folder, deviceID, err)
|
|
m.evLogger.Log(events.Failure, msg)
|
|
continue
|
|
}
|
|
expiredPendingList = append(expiredPendingList, map[string]string{
|
|
"folderID": folder,
|
|
"deviceID": deviceID.String(),
|
|
})
|
|
}
|
|
if len(updatedPending) > 0 || len(expiredPendingList) > 0 {
|
|
m.evLogger.Log(events.PendingFoldersChanged, map[string]interface{}{
|
|
"added": updatedPending,
|
|
"removed": expiredPendingList,
|
|
})
|
|
}
|
|
|
|
return tempIndexFolders, seenFolders, nil
|
|
}
|
|
|
|
func (m *model) ccCheckEncryption(fcfg config.FolderConfiguration, folderDevice config.FolderDeviceConfiguration, ccDeviceInfos *clusterConfigDeviceInfo, deviceUntrusted bool) error {
|
|
hasTokenRemote := len(ccDeviceInfos.remote.EncryptionPasswordToken) > 0
|
|
hasTokenLocal := len(ccDeviceInfos.local.EncryptionPasswordToken) > 0
|
|
isEncryptedRemote := folderDevice.EncryptionPassword != ""
|
|
isEncryptedLocal := fcfg.Type == config.FolderTypeReceiveEncrypted
|
|
|
|
if !isEncryptedRemote && !isEncryptedLocal && deviceUntrusted {
|
|
return errEncryptionNotEncryptedUntrusted
|
|
}
|
|
|
|
if !(hasTokenRemote || hasTokenLocal || isEncryptedRemote || isEncryptedLocal) {
|
|
// No one cares about encryption here
|
|
return nil
|
|
}
|
|
|
|
if isEncryptedRemote && isEncryptedLocal {
|
|
// Should never happen, but config racyness and be safe.
|
|
return errEncryptionInvConfigLocal
|
|
}
|
|
|
|
if hasTokenRemote && hasTokenLocal {
|
|
return errEncryptionInvConfigRemote
|
|
}
|
|
|
|
if !(hasTokenRemote || hasTokenLocal) {
|
|
if isEncryptedRemote {
|
|
return errEncryptionPlainForRemoteEncrypted
|
|
} else {
|
|
return errEncryptionPlainForReceiveEncrypted
|
|
}
|
|
}
|
|
|
|
if !(isEncryptedRemote || isEncryptedLocal) {
|
|
return errEncryptionNotEncryptedLocal
|
|
}
|
|
|
|
if isEncryptedRemote {
|
|
passwordToken := protocol.PasswordToken(m.keyGen, fcfg.ID, folderDevice.EncryptionPassword)
|
|
match := false
|
|
if hasTokenLocal {
|
|
match = bytes.Equal(passwordToken, ccDeviceInfos.local.EncryptionPasswordToken)
|
|
} else {
|
|
// hasTokenRemote == true
|
|
match = bytes.Equal(passwordToken, ccDeviceInfos.remote.EncryptionPasswordToken)
|
|
}
|
|
if !match {
|
|
return errEncryptionPassword
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// isEncryptedLocal == true
|
|
|
|
var ccToken []byte
|
|
if hasTokenLocal {
|
|
ccToken = ccDeviceInfos.local.EncryptionPasswordToken
|
|
} else {
|
|
// hasTokenRemote == true
|
|
ccToken = ccDeviceInfos.remote.EncryptionPasswordToken
|
|
}
|
|
m.fmut.RLock()
|
|
token, ok := m.folderEncryptionPasswordTokens[fcfg.ID]
|
|
m.fmut.RUnlock()
|
|
if !ok {
|
|
var err error
|
|
token, err = readEncryptionToken(fcfg)
|
|
if err != nil && !fs.IsNotExist(err) {
|
|
if rerr, ok := redactPathError(err); ok {
|
|
return rerr
|
|
}
|
|
return &redactedError{
|
|
error: err,
|
|
redacted: errEncryptionTokenRead,
|
|
}
|
|
}
|
|
if err == nil {
|
|
m.fmut.Lock()
|
|
m.folderEncryptionPasswordTokens[fcfg.ID] = token
|
|
m.fmut.Unlock()
|
|
} else {
|
|
if err := writeEncryptionToken(ccToken, fcfg); err != nil {
|
|
if rerr, ok := redactPathError(err); ok {
|
|
return rerr
|
|
} else {
|
|
return &redactedError{
|
|
error: err,
|
|
redacted: errEncryptionTokenWrite,
|
|
}
|
|
}
|
|
}
|
|
m.fmut.Lock()
|
|
m.folderEncryptionPasswordTokens[fcfg.ID] = ccToken
|
|
m.fmut.Unlock()
|
|
// We can only announce ourselves once we have the token,
|
|
// thus we need to resend CCs now that we have it.
|
|
m.sendClusterConfig(fcfg.DeviceIDs())
|
|
return nil
|
|
}
|
|
}
|
|
if !bytes.Equal(token, ccToken) {
|
|
return errEncryptionPassword
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m *model) sendClusterConfig(ids []protocol.DeviceID) {
|
|
if len(ids) == 0 {
|
|
return
|
|
}
|
|
ccConns := make([]protocol.Connection, 0, len(ids))
|
|
m.pmut.RLock()
|
|
for _, id := range ids {
|
|
if conn, ok := m.conn[id]; ok {
|
|
ccConns = append(ccConns, conn)
|
|
}
|
|
}
|
|
m.pmut.RUnlock()
|
|
// Generating cluster-configs acquires fmut -> must happen outside of pmut.
|
|
for _, conn := range ccConns {
|
|
cm, passwords := m.generateClusterConfig(conn.DeviceID())
|
|
conn.SetFolderPasswords(passwords)
|
|
go conn.ClusterConfig(cm)
|
|
}
|
|
}
|
|
|
|
// handleIntroductions handles adding devices/folders that are shared by an introducer device
|
|
func (m *model) handleIntroductions(introducerCfg config.DeviceConfiguration, cm protocol.ClusterConfig, folders map[string]config.FolderConfiguration, devices map[protocol.DeviceID]config.DeviceConfiguration) (map[string]config.FolderConfiguration, map[protocol.DeviceID]config.DeviceConfiguration, folderDeviceSet, bool) {
|
|
changed := false
|
|
|
|
foldersDevices := make(folderDeviceSet)
|
|
|
|
for _, folder := range cm.Folders {
|
|
// Adds devices which we do not have, but the introducer has
|
|
// for the folders that we have in common. Also, shares folders
|
|
// with devices that we have in common, yet are currently not sharing
|
|
// the folder.
|
|
|
|
fcfg, ok := folders[folder.ID]
|
|
if !ok {
|
|
// Don't have this folder, carry on.
|
|
continue
|
|
}
|
|
|
|
folderChanged := false
|
|
|
|
for _, device := range folder.Devices {
|
|
// No need to share with self.
|
|
if device.ID == m.id {
|
|
continue
|
|
}
|
|
|
|
foldersDevices.set(device.ID, folder.ID)
|
|
|
|
if _, ok := devices[device.ID]; !ok {
|
|
// The device is currently unknown. Add it to the config.
|
|
devices[device.ID] = m.introduceDevice(device, introducerCfg)
|
|
} else if fcfg.SharedWith(device.ID) {
|
|
// We already share the folder with this device, so
|
|
// nothing to do.
|
|
continue
|
|
}
|
|
|
|
if fcfg.Type != config.FolderTypeReceiveEncrypted && device.EncryptionPasswordToken != nil {
|
|
l.Infof("Cannot share folder %s with %v because the introducer %v encrypts data, which requires a password", folder.Description(), device.ID, introducerCfg.DeviceID)
|
|
continue
|
|
}
|
|
|
|
// We don't yet share this folder with this device. Add the device
|
|
// to sharing list of the folder.
|
|
l.Infof("Sharing folder %s with %v (vouched for by introducer %v)", folder.Description(), device.ID, introducerCfg.DeviceID)
|
|
fcfg.Devices = append(fcfg.Devices, config.FolderDeviceConfiguration{
|
|
DeviceID: device.ID,
|
|
IntroducedBy: introducerCfg.DeviceID,
|
|
})
|
|
folderChanged = true
|
|
}
|
|
|
|
if folderChanged {
|
|
folders[fcfg.ID] = fcfg
|
|
changed = true
|
|
}
|
|
}
|
|
|
|
return folders, devices, foldersDevices, changed
|
|
}
|
|
|
|
// handleDeintroductions handles removals of devices/shares that are removed by an introducer device
|
|
func (*model) handleDeintroductions(introducerCfg config.DeviceConfiguration, foldersDevices folderDeviceSet, folders map[string]config.FolderConfiguration, devices map[protocol.DeviceID]config.DeviceConfiguration) (map[string]config.FolderConfiguration, map[protocol.DeviceID]config.DeviceConfiguration, bool) {
|
|
if introducerCfg.SkipIntroductionRemovals {
|
|
return folders, devices, false
|
|
}
|
|
|
|
changed := false
|
|
devicesNotIntroduced := make(map[protocol.DeviceID]struct{})
|
|
|
|
// Check if we should unshare some folders, if the introducer has unshared them.
|
|
for folderID, folderCfg := range folders {
|
|
for k := 0; k < len(folderCfg.Devices); k++ {
|
|
if folderCfg.Devices[k].IntroducedBy != introducerCfg.DeviceID {
|
|
devicesNotIntroduced[folderCfg.Devices[k].DeviceID] = struct{}{}
|
|
continue
|
|
}
|
|
if !foldersDevices.has(folderCfg.Devices[k].DeviceID, folderCfg.ID) {
|
|
// We could not find that folder shared on the
|
|
// introducer with the device that was introduced to us.
|
|
// We should follow and unshare as well.
|
|
l.Infof("Unsharing folder %s with %v as introducer %v no longer shares the folder with that device", folderCfg.Description(), folderCfg.Devices[k].DeviceID, folderCfg.Devices[k].IntroducedBy)
|
|
folderCfg.Devices = append(folderCfg.Devices[:k], folderCfg.Devices[k+1:]...)
|
|
folders[folderID] = folderCfg
|
|
k--
|
|
changed = true
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check if we should remove some devices, if the introducer no longer
|
|
// shares any folder with them. Yet do not remove if we share other
|
|
// folders that haven't been introduced by the introducer.
|
|
for deviceID, device := range devices {
|
|
if device.IntroducedBy == introducerCfg.DeviceID {
|
|
if !foldersDevices.hasDevice(deviceID) {
|
|
if _, ok := devicesNotIntroduced[deviceID]; !ok {
|
|
// The introducer no longer shares any folder with the
|
|
// device, remove the device.
|
|
l.Infof("Removing device %v as introducer %v no longer shares any folders with that device", deviceID, device.IntroducedBy)
|
|
changed = true
|
|
delete(devices, deviceID)
|
|
continue
|
|
}
|
|
l.Infof("Would have removed %v as %v no longer shares any folders, yet there are other folders that are shared with this device that haven't been introduced by this introducer.", deviceID, device.IntroducedBy)
|
|
}
|
|
}
|
|
}
|
|
|
|
return folders, devices, changed
|
|
}
|
|
|
|
// handleAutoAccepts handles adding and sharing folders for devices that have
|
|
// AutoAcceptFolders set to true.
|
|
func (m *model) handleAutoAccepts(deviceID protocol.DeviceID, folder protocol.Folder, ccDeviceInfos *clusterConfigDeviceInfo, cfg config.FolderConfiguration, haveCfg bool, defaultFolderCfg config.FolderConfiguration) (config.FolderConfiguration, bool) {
|
|
if !haveCfg {
|
|
defaultPathFs := fs.NewFilesystem(defaultFolderCfg.FilesystemType, defaultFolderCfg.Path)
|
|
var pathAlternatives []string
|
|
if alt := fs.SanitizePath(folder.Label); alt != "" {
|
|
pathAlternatives = append(pathAlternatives, alt)
|
|
}
|
|
if alt := fs.SanitizePath(folder.ID); alt != "" {
|
|
pathAlternatives = append(pathAlternatives, alt)
|
|
}
|
|
if len(pathAlternatives) == 0 {
|
|
l.Infof("Failed to auto-accept folder %s from %s due to lack of path alternatives", folder.Description(), deviceID)
|
|
return config.FolderConfiguration{}, false
|
|
}
|
|
for _, path := range pathAlternatives {
|
|
// Make sure the folder path doesn't already exist.
|
|
if _, err := defaultPathFs.Lstat(path); !fs.IsNotExist(err) {
|
|
continue
|
|
}
|
|
|
|
// Attempt to create it to make sure it does, now.
|
|
fullPath := filepath.Join(defaultFolderCfg.Path, path)
|
|
if err := defaultPathFs.MkdirAll(path, 0o700); err != nil {
|
|
l.Warnf("Failed to create path for auto-accepted folder %s at path %s: %v", folder.Description(), fullPath, err)
|
|
continue
|
|
}
|
|
|
|
fcfg := newFolderConfiguration(m.cfg, folder.ID, folder.Label, defaultFolderCfg.FilesystemType, fullPath)
|
|
fcfg.Devices = append(fcfg.Devices, config.FolderDeviceConfiguration{
|
|
DeviceID: deviceID,
|
|
})
|
|
|
|
if len(ccDeviceInfos.remote.EncryptionPasswordToken) > 0 || len(ccDeviceInfos.local.EncryptionPasswordToken) > 0 {
|
|
fcfg.Type = config.FolderTypeReceiveEncrypted
|
|
// Override the user-configured defaults, as normally done by the GUI
|
|
fcfg.FSWatcherEnabled = false
|
|
if fcfg.RescanIntervalS != 0 {
|
|
minRescanInterval := 3600 * 24
|
|
if fcfg.RescanIntervalS < minRescanInterval {
|
|
fcfg.RescanIntervalS = minRescanInterval
|
|
}
|
|
}
|
|
fcfg.Versioning.Reset()
|
|
// Other necessary settings are ensured by FolderConfiguration itself
|
|
} else {
|
|
ignores := m.cfg.DefaultIgnores()
|
|
if err := m.setIgnores(fcfg, ignores.Lines); err != nil {
|
|
l.Warnf("Failed to apply default ignores to auto-accepted folder %s at path %s: %v", folder.Description(), fcfg.Path, err)
|
|
}
|
|
}
|
|
|
|
l.Infof("Auto-accepted %s folder %s at path %s", deviceID, folder.Description(), fcfg.Path)
|
|
return fcfg, true
|
|
}
|
|
l.Infof("Failed to auto-accept folder %s from %s due to path conflict", folder.Description(), deviceID)
|
|
return config.FolderConfiguration{}, false
|
|
} else {
|
|
for _, device := range cfg.DeviceIDs() {
|
|
if device == deviceID {
|
|
// Already shared nothing todo.
|
|
return config.FolderConfiguration{}, false
|
|
}
|
|
}
|
|
if cfg.Type == config.FolderTypeReceiveEncrypted {
|
|
if len(ccDeviceInfos.remote.EncryptionPasswordToken) == 0 && len(ccDeviceInfos.local.EncryptionPasswordToken) == 0 {
|
|
l.Infof("Failed to auto-accept device %s on existing folder %s as the remote wants to send us unencrypted data, but the folder type is receive-encrypted", folder.Description(), deviceID)
|
|
return config.FolderConfiguration{}, false
|
|
}
|
|
} else {
|
|
if len(ccDeviceInfos.remote.EncryptionPasswordToken) > 0 || len(ccDeviceInfos.local.EncryptionPasswordToken) > 0 {
|
|
l.Infof("Failed to auto-accept device %s on existing folder %s as the remote wants to send us encrypted data, but the folder type is not receive-encrypted", folder.Description(), deviceID)
|
|
return config.FolderConfiguration{}, false
|
|
}
|
|
}
|
|
cfg.Devices = append(cfg.Devices, config.FolderDeviceConfiguration{
|
|
DeviceID: deviceID,
|
|
})
|
|
l.Infof("Shared %s with %s due to auto-accept", folder.ID, deviceID)
|
|
return cfg, true
|
|
}
|
|
}
|
|
|
|
func (m *model) introduceDevice(device protocol.Device, introducerCfg config.DeviceConfiguration) config.DeviceConfiguration {
|
|
addresses := []string{"dynamic"}
|
|
for _, addr := range device.Addresses {
|
|
if addr != "dynamic" {
|
|
addresses = append(addresses, addr)
|
|
}
|
|
}
|
|
|
|
l.Infof("Adding device %v to config (vouched for by introducer %v)", device.ID, introducerCfg.DeviceID)
|
|
newDeviceCfg := m.cfg.DefaultDevice()
|
|
newDeviceCfg.DeviceID = device.ID
|
|
newDeviceCfg.Name = device.Name
|
|
newDeviceCfg.Compression = introducerCfg.Compression
|
|
newDeviceCfg.Addresses = addresses
|
|
newDeviceCfg.CertName = device.CertName
|
|
newDeviceCfg.IntroducedBy = introducerCfg.DeviceID
|
|
|
|
// The introducers' introducers are also our introducers.
|
|
if device.Introducer {
|
|
l.Infof("Device %v is now also an introducer", device.ID)
|
|
newDeviceCfg.Introducer = true
|
|
newDeviceCfg.SkipIntroductionRemovals = device.SkipIntroductionRemovals
|
|
}
|
|
|
|
return newDeviceCfg
|
|
}
|
|
|
|
// Closed is called when a connection has been closed
|
|
func (m *model) Closed(conn protocol.Connection, err error) {
|
|
device := conn.DeviceID()
|
|
m.pmut.Lock()
|
|
conn, ok := m.conn[device]
|
|
if !ok {
|
|
m.pmut.Unlock()
|
|
return
|
|
}
|
|
|
|
delete(m.conn, device)
|
|
delete(m.connRequestLimiters, device)
|
|
delete(m.helloMessages, device)
|
|
delete(m.deviceDownloads, device)
|
|
delete(m.remoteFolderStates, device)
|
|
closed := m.closed[device]
|
|
delete(m.closed, device)
|
|
wait := m.indexHandlers.RemoveAndWaitChan(device, 0)
|
|
m.pmut.Unlock()
|
|
<-wait
|
|
|
|
m.progressEmitter.temporaryIndexUnsubscribe(conn)
|
|
m.deviceDidClose(device, time.Since(conn.EstablishedAt()))
|
|
|
|
l.Infof("Connection to %s at %s closed: %v", device, conn, err)
|
|
m.evLogger.Log(events.DeviceDisconnected, map[string]string{
|
|
"id": device.String(),
|
|
"error": err.Error(),
|
|
})
|
|
close(closed)
|
|
}
|
|
|
|
// Implements protocol.RequestResponse
|
|
type requestResponse struct {
|
|
data []byte
|
|
closed chan struct{}
|
|
once stdsync.Once
|
|
}
|
|
|
|
func newRequestResponse(size int) *requestResponse {
|
|
return &requestResponse{
|
|
data: protocol.BufferPool.Get(size),
|
|
closed: make(chan struct{}),
|
|
}
|
|
}
|
|
|
|
func (r *requestResponse) Data() []byte {
|
|
return r.data
|
|
}
|
|
|
|
func (r *requestResponse) Close() {
|
|
r.once.Do(func() {
|
|
protocol.BufferPool.Put(r.data)
|
|
close(r.closed)
|
|
})
|
|
}
|
|
|
|
func (r *requestResponse) Wait() {
|
|
<-r.closed
|
|
}
|
|
|
|
// Request returns the specified data segment by reading it from local disk.
|
|
// Implements the protocol.Model interface.
|
|
func (m *model) Request(conn protocol.Connection, folder, name string, _, size int32, offset int64, hash []byte, weakHash uint32, fromTemporary bool) (out protocol.RequestResponse, err error) {
|
|
if size < 0 || offset < 0 {
|
|
return nil, protocol.ErrInvalid
|
|
}
|
|
|
|
deviceID := conn.DeviceID()
|
|
|
|
m.fmut.RLock()
|
|
folderCfg, ok := m.folderCfgs[folder]
|
|
folderIgnores := m.folderIgnores[folder]
|
|
m.fmut.RUnlock()
|
|
if !ok {
|
|
// The folder might be already unpaused in the config, but not yet
|
|
// in the model.
|
|
l.Debugf("Request from %s for file %s in unstarted folder %q", deviceID, name, folder)
|
|
return nil, protocol.ErrGeneric
|
|
}
|
|
|
|
if !folderCfg.SharedWith(deviceID) {
|
|
l.Warnf("Request from %s for file %s in unshared folder %q", deviceID, name, folder)
|
|
return nil, protocol.ErrGeneric
|
|
}
|
|
if folderCfg.Paused {
|
|
l.Debugf("Request from %s for file %s in paused folder %q", deviceID, name, folder)
|
|
return nil, protocol.ErrGeneric
|
|
}
|
|
|
|
// Make sure the path is valid and in canonical form
|
|
if name, err = fs.Canonicalize(name); err != nil {
|
|
l.Debugf("Request from %s in folder %q for invalid filename %s", deviceID, folder, name)
|
|
return nil, protocol.ErrGeneric
|
|
}
|
|
|
|
if deviceID != protocol.LocalDeviceID {
|
|
l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d t=%v", m, deviceID, folder, name, offset, size, fromTemporary)
|
|
}
|
|
|
|
if fs.IsInternal(name) {
|
|
l.Debugf("%v REQ(in) for internal file: %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, size)
|
|
return nil, protocol.ErrInvalid
|
|
}
|
|
|
|
if folderIgnores.Match(name).IsIgnored() {
|
|
l.Debugf("%v REQ(in) for ignored file: %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, size)
|
|
return nil, protocol.ErrInvalid
|
|
}
|
|
|
|
// Restrict parallel requests by connection/device
|
|
|
|
m.pmut.RLock()
|
|
limiter := m.connRequestLimiters[deviceID]
|
|
m.pmut.RUnlock()
|
|
|
|
// The requestResponse releases the bytes to the buffer pool and the
|
|
// limiters when its Close method is called.
|
|
res := newLimitedRequestResponse(int(size), limiter, m.globalRequestLimiter)
|
|
|
|
defer func() {
|
|
// Close it ourselves if it isn't returned due to an error
|
|
if err != nil {
|
|
res.Close()
|
|
}
|
|
}()
|
|
|
|
// Grab the FS after limiting, as it causes I/O and we want to minimize
|
|
// the race time between the symlink check and the read.
|
|
|
|
folderFs := folderCfg.Filesystem(nil)
|
|
|
|
if err := osutil.TraversesSymlink(folderFs, filepath.Dir(name)); err != nil {
|
|
l.Debugf("%v REQ(in) traversal check: %s - %s: %q / %q o=%d s=%d", m, err, deviceID, folder, name, offset, size)
|
|
return nil, protocol.ErrNoSuchFile
|
|
}
|
|
|
|
// Only check temp files if the flag is set, and if we are set to advertise
|
|
// the temp indexes.
|
|
if fromTemporary && !folderCfg.DisableTempIndexes {
|
|
tempFn := fs.TempName(name)
|
|
|
|
if info, err := folderFs.Lstat(tempFn); err != nil || !info.IsRegular() {
|
|
// Reject reads for anything that doesn't exist or is something
|
|
// other than a regular file.
|
|
l.Debugf("%v REQ(in) failed stating temp file (%v): %s: %q / %q o=%d s=%d", m, err, deviceID, folder, name, offset, size)
|
|
return nil, protocol.ErrNoSuchFile
|
|
}
|
|
_, err := readOffsetIntoBuf(folderFs, tempFn, offset, res.data)
|
|
if err == nil && scanner.Validate(res.data, hash, weakHash) {
|
|
return res, nil
|
|
}
|
|
// Fall through to reading from a non-temp file, just in case the temp
|
|
// file has finished downloading.
|
|
}
|
|
|
|
if info, err := folderFs.Lstat(name); err != nil || !info.IsRegular() {
|
|
// Reject reads for anything that doesn't exist or is something
|
|
// other than a regular file.
|
|
l.Debugf("%v REQ(in) failed stating file (%v): %s: %q / %q o=%d s=%d", m, err, deviceID, folder, name, offset, size)
|
|
return nil, protocol.ErrNoSuchFile
|
|
}
|
|
|
|
n, err := readOffsetIntoBuf(folderFs, name, offset, res.data)
|
|
if fs.IsNotExist(err) {
|
|
l.Debugf("%v REQ(in) file doesn't exist: %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, size)
|
|
return nil, protocol.ErrNoSuchFile
|
|
} else if err == io.EOF {
|
|
// Read beyond end of file. This might indicate a problem, or it
|
|
// might be a short block that gets padded when read for encrypted
|
|
// folders. We ignore the error and let the hash validation in the
|
|
// next step take care of it, by only hashing the part we actually
|
|
// managed to read.
|
|
} else if err != nil {
|
|
l.Debugf("%v REQ(in) failed reading file (%v): %s: %q / %q o=%d s=%d", m, err, deviceID, folder, name, offset, size)
|
|
return nil, protocol.ErrGeneric
|
|
}
|
|
|
|
if folderCfg.Type != config.FolderTypeReceiveEncrypted && len(hash) > 0 && !scanner.Validate(res.data[:n], hash, weakHash) {
|
|
m.recheckFile(deviceID, folder, name, offset, hash, weakHash)
|
|
l.Debugf("%v REQ(in) failed validating data: %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, size)
|
|
return nil, protocol.ErrNoSuchFile
|
|
}
|
|
|
|
return res, nil
|
|
}
|
|
|
|
// newLimitedRequestResponse takes size bytes from the limiters in order,
|
|
// skipping nil limiters, then returns a requestResponse of the given size.
|
|
// When the requestResponse is closed the limiters are given back the bytes,
|
|
// in reverse order.
|
|
func newLimitedRequestResponse(size int, limiters ...*semaphore.Semaphore) *requestResponse {
|
|
multi := semaphore.MultiSemaphore(limiters)
|
|
multi.Take(size)
|
|
|
|
res := newRequestResponse(size)
|
|
|
|
go func() {
|
|
res.Wait()
|
|
multi.Give(size)
|
|
}()
|
|
|
|
return res
|
|
}
|
|
|
|
func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, weakHash uint32) {
|
|
cf, ok, err := m.CurrentFolderFile(folder, name)
|
|
if err != nil {
|
|
l.Debugf("%v recheckFile: %s: %q / %q: current file error: %v", m, deviceID, folder, name, err)
|
|
return
|
|
}
|
|
if !ok {
|
|
l.Debugf("%v recheckFile: %s: %q / %q: no current file", m, deviceID, folder, name)
|
|
return
|
|
}
|
|
|
|
if cf.IsDeleted() || cf.IsInvalid() || cf.IsSymlink() || cf.IsDirectory() {
|
|
l.Debugf("%v recheckFile: %s: %q / %q: not a regular file", m, deviceID, folder, name)
|
|
return
|
|
}
|
|
|
|
blockIndex := int(offset / int64(cf.BlockSize()))
|
|
if blockIndex >= len(cf.Blocks) {
|
|
l.Debugf("%v recheckFile: %s: %q / %q i=%d: block index too far", m, deviceID, folder, name, blockIndex)
|
|
return
|
|
}
|
|
|
|
block := cf.Blocks[blockIndex]
|
|
|
|
// Seems to want a different version of the file, whatever.
|
|
if !bytes.Equal(block.Hash, hash) {
|
|
l.Debugf("%v recheckFile: %s: %q / %q i=%d: hash mismatch %x != %x", m, deviceID, folder, name, blockIndex, block.Hash, hash)
|
|
return
|
|
}
|
|
if weakHash != 0 && block.WeakHash != weakHash {
|
|
l.Debugf("%v recheckFile: %s: %q / %q i=%d: weak hash mismatch %v != %v", m, deviceID, folder, name, blockIndex, block.WeakHash, weakHash)
|
|
return
|
|
}
|
|
|
|
// The hashes provided part of the request match what we expect to find according
|
|
// to what we have in the database, yet the content we've read off the filesystem doesn't
|
|
// Something is fishy, invalidate the file and rescan it.
|
|
// The file will temporarily become invalid, which is ok as the content is messed up.
|
|
m.fmut.RLock()
|
|
runner, ok := m.folderRunners.Get(folder)
|
|
m.fmut.RUnlock()
|
|
if !ok {
|
|
l.Debugf("%v recheckFile: %s: %q / %q: Folder stopped before rescan could be scheduled", m, deviceID, folder, name)
|
|
return
|
|
}
|
|
|
|
runner.ScheduleForceRescan(name)
|
|
|
|
l.Debugf("%v recheckFile: %s: %q / %q", m, deviceID, folder, name)
|
|
}
|
|
|
|
func (m *model) CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool, error) {
|
|
m.fmut.RLock()
|
|
fs, ok := m.folderFiles[folder]
|
|
m.fmut.RUnlock()
|
|
if !ok {
|
|
return protocol.FileInfo{}, false, ErrFolderMissing
|
|
}
|
|
snap, err := fs.Snapshot()
|
|
if err != nil {
|
|
return protocol.FileInfo{}, false, err
|
|
}
|
|
f, ok := snap.Get(protocol.LocalDeviceID, file)
|
|
snap.Release()
|
|
return f, ok, nil
|
|
}
|
|
|
|
func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool, error) {
|
|
m.fmut.RLock()
|
|
ffs, ok := m.folderFiles[folder]
|
|
m.fmut.RUnlock()
|
|
if !ok {
|
|
return protocol.FileInfo{}, false, ErrFolderMissing
|
|
}
|
|
snap, err := ffs.Snapshot()
|
|
if err != nil {
|
|
return protocol.FileInfo{}, false, err
|
|
}
|
|
f, ok := snap.GetGlobal(file)
|
|
snap.Release()
|
|
return f, ok, nil
|
|
}
|
|
|
|
func (m *model) GetMtimeMapping(folder string, file string) (fs.MtimeMapping, error) {
|
|
m.fmut.RLock()
|
|
ffs, ok := m.folderFiles[folder]
|
|
fcfg := m.folderCfgs[folder]
|
|
m.fmut.RUnlock()
|
|
if !ok {
|
|
return fs.MtimeMapping{}, ErrFolderMissing
|
|
}
|
|
return fs.GetMtimeMapping(fcfg.Filesystem(ffs), file)
|
|
}
|
|
|
|
// Connection returns the current connection for device, and a boolean whether a connection was found.
|
|
func (m *model) Connection(deviceID protocol.DeviceID) (protocol.Connection, bool) {
|
|
m.pmut.RLock()
|
|
cn, ok := m.conn[deviceID]
|
|
m.pmut.RUnlock()
|
|
if ok {
|
|
m.deviceWasSeen(deviceID)
|
|
}
|
|
return cn, ok
|
|
}
|
|
|
|
// LoadIgnores loads or refreshes the ignore patterns from disk, if the
|
|
// folder is healthy, and returns the refreshed lines and patterns.
|
|
func (m *model) LoadIgnores(folder string) ([]string, []string, error) {
|
|
m.fmut.RLock()
|
|
cfg, cfgOk := m.folderCfgs[folder]
|
|
ignores, ignoresOk := m.folderIgnores[folder]
|
|
m.fmut.RUnlock()
|
|
|
|
if !cfgOk {
|
|
cfg, cfgOk = m.cfg.Folder(folder)
|
|
if !cfgOk {
|
|
return nil, nil, fmt.Errorf("folder %s does not exist", folder)
|
|
}
|
|
}
|
|
|
|
if cfg.Type == config.FolderTypeReceiveEncrypted {
|
|
return nil, nil, nil
|
|
}
|
|
|
|
if !ignoresOk {
|
|
ignores = ignore.New(cfg.Filesystem(nil))
|
|
}
|
|
|
|
err := ignores.Load(".stignore")
|
|
if fs.IsNotExist(err) {
|
|
// Having no ignores is not an error.
|
|
return nil, nil, nil
|
|
}
|
|
|
|
// Return lines and patterns, which may have some meaning even when err
|
|
// != nil, depending on the specific error.
|
|
return ignores.Lines(), ignores.Patterns(), err
|
|
}
|
|
|
|
// CurrentIgnores returns the currently loaded set of ignore patterns,
|
|
// whichever it may be. No attempt is made to load or refresh ignore
|
|
// patterns from disk.
|
|
func (m *model) CurrentIgnores(folder string) ([]string, []string, error) {
|
|
m.fmut.RLock()
|
|
_, cfgOk := m.folderCfgs[folder]
|
|
ignores, ignoresOk := m.folderIgnores[folder]
|
|
m.fmut.RUnlock()
|
|
|
|
if !cfgOk {
|
|
return nil, nil, fmt.Errorf("folder %s does not exist", folder)
|
|
}
|
|
|
|
if !ignoresOk {
|
|
// Empty ignore patterns
|
|
return []string{}, []string{}, nil
|
|
}
|
|
|
|
return ignores.Lines(), ignores.Patterns(), nil
|
|
}
|
|
|
|
func (m *model) SetIgnores(folder string, content []string) error {
|
|
cfg, ok := m.cfg.Folder(folder)
|
|
if !ok {
|
|
return fmt.Errorf("folder %s does not exist", cfg.Description())
|
|
}
|
|
return m.setIgnores(cfg, content)
|
|
}
|
|
|
|
func (m *model) setIgnores(cfg config.FolderConfiguration, content []string) error {
|
|
err := cfg.CheckPath()
|
|
if err == config.ErrPathMissing {
|
|
if err = cfg.CreateRoot(); err != nil {
|
|
return fmt.Errorf("failed to create folder root: %w", err)
|
|
}
|
|
err = cfg.CheckPath()
|
|
}
|
|
if err != nil && err != config.ErrMarkerMissing {
|
|
return err
|
|
}
|
|
|
|
if err := ignore.WriteIgnores(cfg.Filesystem(nil), ".stignore", content); err != nil {
|
|
l.Warnln("Saving .stignore:", err)
|
|
return err
|
|
}
|
|
|
|
m.fmut.RLock()
|
|
runner, ok := m.folderRunners.Get(cfg.ID)
|
|
m.fmut.RUnlock()
|
|
if ok {
|
|
runner.ScheduleScan()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// OnHello is called when an device connects to us.
|
|
// This allows us to extract some information from the Hello message
|
|
// and add it to a list of known devices ahead of any checks.
|
|
func (m *model) OnHello(remoteID protocol.DeviceID, addr net.Addr, hello protocol.Hello) error {
|
|
if _, ok := m.cfg.Device(remoteID); !ok {
|
|
if err := m.db.AddOrUpdatePendingDevice(remoteID, hello.DeviceName, addr.String()); err != nil {
|
|
l.Warnf("Failed to persist pending device entry to database: %v", err)
|
|
}
|
|
m.evLogger.Log(events.PendingDevicesChanged, map[string][]interface{}{
|
|
"added": {map[string]string{
|
|
"deviceID": remoteID.String(),
|
|
"name": hello.DeviceName,
|
|
"address": addr.String(),
|
|
}},
|
|
})
|
|
// DEPRECATED: Only for backwards compatibility, should be removed.
|
|
m.evLogger.Log(events.DeviceRejected, map[string]string{
|
|
"name": hello.DeviceName,
|
|
"device": remoteID.String(),
|
|
"address": addr.String(),
|
|
})
|
|
return errDeviceUnknown
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// GetHello is called when we are about to connect to some remote device.
|
|
func (m *model) GetHello(id protocol.DeviceID) protocol.HelloIntf {
|
|
name := ""
|
|
if _, ok := m.cfg.Device(id); ok {
|
|
// Set our name (from the config of our device ID) only if we already know about the other side device ID.
|
|
if myCfg, ok := m.cfg.Device(m.id); ok {
|
|
name = myCfg.Name
|
|
}
|
|
}
|
|
return &protocol.Hello{
|
|
DeviceName: name,
|
|
ClientName: m.clientName,
|
|
ClientVersion: m.clientVersion,
|
|
}
|
|
}
|
|
|
|
// AddConnection adds a new peer connection to the model. An initial index will
|
|
// be sent to the connected peer, thereafter index updates whenever the local
|
|
// folder changes.
|
|
func (m *model) AddConnection(conn protocol.Connection, hello protocol.Hello) {
|
|
deviceID := conn.DeviceID()
|
|
device, ok := m.cfg.Device(deviceID)
|
|
if !ok {
|
|
l.Infoln("Trying to add connection to unknown device")
|
|
return
|
|
}
|
|
|
|
// The slightly unusual locking sequence here is because we must acquire
|
|
// fmut before pmut. (The locks can be *released* in any order.)
|
|
m.fmut.RLock()
|
|
m.pmut.Lock()
|
|
if oldConn, ok := m.conn[deviceID]; ok {
|
|
l.Infoln("Replacing old connection", oldConn, "with", conn, "for", deviceID)
|
|
// There is an existing connection to this device that we are
|
|
// replacing. We must close the existing connection and wait for the
|
|
// close to complete before adding the new connection. We do the
|
|
// actual close without holding pmut as the connection will call
|
|
// back into Closed() for the cleanup.
|
|
closed := m.closed[deviceID]
|
|
m.fmut.RUnlock()
|
|
m.pmut.Unlock()
|
|
oldConn.Close(errReplacingConnection)
|
|
<-closed
|
|
// Again, lock fmut before pmut.
|
|
m.fmut.RLock()
|
|
m.pmut.Lock()
|
|
}
|
|
|
|
m.conn[deviceID] = conn
|
|
closed := make(chan struct{})
|
|
m.closed[deviceID] = closed
|
|
m.deviceDownloads[deviceID] = newDeviceDownloadState()
|
|
indexRegistry := newIndexHandlerRegistry(conn, m.deviceDownloads[deviceID], m.evLogger)
|
|
for id, fcfg := range m.folderCfgs {
|
|
runner, _ := m.folderRunners.Get(id)
|
|
indexRegistry.RegisterFolderState(fcfg, m.folderFiles[id], runner)
|
|
}
|
|
m.indexHandlers.Add(deviceID, indexRegistry)
|
|
m.fmut.RUnlock()
|
|
// 0: default, <0: no limiting
|
|
switch {
|
|
case device.MaxRequestKiB > 0:
|
|
m.connRequestLimiters[deviceID] = semaphore.New(1024 * device.MaxRequestKiB)
|
|
case device.MaxRequestKiB == 0:
|
|
m.connRequestLimiters[deviceID] = semaphore.New(1024 * defaultPullerPendingKiB)
|
|
}
|
|
|
|
m.helloMessages[deviceID] = hello
|
|
|
|
event := map[string]string{
|
|
"id": deviceID.String(),
|
|
"deviceName": hello.DeviceName,
|
|
"clientName": hello.ClientName,
|
|
"clientVersion": hello.ClientVersion,
|
|
"type": conn.Type(),
|
|
}
|
|
|
|
addr := conn.RemoteAddr()
|
|
if addr != nil {
|
|
event["addr"] = addr.String()
|
|
}
|
|
|
|
m.evLogger.Log(events.DeviceConnected, event)
|
|
|
|
l.Infof(`Device %s client is "%s %s" named "%s" at %s`, deviceID, hello.ClientName, hello.ClientVersion, hello.DeviceName, conn)
|
|
|
|
conn.Start()
|
|
m.pmut.Unlock()
|
|
|
|
// Acquires fmut, so has to be done outside of pmut.
|
|
cm, passwords := m.generateClusterConfig(deviceID)
|
|
conn.SetFolderPasswords(passwords)
|
|
conn.ClusterConfig(cm)
|
|
|
|
if (device.Name == "" || m.cfg.Options().OverwriteRemoteDevNames) && hello.DeviceName != "" {
|
|
m.cfg.Modify(func(cfg *config.Configuration) {
|
|
for i := range cfg.Devices {
|
|
if cfg.Devices[i].DeviceID == deviceID {
|
|
if cfg.Devices[i].Name == "" || cfg.Options.OverwriteRemoteDevNames {
|
|
cfg.Devices[i].Name = hello.DeviceName
|
|
}
|
|
return
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
m.deviceWasSeen(deviceID)
|
|
}
|
|
|
|
func (m *model) DownloadProgress(conn protocol.Connection, folder string, updates []protocol.FileDownloadProgressUpdate) error {
|
|
m.fmut.RLock()
|
|
cfg, ok := m.folderCfgs[folder]
|
|
m.fmut.RUnlock()
|
|
|
|
device := conn.DeviceID()
|
|
if !ok || cfg.DisableTempIndexes || !cfg.SharedWith(device) {
|
|
return nil
|
|
}
|
|
|
|
m.pmut.RLock()
|
|
downloads := m.deviceDownloads[device]
|
|
m.pmut.RUnlock()
|
|
downloads.Update(folder, updates)
|
|
state := downloads.GetBlockCounts(folder)
|
|
|
|
m.evLogger.Log(events.RemoteDownloadProgress, map[string]interface{}{
|
|
"device": device.String(),
|
|
"folder": folder,
|
|
"state": state,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
func (m *model) deviceWasSeen(deviceID protocol.DeviceID) {
|
|
m.fmut.RLock()
|
|
sr, ok := m.deviceStatRefs[deviceID]
|
|
m.fmut.RUnlock()
|
|
if ok {
|
|
_ = sr.WasSeen()
|
|
}
|
|
}
|
|
|
|
func (m *model) deviceDidClose(deviceID protocol.DeviceID, duration time.Duration) {
|
|
m.fmut.RLock()
|
|
sr, ok := m.deviceStatRefs[deviceID]
|
|
m.fmut.RUnlock()
|
|
if ok {
|
|
_ = sr.LastConnectionDuration(duration)
|
|
}
|
|
}
|
|
|
|
func (m *model) requestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
|
|
m.pmut.RLock()
|
|
nc, ok := m.conn[deviceID]
|
|
m.pmut.RUnlock()
|
|
|
|
if !ok {
|
|
return nil, fmt.Errorf("requestGlobal: no such device: %s", deviceID)
|
|
}
|
|
|
|
l.Debugf("%v REQ(out): %s: %q / %q b=%d o=%d s=%d h=%x wh=%x ft=%t", m, deviceID, folder, name, blockNo, offset, size, hash, weakHash, fromTemporary)
|
|
|
|
return nc.Request(ctx, folder, name, blockNo, offset, size, hash, weakHash, fromTemporary)
|
|
}
|
|
|
|
func (m *model) ScanFolders() map[string]error {
|
|
m.fmut.RLock()
|
|
folders := make([]string, 0, len(m.folderCfgs))
|
|
for folder := range m.folderCfgs {
|
|
folders = append(folders, folder)
|
|
}
|
|
m.fmut.RUnlock()
|
|
|
|
errors := make(map[string]error, len(m.folderCfgs))
|
|
errorsMut := sync.NewMutex()
|
|
|
|
wg := sync.NewWaitGroup()
|
|
wg.Add(len(folders))
|
|
for _, folder := range folders {
|
|
folder := folder
|
|
go func() {
|
|
err := m.ScanFolder(folder)
|
|
if err != nil {
|
|
errorsMut.Lock()
|
|
errors[folder] = err
|
|
errorsMut.Unlock()
|
|
}
|
|
wg.Done()
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
return errors
|
|
}
|
|
|
|
func (m *model) ScanFolder(folder string) error {
|
|
return m.ScanFolderSubdirs(folder, nil)
|
|
}
|
|
|
|
func (m *model) ScanFolderSubdirs(folder string, subs []string) error {
|
|
m.fmut.RLock()
|
|
err := m.checkFolderRunningLocked(folder)
|
|
runner, _ := m.folderRunners.Get(folder)
|
|
m.fmut.RUnlock()
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return runner.Scan(subs)
|
|
}
|
|
|
|
func (m *model) DelayScan(folder string, next time.Duration) {
|
|
m.fmut.RLock()
|
|
runner, ok := m.folderRunners.Get(folder)
|
|
m.fmut.RUnlock()
|
|
if !ok {
|
|
return
|
|
}
|
|
runner.DelayScan(next)
|
|
}
|
|
|
|
// numHashers returns the number of hasher routines to use for a given folder,
|
|
// taking into account configuration and available CPU cores.
|
|
func (m *model) numHashers(folder string) int {
|
|
m.fmut.RLock()
|
|
folderCfg := m.folderCfgs[folder]
|
|
numFolders := len(m.folderCfgs)
|
|
m.fmut.RUnlock()
|
|
|
|
if folderCfg.Hashers > 0 {
|
|
// Specific value set in the config, use that.
|
|
return folderCfg.Hashers
|
|
}
|
|
|
|
if build.IsWindows || build.IsDarwin || build.IsAndroid {
|
|
// Interactive operating systems; don't load the system too heavily by
|
|
// default.
|
|
return 1
|
|
}
|
|
|
|
// For other operating systems and architectures, lets try to get some
|
|
// work done... Divide the available CPU cores among the configured
|
|
// folders.
|
|
if perFolder := runtime.GOMAXPROCS(-1) / numFolders; perFolder > 0 {
|
|
return perFolder
|
|
}
|
|
|
|
return 1
|
|
}
|
|
|
|
// generateClusterConfig returns a ClusterConfigMessage that is correct and the
|
|
// set of folder passwords for the given peer device
|
|
func (m *model) generateClusterConfig(device protocol.DeviceID) (protocol.ClusterConfig, map[string]string) {
|
|
var message protocol.ClusterConfig
|
|
|
|
m.fmut.RLock()
|
|
defer m.fmut.RUnlock()
|
|
|
|
folders := m.cfg.FolderList()
|
|
passwords := make(map[string]string, len(folders))
|
|
for _, folderCfg := range folders {
|
|
if !folderCfg.SharedWith(device) {
|
|
continue
|
|
}
|
|
|
|
encryptionToken, hasEncryptionToken := m.folderEncryptionPasswordTokens[folderCfg.ID]
|
|
if folderCfg.Type == config.FolderTypeReceiveEncrypted && !hasEncryptionToken {
|
|
// We haven't gotten a token for us yet and without one the other
|
|
// side can't validate us - pretend we don't have the folder yet.
|
|
continue
|
|
}
|
|
|
|
protocolFolder := protocol.Folder{
|
|
ID: folderCfg.ID,
|
|
Label: folderCfg.Label,
|
|
ReadOnly: folderCfg.Type == config.FolderTypeSendOnly,
|
|
IgnorePermissions: folderCfg.IgnorePerms,
|
|
IgnoreDelete: folderCfg.IgnoreDelete,
|
|
DisableTempIndexes: folderCfg.DisableTempIndexes,
|
|
}
|
|
|
|
fs := m.folderFiles[folderCfg.ID]
|
|
|
|
// Even if we aren't paused, if we haven't started the folder yet
|
|
// pretend we are. Otherwise the remote might get confused about
|
|
// the missing index info (and drop all the info). We will send
|
|
// another cluster config once the folder is started.
|
|
protocolFolder.Paused = folderCfg.Paused || fs == nil
|
|
|
|
for _, folderDevice := range folderCfg.Devices {
|
|
deviceCfg, _ := m.cfg.Device(folderDevice.DeviceID)
|
|
|
|
protocolDevice := protocol.Device{
|
|
ID: deviceCfg.DeviceID,
|
|
Name: deviceCfg.Name,
|
|
Addresses: deviceCfg.Addresses,
|
|
Compression: deviceCfg.Compression,
|
|
CertName: deviceCfg.CertName,
|
|
Introducer: deviceCfg.Introducer,
|
|
}
|
|
|
|
if deviceCfg.DeviceID == m.id && hasEncryptionToken {
|
|
protocolDevice.EncryptionPasswordToken = encryptionToken
|
|
} else if folderDevice.EncryptionPassword != "" {
|
|
protocolDevice.EncryptionPasswordToken = protocol.PasswordToken(m.keyGen, folderCfg.ID, folderDevice.EncryptionPassword)
|
|
if folderDevice.DeviceID == device {
|
|
passwords[folderCfg.ID] = folderDevice.EncryptionPassword
|
|
}
|
|
}
|
|
|
|
if fs != nil {
|
|
if deviceCfg.DeviceID == m.id {
|
|
protocolDevice.IndexID = fs.IndexID(protocol.LocalDeviceID)
|
|
protocolDevice.MaxSequence = fs.Sequence(protocol.LocalDeviceID)
|
|
} else {
|
|
protocolDevice.IndexID = fs.IndexID(deviceCfg.DeviceID)
|
|
protocolDevice.MaxSequence = fs.Sequence(deviceCfg.DeviceID)
|
|
}
|
|
}
|
|
|
|
protocolFolder.Devices = append(protocolFolder.Devices, protocolDevice)
|
|
}
|
|
|
|
message.Folders = append(message.Folders, protocolFolder)
|
|
}
|
|
|
|
return message, passwords
|
|
}
|
|
|
|
func (m *model) State(folder string) (string, time.Time, error) {
|
|
m.fmut.RLock()
|
|
runner, ok := m.folderRunners.Get(folder)
|
|
m.fmut.RUnlock()
|
|
if !ok {
|
|
// The returned error should be an actual folder error, so returning
|
|
// errors.New("does not exist") or similar here would be
|
|
// inappropriate.
|
|
return "", time.Time{}, nil
|
|
}
|
|
state, changed, err := runner.getState()
|
|
return state.String(), changed, err
|
|
}
|
|
|
|
func (m *model) FolderErrors(folder string) ([]FileError, error) {
|
|
m.fmut.RLock()
|
|
err := m.checkFolderRunningLocked(folder)
|
|
runner, _ := m.folderRunners.Get(folder)
|
|
m.fmut.RUnlock()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return runner.Errors(), nil
|
|
}
|
|
|
|
func (m *model) WatchError(folder string) error {
|
|
m.fmut.RLock()
|
|
err := m.checkFolderRunningLocked(folder)
|
|
runner, _ := m.folderRunners.Get(folder)
|
|
m.fmut.RUnlock()
|
|
if err != nil {
|
|
return nil // If the folder isn't running, there's no error to report.
|
|
}
|
|
return runner.WatchError()
|
|
}
|
|
|
|
func (m *model) Override(folder string) {
|
|
// Grab the runner and the file set.
|
|
|
|
m.fmut.RLock()
|
|
runner, ok := m.folderRunners.Get(folder)
|
|
m.fmut.RUnlock()
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
// Run the override, taking updates as if they came from scanning.
|
|
|
|
runner.Override()
|
|
}
|
|
|
|
func (m *model) Revert(folder string) {
|
|
// Grab the runner and the file set.
|
|
|
|
m.fmut.RLock()
|
|
runner, ok := m.folderRunners.Get(folder)
|
|
m.fmut.RUnlock()
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
// Run the revert, taking updates as if they came from scanning.
|
|
|
|
runner.Revert()
|
|
}
|
|
|
|
type TreeEntry struct {
|
|
Name string `json:"name"`
|
|
ModTime time.Time `json:"modTime"`
|
|
Size int64 `json:"size"`
|
|
Type protocol.FileInfoType `json:"type"`
|
|
Children []*TreeEntry `json:"children,omitempty"`
|
|
}
|
|
|
|
func findByName(slice []*TreeEntry, name string) *TreeEntry {
|
|
for _, child := range slice {
|
|
if child.Name == name {
|
|
return child
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsOnly bool) ([]*TreeEntry, error) {
|
|
m.fmut.RLock()
|
|
files, ok := m.folderFiles[folder]
|
|
m.fmut.RUnlock()
|
|
if !ok {
|
|
return nil, ErrFolderMissing
|
|
}
|
|
|
|
root := &TreeEntry{
|
|
Children: make([]*TreeEntry, 0),
|
|
}
|
|
sep := string(filepath.Separator)
|
|
prefix = osutil.NativeFilename(prefix)
|
|
|
|
if prefix != "" && !strings.HasSuffix(prefix, sep) {
|
|
prefix = prefix + sep
|
|
}
|
|
|
|
snap, err := files.Snapshot()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer snap.Release()
|
|
snap.WithPrefixedGlobalTruncated(prefix, func(fi protocol.FileIntf) bool {
|
|
f := fi.(db.FileInfoTruncated)
|
|
|
|
// Don't include the prefix itself.
|
|
if f.IsInvalid() || f.IsDeleted() || strings.HasPrefix(prefix, f.Name) {
|
|
return true
|
|
}
|
|
|
|
f.Name = strings.Replace(f.Name, prefix, "", 1)
|
|
|
|
dir := filepath.Dir(f.Name)
|
|
base := filepath.Base(f.Name)
|
|
|
|
if levels > -1 && strings.Count(f.Name, sep) > levels {
|
|
return true
|
|
}
|
|
|
|
parent := root
|
|
if dir != "." {
|
|
for _, path := range strings.Split(dir, sep) {
|
|
child := findByName(parent.Children, path)
|
|
if child == nil {
|
|
err = fmt.Errorf("could not find child '%s' for path '%s' in parent '%s'", path, f.Name, parent.Name)
|
|
return false
|
|
}
|
|
parent = child
|
|
}
|
|
}
|
|
|
|
if dirsOnly && !f.IsDirectory() {
|
|
return true
|
|
}
|
|
|
|
parent.Children = append(parent.Children, &TreeEntry{
|
|
Name: base,
|
|
Type: f.Type,
|
|
ModTime: f.ModTime(),
|
|
Size: f.FileSize(),
|
|
})
|
|
|
|
return true
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return root.Children, nil
|
|
}
|
|
|
|
func (m *model) GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error) {
|
|
m.fmut.RLock()
|
|
err := m.checkFolderRunningLocked(folder)
|
|
ver := m.folderVersioners[folder]
|
|
m.fmut.RUnlock()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if ver == nil {
|
|
return nil, errNoVersioner
|
|
}
|
|
|
|
return ver.GetVersions()
|
|
}
|
|
|
|
func (m *model) RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]error, error) {
|
|
m.fmut.RLock()
|
|
err := m.checkFolderRunningLocked(folder)
|
|
fcfg := m.folderCfgs[folder]
|
|
ver := m.folderVersioners[folder]
|
|
m.fmut.RUnlock()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if ver == nil {
|
|
return nil, errNoVersioner
|
|
}
|
|
|
|
restoreErrors := make(map[string]error)
|
|
|
|
for file, version := range versions {
|
|
if err := ver.Restore(file, version); err != nil {
|
|
restoreErrors[file] = err
|
|
}
|
|
}
|
|
|
|
// Trigger scan
|
|
if !fcfg.FSWatcherEnabled {
|
|
go func() { _ = m.ScanFolder(folder) }()
|
|
}
|
|
|
|
return restoreErrors, nil
|
|
}
|
|
|
|
func (m *model) Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) ([]Availability, error) {
|
|
// The slightly unusual locking sequence here is because we need to hold
|
|
// pmut for the duration (as the value returned from foldersFiles can
|
|
// get heavily modified on Close()), but also must acquire fmut before
|
|
// pmut. (The locks can be *released* in any order.)
|
|
m.fmut.RLock()
|
|
m.pmut.RLock()
|
|
defer m.pmut.RUnlock()
|
|
|
|
fs, ok := m.folderFiles[folder]
|
|
cfg := m.folderCfgs[folder]
|
|
m.fmut.RUnlock()
|
|
|
|
if !ok {
|
|
return nil, ErrFolderMissing
|
|
}
|
|
|
|
snap, err := fs.Snapshot()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer snap.Release()
|
|
|
|
return m.availabilityInSnapshotPRlocked(cfg, snap, file, block), nil
|
|
}
|
|
|
|
func (m *model) availabilityInSnapshot(cfg config.FolderConfiguration, snap *db.Snapshot, file protocol.FileInfo, block protocol.BlockInfo) []Availability {
|
|
m.pmut.RLock()
|
|
defer m.pmut.RUnlock()
|
|
return m.availabilityInSnapshotPRlocked(cfg, snap, file, block)
|
|
}
|
|
|
|
func (m *model) availabilityInSnapshotPRlocked(cfg config.FolderConfiguration, snap *db.Snapshot, file protocol.FileInfo, block protocol.BlockInfo) []Availability {
|
|
var availabilities []Availability
|
|
for _, device := range snap.Availability(file.Name) {
|
|
if _, ok := m.remoteFolderStates[device]; !ok {
|
|
continue
|
|
}
|
|
if state := m.remoteFolderStates[device][cfg.ID]; state != remoteFolderValid {
|
|
continue
|
|
}
|
|
_, ok := m.conn[device]
|
|
if ok {
|
|
availabilities = append(availabilities, Availability{ID: device, FromTemporary: false})
|
|
}
|
|
}
|
|
|
|
for _, device := range cfg.Devices {
|
|
if m.deviceDownloads[device.DeviceID].Has(cfg.ID, file.Name, file.Version, int(block.Offset/int64(file.BlockSize()))) {
|
|
availabilities = append(availabilities, Availability{ID: device.DeviceID, FromTemporary: true})
|
|
}
|
|
}
|
|
|
|
return availabilities
|
|
}
|
|
|
|
// BringToFront bumps the given files priority in the job queue.
|
|
func (m *model) BringToFront(folder, file string) {
|
|
m.fmut.RLock()
|
|
runner, ok := m.folderRunners.Get(folder)
|
|
m.fmut.RUnlock()
|
|
|
|
if ok {
|
|
runner.BringToFront(file)
|
|
}
|
|
}
|
|
|
|
func (m *model) ResetFolder(folder string) error {
|
|
m.fmut.RLock()
|
|
defer m.fmut.RUnlock()
|
|
_, ok := m.folderRunners.Get(folder)
|
|
if ok {
|
|
return errors.New("folder must be paused when resetting")
|
|
}
|
|
l.Infof("Cleaning metadata for reset folder %q", folder)
|
|
db.DropFolder(m.db, folder)
|
|
return nil
|
|
}
|
|
|
|
func (m *model) String() string {
|
|
return fmt.Sprintf("model@%p", m)
|
|
}
|
|
|
|
func (*model) VerifyConfiguration(from, to config.Configuration) error {
|
|
toFolders := to.FolderMap()
|
|
for _, from := range from.Folders {
|
|
to, ok := toFolders[from.ID]
|
|
if ok && from.Type != to.Type && (from.Type == config.FolderTypeReceiveEncrypted || to.Type == config.FolderTypeReceiveEncrypted) {
|
|
return errors.New("folder type must not be changed from/to receive-encrypted")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m *model) CommitConfiguration(from, to config.Configuration) bool {
|
|
// TODO: This should not use reflect, and should take more care to try to handle stuff without restart.
|
|
|
|
// Delay processing config changes until after the initial setup
|
|
<-m.started
|
|
|
|
// Go through the folder configs and figure out if we need to restart or not.
|
|
|
|
// Tracks devices affected by any configuration change to resend ClusterConfig.
|
|
clusterConfigDevices := make(deviceIDSet, len(from.Devices)+len(to.Devices))
|
|
closeDevices := make([]protocol.DeviceID, 0, len(to.Devices))
|
|
|
|
fromFolders := mapFolders(from.Folders)
|
|
toFolders := mapFolders(to.Folders)
|
|
for folderID, cfg := range toFolders {
|
|
if _, ok := fromFolders[folderID]; !ok {
|
|
// A folder was added.
|
|
if cfg.Paused {
|
|
l.Infoln("Paused folder", cfg.Description())
|
|
} else {
|
|
l.Infoln("Adding folder", cfg.Description())
|
|
if err := m.newFolder(cfg, to.Options.CacheIgnoredFiles); err != nil {
|
|
m.fatal(err)
|
|
return true
|
|
}
|
|
}
|
|
clusterConfigDevices.add(cfg.DeviceIDs())
|
|
}
|
|
}
|
|
|
|
removedFolders := make(map[string]struct{})
|
|
for folderID, fromCfg := range fromFolders {
|
|
toCfg, ok := toFolders[folderID]
|
|
if !ok {
|
|
// The folder was removed.
|
|
m.removeFolder(fromCfg)
|
|
clusterConfigDevices.add(fromCfg.DeviceIDs())
|
|
removedFolders[fromCfg.ID] = struct{}{}
|
|
continue
|
|
}
|
|
|
|
if fromCfg.Paused && toCfg.Paused {
|
|
continue
|
|
}
|
|
|
|
// This folder exists on both sides. Settings might have changed.
|
|
// Check if anything differs that requires a restart.
|
|
if !reflect.DeepEqual(fromCfg.RequiresRestartOnly(), toCfg.RequiresRestartOnly()) || from.Options.CacheIgnoredFiles != to.Options.CacheIgnoredFiles {
|
|
if err := m.restartFolder(fromCfg, toCfg, to.Options.CacheIgnoredFiles); err != nil {
|
|
m.fatal(err)
|
|
return true
|
|
}
|
|
clusterConfigDevices.add(fromCfg.DeviceIDs())
|
|
if toCfg.Type != config.FolderTypeReceiveEncrypted {
|
|
clusterConfigDevices.add(toCfg.DeviceIDs())
|
|
} else {
|
|
// If we don't have the encryption token yet, we need to drop
|
|
// the connection to make the remote re-send the cluster-config
|
|
// and with it the token.
|
|
m.fmut.RLock()
|
|
_, ok := m.folderEncryptionPasswordTokens[toCfg.ID]
|
|
m.fmut.RUnlock()
|
|
if !ok {
|
|
closeDevices = append(closeDevices, toCfg.DeviceIDs()...)
|
|
} else {
|
|
clusterConfigDevices.add(toCfg.DeviceIDs())
|
|
}
|
|
}
|
|
}
|
|
|
|
// Emit the folder pause/resume event
|
|
if fromCfg.Paused != toCfg.Paused {
|
|
eventType := events.FolderResumed
|
|
if toCfg.Paused {
|
|
eventType = events.FolderPaused
|
|
}
|
|
m.evLogger.Log(eventType, map[string]string{"id": toCfg.ID, "label": toCfg.Label})
|
|
}
|
|
}
|
|
|
|
// Removing a device. We actually don't need to do anything.
|
|
// Because folder config has changed (since the device lists do not match)
|
|
// Folders for that had device got "restarted", which involves killing
|
|
// connections to all devices that we were sharing the folder with.
|
|
// At some point model.Close() will get called for that device which will
|
|
// clean residue device state that is not part of any folder.
|
|
|
|
// Pausing a device, unpausing is handled by the connection service.
|
|
fromDevices := from.DeviceMap()
|
|
toDevices := to.DeviceMap()
|
|
for deviceID, toCfg := range toDevices {
|
|
fromCfg, ok := fromDevices[deviceID]
|
|
if !ok {
|
|
sr := stats.NewDeviceStatisticsReference(m.db, deviceID)
|
|
m.fmut.Lock()
|
|
m.deviceStatRefs[deviceID] = sr
|
|
m.fmut.Unlock()
|
|
continue
|
|
}
|
|
delete(fromDevices, deviceID)
|
|
if fromCfg.Paused == toCfg.Paused {
|
|
continue
|
|
}
|
|
|
|
if toCfg.Paused {
|
|
l.Infoln("Pausing", deviceID)
|
|
closeDevices = append(closeDevices, deviceID)
|
|
m.evLogger.Log(events.DevicePaused, map[string]string{"device": deviceID.String()})
|
|
} else {
|
|
// Ignored folder was removed, reconnect to retrigger the prompt.
|
|
if len(fromCfg.IgnoredFolders) > len(toCfg.IgnoredFolders) {
|
|
closeDevices = append(closeDevices, deviceID)
|
|
}
|
|
|
|
l.Infoln("Resuming", deviceID)
|
|
m.evLogger.Log(events.DeviceResumed, map[string]string{"device": deviceID.String()})
|
|
}
|
|
}
|
|
// Clean up after removed devices
|
|
removedDevices := make([]protocol.DeviceID, 0, len(fromDevices))
|
|
m.fmut.Lock()
|
|
for deviceID := range fromDevices {
|
|
delete(m.deviceStatRefs, deviceID)
|
|
removedDevices = append(removedDevices, deviceID)
|
|
delete(clusterConfigDevices, deviceID)
|
|
}
|
|
m.fmut.Unlock()
|
|
|
|
m.pmut.RLock()
|
|
for _, id := range closeDevices {
|
|
delete(clusterConfigDevices, id)
|
|
if conn, ok := m.conn[id]; ok {
|
|
go conn.Close(errDevicePaused)
|
|
}
|
|
}
|
|
for _, id := range removedDevices {
|
|
delete(clusterConfigDevices, id)
|
|
if conn, ok := m.conn[id]; ok {
|
|
go conn.Close(errDeviceRemoved)
|
|
}
|
|
}
|
|
m.pmut.RUnlock()
|
|
// Generating cluster-configs acquires fmut -> must happen outside of pmut.
|
|
m.sendClusterConfig(clusterConfigDevices.AsSlice())
|
|
|
|
ignoredDevices := observedDeviceSet(to.IgnoredDevices)
|
|
m.cleanPending(toDevices, toFolders, ignoredDevices, removedFolders)
|
|
|
|
m.globalRequestLimiter.SetCapacity(1024 * to.Options.MaxConcurrentIncomingRequestKiB())
|
|
m.folderIOLimiter.SetCapacity(to.Options.MaxFolderConcurrency())
|
|
|
|
// Some options don't require restart as those components handle it fine
|
|
// by themselves. Compare the options structs containing only the
|
|
// attributes that require restart and act apprioriately.
|
|
if !reflect.DeepEqual(from.Options.RequiresRestartOnly(), to.Options.RequiresRestartOnly()) {
|
|
l.Debugln(m, "requires restart, options differ")
|
|
return false
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func (m *model) cleanPending(existingDevices map[protocol.DeviceID]config.DeviceConfiguration, existingFolders map[string]config.FolderConfiguration, ignoredDevices deviceIDSet, removedFolders map[string]struct{}) {
|
|
var removedPendingFolders []map[string]string
|
|
pendingFolders, err := m.db.PendingFolders()
|
|
if err != nil {
|
|
msg := "Could not iterate through pending folder entries for cleanup"
|
|
l.Warnf("%v: %v", msg, err)
|
|
m.evLogger.Log(events.Failure, msg)
|
|
// Continue with pending devices below, loop is skipped.
|
|
}
|
|
for folderID, pf := range pendingFolders {
|
|
if _, ok := removedFolders[folderID]; ok {
|
|
// Forget pending folder device associations for recently removed
|
|
// folders as well, assuming the folder is no longer of interest
|
|
// at all (but might become pending again).
|
|
l.Debugf("Discarding pending removed folder %v from all devices", folderID)
|
|
if err := m.db.RemovePendingFolder(folderID); err != nil {
|
|
msg := "Failed to remove pending folder entry"
|
|
l.Warnf("%v (%v): %v", msg, folderID, err)
|
|
m.evLogger.Log(events.Failure, msg)
|
|
} else {
|
|
removedPendingFolders = append(removedPendingFolders, map[string]string{
|
|
"folderID": folderID,
|
|
})
|
|
}
|
|
continue
|
|
}
|
|
for deviceID := range pf.OfferedBy {
|
|
if dev, ok := existingDevices[deviceID]; !ok {
|
|
l.Debugf("Discarding pending folder %v from unknown device %v", folderID, deviceID)
|
|
goto removeFolderForDevice
|
|
} else if dev.IgnoredFolder(folderID) {
|
|
l.Debugf("Discarding now ignored pending folder %v for device %v", folderID, deviceID)
|
|
goto removeFolderForDevice
|
|
}
|
|
if folderCfg, ok := existingFolders[folderID]; ok {
|
|
if folderCfg.SharedWith(deviceID) {
|
|
l.Debugf("Discarding now shared pending folder %v for device %v", folderID, deviceID)
|
|
goto removeFolderForDevice
|
|
}
|
|
}
|
|
continue
|
|
removeFolderForDevice:
|
|
if err := m.db.RemovePendingFolderForDevice(folderID, deviceID); err != nil {
|
|
msg := "Failed to remove pending folder-device entry"
|
|
l.Warnf("%v (%v, %v): %v", msg, folderID, deviceID, err)
|
|
m.evLogger.Log(events.Failure, msg)
|
|
continue
|
|
}
|
|
removedPendingFolders = append(removedPendingFolders, map[string]string{
|
|
"folderID": folderID,
|
|
"deviceID": deviceID.String(),
|
|
})
|
|
}
|
|
}
|
|
if len(removedPendingFolders) > 0 {
|
|
m.evLogger.Log(events.PendingFoldersChanged, map[string]interface{}{
|
|
"removed": removedPendingFolders,
|
|
})
|
|
}
|
|
|
|
var removedPendingDevices []map[string]string
|
|
pendingDevices, err := m.db.PendingDevices()
|
|
if err != nil {
|
|
msg := "Could not iterate through pending device entries for cleanup"
|
|
l.Warnf("%v: %v", msg, err)
|
|
m.evLogger.Log(events.Failure, msg)
|
|
return
|
|
}
|
|
for deviceID := range pendingDevices {
|
|
if _, ok := ignoredDevices[deviceID]; ok {
|
|
l.Debugf("Discarding now ignored pending device %v", deviceID)
|
|
goto removeDevice
|
|
}
|
|
if _, ok := existingDevices[deviceID]; ok {
|
|
l.Debugf("Discarding now added pending device %v", deviceID)
|
|
goto removeDevice
|
|
}
|
|
continue
|
|
removeDevice:
|
|
if err := m.db.RemovePendingDevice(deviceID); err != nil {
|
|
msg := "Failed to remove pending device entry"
|
|
l.Warnf("%v: %v", msg, err)
|
|
m.evLogger.Log(events.Failure, msg)
|
|
continue
|
|
}
|
|
removedPendingDevices = append(removedPendingDevices, map[string]string{
|
|
"deviceID": deviceID.String(),
|
|
})
|
|
}
|
|
if len(removedPendingDevices) > 0 {
|
|
m.evLogger.Log(events.PendingDevicesChanged, map[string]interface{}{
|
|
"removed": removedPendingDevices,
|
|
})
|
|
}
|
|
}
|
|
|
|
// checkFolderRunningLocked returns nil if the folder is up and running and a
|
|
// descriptive error if not.
|
|
// Need to hold (read) lock on m.fmut when calling this.
|
|
func (m *model) checkFolderRunningLocked(folder string) error {
|
|
_, ok := m.folderRunners.Get(folder)
|
|
if ok {
|
|
return nil
|
|
}
|
|
|
|
if cfg, ok := m.cfg.Folder(folder); !ok {
|
|
return ErrFolderMissing
|
|
} else if cfg.Paused {
|
|
return ErrFolderPaused
|
|
}
|
|
|
|
return ErrFolderNotRunning
|
|
}
|
|
|
|
// PendingDevices lists unknown devices that tried to connect.
|
|
func (m *model) PendingDevices() (map[protocol.DeviceID]db.ObservedDevice, error) {
|
|
return m.db.PendingDevices()
|
|
}
|
|
|
|
// PendingFolders lists folders that we don't yet share with the offering devices. It
|
|
// returns the entries grouped by folder and filters for a given device unless the
|
|
// argument is specified as EmptyDeviceID.
|
|
func (m *model) PendingFolders(device protocol.DeviceID) (map[string]db.PendingFolder, error) {
|
|
return m.db.PendingFoldersForDevice(device)
|
|
}
|
|
|
|
// DismissPendingDevices removes the record of a specific pending device.
|
|
func (m *model) DismissPendingDevice(device protocol.DeviceID) error {
|
|
l.Debugf("Discarding pending device %v", device)
|
|
err := m.db.RemovePendingDevice(device)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
removedPendingDevices := []map[string]string{
|
|
{"deviceID": device.String()},
|
|
}
|
|
m.evLogger.Log(events.PendingDevicesChanged, map[string]interface{}{
|
|
"removed": removedPendingDevices,
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// DismissPendingFolders removes records of pending folders. Either a specific folder /
|
|
// device combination, or all matching a specific folder ID if the device argument is
|
|
// specified as EmptyDeviceID.
|
|
func (m *model) DismissPendingFolder(device protocol.DeviceID, folder string) error {
|
|
var removedPendingFolders []map[string]string
|
|
if device == protocol.EmptyDeviceID {
|
|
l.Debugf("Discarding pending removed folder %s from all devices", folder)
|
|
err := m.db.RemovePendingFolder(folder)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
removedPendingFolders = []map[string]string{
|
|
{"folderID": folder},
|
|
}
|
|
} else {
|
|
l.Debugf("Discarding pending folder %s from device %v", folder, device)
|
|
err := m.db.RemovePendingFolderForDevice(folder, device)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
removedPendingFolders = []map[string]string{
|
|
{
|
|
"folderID": folder,
|
|
"deviceID": device.String(),
|
|
},
|
|
}
|
|
}
|
|
if len(removedPendingFolders) > 0 {
|
|
m.evLogger.Log(events.PendingFoldersChanged, map[string]interface{}{
|
|
"removed": removedPendingFolders,
|
|
})
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// mapFolders returns a map of folder ID to folder configuration for the given
|
|
// slice of folder configurations.
|
|
func mapFolders(folders []config.FolderConfiguration) map[string]config.FolderConfiguration {
|
|
m := make(map[string]config.FolderConfiguration, len(folders))
|
|
for _, cfg := range folders {
|
|
m[cfg.ID] = cfg
|
|
}
|
|
return m
|
|
}
|
|
|
|
// mapDevices returns a map of device ID to nothing for the given slice of
|
|
// device IDs.
|
|
func mapDevices(devices []protocol.DeviceID) map[protocol.DeviceID]struct{} {
|
|
m := make(map[protocol.DeviceID]struct{}, len(devices))
|
|
for _, dev := range devices {
|
|
m[dev] = struct{}{}
|
|
}
|
|
return m
|
|
}
|
|
|
|
func observedDeviceSet(devices []config.ObservedDevice) deviceIDSet {
|
|
res := make(deviceIDSet, len(devices))
|
|
for _, dev := range devices {
|
|
res[dev.ID] = struct{}{}
|
|
}
|
|
return res
|
|
}
|
|
|
|
func readOffsetIntoBuf(fs fs.Filesystem, file string, offset int64, buf []byte) (int, error) {
|
|
fd, err := fs.Open(file)
|
|
if err != nil {
|
|
l.Debugln("readOffsetIntoBuf.Open", file, err)
|
|
return 0, err
|
|
}
|
|
|
|
defer fd.Close()
|
|
n, err := fd.ReadAt(buf, offset)
|
|
if err != nil {
|
|
l.Debugln("readOffsetIntoBuf.ReadAt", file, err)
|
|
}
|
|
return n, err
|
|
}
|
|
|
|
// folderDeviceSet is a set of (folder, deviceID) pairs
|
|
type folderDeviceSet map[string]map[protocol.DeviceID]struct{}
|
|
|
|
// set adds the (dev, folder) pair to the set
|
|
func (s folderDeviceSet) set(dev protocol.DeviceID, folder string) {
|
|
devs, ok := s[folder]
|
|
if !ok {
|
|
devs = make(map[protocol.DeviceID]struct{})
|
|
s[folder] = devs
|
|
}
|
|
devs[dev] = struct{}{}
|
|
}
|
|
|
|
// has returns true if the (dev, folder) pair is in the set
|
|
func (s folderDeviceSet) has(dev protocol.DeviceID, folder string) bool {
|
|
_, ok := s[folder][dev]
|
|
return ok
|
|
}
|
|
|
|
// hasDevice returns true if the device is set on any folder
|
|
func (s folderDeviceSet) hasDevice(dev protocol.DeviceID) bool {
|
|
for _, devices := range s {
|
|
if _, ok := devices[dev]; ok {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// syncMutexMap is a type safe wrapper for a sync.Map that holds mutexes
|
|
type syncMutexMap struct {
|
|
inner stdsync.Map
|
|
}
|
|
|
|
func (m *syncMutexMap) Get(key string) sync.Mutex {
|
|
v, _ := m.inner.LoadOrStore(key, sync.NewMutex())
|
|
return v.(sync.Mutex)
|
|
}
|
|
|
|
type deviceIDSet map[protocol.DeviceID]struct{}
|
|
|
|
func (s deviceIDSet) add(ids []protocol.DeviceID) {
|
|
for _, id := range ids {
|
|
if _, ok := s[id]; !ok {
|
|
s[id] = struct{}{}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s deviceIDSet) AsSlice() []protocol.DeviceID {
|
|
ids := make([]protocol.DeviceID, 0, len(s))
|
|
for id := range s {
|
|
ids = append(ids, id)
|
|
}
|
|
return ids
|
|
}
|
|
|
|
func encryptionTokenPath(cfg config.FolderConfiguration) string {
|
|
return filepath.Join(cfg.MarkerName, config.EncryptionTokenName)
|
|
}
|
|
|
|
type storedEncryptionToken struct {
|
|
FolderID string
|
|
Token []byte
|
|
}
|
|
|
|
func readEncryptionToken(cfg config.FolderConfiguration) ([]byte, error) {
|
|
fd, err := cfg.Filesystem(nil).Open(encryptionTokenPath(cfg))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer fd.Close()
|
|
var stored storedEncryptionToken
|
|
if err := json.NewDecoder(fd).Decode(&stored); err != nil {
|
|
return nil, err
|
|
}
|
|
return stored.Token, nil
|
|
}
|
|
|
|
func writeEncryptionToken(token []byte, cfg config.FolderConfiguration) error {
|
|
tokenName := encryptionTokenPath(cfg)
|
|
fd, err := cfg.Filesystem(nil).OpenFile(tokenName, fs.OptReadWrite|fs.OptCreate, 0o666)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer fd.Close()
|
|
return json.NewEncoder(fd).Encode(storedEncryptionToken{
|
|
FolderID: cfg.ID,
|
|
Token: token,
|
|
})
|
|
}
|
|
|
|
func newFolderConfiguration(w config.Wrapper, id, label string, fsType fs.FilesystemType, path string) config.FolderConfiguration {
|
|
fcfg := w.DefaultFolder()
|
|
fcfg.ID = id
|
|
fcfg.Label = label
|
|
fcfg.FilesystemType = fsType
|
|
fcfg.Path = path
|
|
return fcfg
|
|
}
|
|
|
|
type updatedPendingFolder struct {
|
|
FolderID string `json:"folderID"`
|
|
FolderLabel string `json:"folderLabel"`
|
|
DeviceID protocol.DeviceID `json:"deviceID"`
|
|
ReceiveEncrypted bool `json:"receiveEncrypted"`
|
|
RemoteEncrypted bool `json:"remoteEncrypted"`
|
|
}
|
|
|
|
// redactPathError checks if the error is actually a os.PathError, and if yes
|
|
// returns a redactedError with the path removed.
|
|
func redactPathError(err error) (error, bool) {
|
|
perr, ok := err.(*os.PathError)
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
return &redactedError{
|
|
error: err,
|
|
redacted: fmt.Errorf("%v: %w", perr.Op, perr.Err),
|
|
}, true
|
|
}
|
|
|
|
type redactedError struct {
|
|
error
|
|
redacted error
|
|
}
|